From 526f9da46368d9f764783696b4e993c8e1e265f4 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 31 Oct 2013 02:45:17 +0000 Subject: [PATCH 01/72] Epoll wrapper + selectors module. --- lib/posix/epoll.nim | 87 ++++++++++++++++ lib/pure/selectors.nim | 228 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 lib/posix/epoll.nim create mode 100644 lib/pure/selectors.nim diff --git a/lib/posix/epoll.nim b/lib/posix/epoll.nim new file mode 100644 index 0000000000..098838624d --- /dev/null +++ b/lib/posix/epoll.nim @@ -0,0 +1,87 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2013 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +const + EPOLLIN* = 0x00000001 + EPOLLPRI* = 0x00000002 + EPOLLOUT* = 0x00000004 + EPOLLERR* = 0x00000008 + EPOLLHUP* = 0x00000010 + EPOLLRDNORM* = 0x00000040 + EPOLLRDBAND* = 0x00000080 + EPOLLWRNORM* = 0x00000100 + EPOLLWRBAND* = 0x00000200 + EPOLLMSG* = 0x00000400 + EPOLLRDHUP* = 0x00002000 + EPOLLWAKEUP* = 1 shl 29 + EPOLLONESHOT* = 1 shl 30 + EPOLLET* = 1 shl 31 + +# Valid opcodes ( "op" parameter ) to issue to epoll_ctl(). + +const + EPOLL_CTL_ADD* = 1 # Add a file descriptor to the interface. + EPOLL_CTL_DEL* = 2 # Remove a file descriptor from the interface. + EPOLL_CTL_MOD* = 3 # Change file descriptor epoll_event structure. + +type + epoll_data* {.pure, final.} = object + thePtr*: pointer + fd*: cint + u32*: uint32 + u64*: uint64 + + epoll_event* {.pure, final.} = object + events*: uint32 # Epoll events + data*: epoll_data # User data variable + +proc epoll_create*(size: cint): cint {.importc: "epoll_create", + header: "".} + ## Creates an epoll instance. Returns an fd for the new instance. + ## The "size" parameter is a hint specifying the number of file + ## descriptors to be associated with the new instance. The fd + ## returned by epoll_create() should be closed with close(). + +proc epoll_create1*(flags: cint): cint {.importc: "epoll_create1", + header: "".} + ## Same as epoll_create but with an FLAGS parameter. The unused SIZE + ## parameter has been dropped. + +proc epoll_ctl*(epfd: cint; op: cint; fd: cint; event: ptr epoll_event): cint {. + importc: "epoll_ctl", header: "".} + ## Manipulate an epoll instance "epfd". Returns 0 in case of success, + ## -1 in case of error ( the "errno" variable will contain the + ## specific error code ) The "op" parameter is one of the EPOLL_CTL_* + ## constants defined above. The "fd" parameter is the target of the + ## operation. The "event" parameter describes which events the caller + ## is interested in and any associated user data. + +proc epoll_wait*(epfd: cint; events: ptr epoll_event; maxevents: cint; + timeout: cint): cint {.importc: "epoll_wait", + header: "".} + ## Wait for events on an epoll instance "epfd". Returns the number of + ## triggered events returned in "events" buffer. Or -1 in case of + ## error with the "errno" variable set to the specific error code. The + ## "events" parameter is a buffer that will contain triggered + ## events. The "maxevents" is the maximum number of events to be + ## returned ( usually size of "events" ). The "timeout" parameter + ## specifies the maximum wait time in milliseconds (-1 == infinite). + ## + ## This function is a cancellation point and therefore not marked with + ## __THROW. + + +#proc epoll_pwait*(epfd: cint; events: ptr epoll_event; maxevents: cint; +# timeout: cint; ss: ptr sigset_t): cint {. +# importc: "epoll_pwait", header: "".} +# Same as epoll_wait, but the thread's signal mask is temporarily +# and atomically replaced with the one provided as parameter. +# +# This function is a cancellation point and therefore not marked with +# __THROW. diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim new file mode 100644 index 0000000000..613f2d57b4 --- /dev/null +++ b/lib/pure/selectors.nim @@ -0,0 +1,228 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2013 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# TODO: Docs. + +import tables, os, unsigned +when defined(windows): + import winlean +else: + import posix + +type + TEvent* = enum + EvRead, EvWrite + + TSelectorKey* = object + fd: cint + events: set[TEvent] + data: PObject + + TReadyInfo* = tuple[key: TSelectorKey, events: set[TEvent]] + + PSelector* = ref object of PObject ## Selector interface. + fds*: TTable[cint, TSelectorKey] + registerImpl*: proc (s: PSelector, fd: cint, events: set[TEvent], + data: var PObject): TSelectorKey {.nimcall, tags: [FWriteIO].} + unregisterImpl*: proc (s: PSelector, fd: cint): TSelectorKey {.nimcall, tags: [FWriteIO].} + selectImpl*: proc (s: PSelector, timeout: int): seq[TReadyInfo] {.nimcall, tags: [FReadIO].} + closeImpl*: proc (s: PSelector) {.nimcall.} + +template initSelector(r: expr) = + new r + r.fds = initTable[cint, TSelectorKey]() + +proc register*(s: PSelector, fd: cint, events: set[TEvent], data: var PObject): + TSelectorKey = + if not s.registerImpl.isNil: result = s.registerImpl(s, fd, events, data) + +proc unregister*(s: PSelector, fd: cint): TSelectorKey = + ## + ## **Note:** For the ``epoll`` implementation the resulting ``TSelectorKey`` + ## will only have the ``fd`` field set. This is an optimisation and may + ## change in the future if a viable use case is presented. + if not s.unregisterImpl.isNil: result = s.unregisterImpl(s, fd) + +proc select*(s: PSelector, timeout = 500): seq[TReadyInfo] = + ## + ## **Note:** For the ``epoll`` implementation the resulting + ## ``TSelectorKey.events`` will not contain the original events. + ## TODO: This breaks what TSelectorKey means... it's not a key anymore. + ## Rename to TSelectorInfo? + + if not s.selectImpl.isNil: result = s.selectImpl(s, timeout) + +proc close*(s: PSelector) = + if not s.closeImpl.isNil: s.closeImpl(s) + +# ---- Select() ---------------------------------------------------------------- + +type + PSelectSelector* = ref object of PSelector ## Implementation of select() + +proc ssRegister(s: PSelector, fd: cint, events: set[TEvent], + data: var PObject): TSelectorKey = + if s.fds.hasKey(fd): + raise newException(EInvalidValue, "FD already exists in selector.") + var sk = TSelectorKey(fd: fd, events: events, data: data) + s.fds[fd] = sk + result = sk + +proc ssUnregister(s: PSelector, fd: cint): TSelectorKey = + result = s.fds[fd] + s.fds.del(fd) + +proc ssClose(s: PSelector) = nil + +proc timeValFromMilliseconds(timeout: int): TTimeVal = + if timeout != -1: + var seconds = timeout div 1000 + result.tv_sec = seconds.int32 + result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + +proc createFdSet(rd, wr: var TFdSet, fds: TTable[cint, TSelectorKey], + m: var int) = + FD_ZERO(rd); FD_ZERO(wr) + for k, v in pairs(fds): + if EvRead in v.events: + m = max(m, int(k)) + FD_SET(k, rd) + if EvWrite in v.events: + m = max(m, int(k)) + FD_SET(k, wr) + +proc getReadyFDs(rd, wr: var TFdSet, fds: TTable[cint, TSelectorKey]): + seq[TReadyInfo] = + result = @[] + for k, v in pairs(fds): + var events: set[TEvent] = {} + if FD_ISSET(k, rd) != 0'i32: + events = events + {EvRead} + if FD_ISSET(k, wr) != 0'i32: + events = events + {EvWrite} + result.add((v, events)) + +proc select(fds: TTable[cint, TSelectorKey], timeout = 500): + seq[TReadyInfo] = + var tv {.noInit.}: TTimeVal = timeValFromMilliseconds(timeout) + + var rd, wr: TFdSet + var m = 0 + createFdSet(rd, wr, fds, m) + + var retCode = 0 + if timeout != -1: + retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, addr(tv))) + else: + retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, nil)) + + if retCode < 0: + OSError(OSLastError()) + elif retCode == 0: + return @[] + else: + return getReadyFDs(rd, wr, fds) + +proc ssSelect(s: PSelector, timeout: int): seq[TReadyInfo] = + result = select(s.fds, timeout) + +proc newSelectSelector*(): PSelectSelector = + initSelector(result) + result.registerImpl = ssRegister + result.unregisterImpl = ssUnregister + result.selectImpl = ssSelect + result.closeImpl = ssClose + +# ---- Epoll ------------------------------------------------------------------- + +when defined(linux): + import epoll + type + PEpollSelector = ref object of PSelector + epollFD: cint + + proc esRegister(s: PSelector, fd: cint, events: set[TEvent], + data: var PObject): TSelectorKey = + var es = PEpollSelector(s) + var event: epoll_event + if EvRead in events: + event.events = EPOLLIN + if EvWrite in events: + event.events = event.events or EPOLLOUT + event.data.fd = fd + event.data.thePtr = addr(data) + + if epoll_ctl(es.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: + OSError(OSLastError()) + + result = TSelectorKey(fd: fd, events: events, data: data) + + proc esUnregister(s: PSelector, fd: cint): TSelectorKey = + # We cannot find out the information about this ``fd`` from the epoll + # context. As such I will simply return an almost empty TSelectorKey. + var es = PEpollSelector(s) + if epoll_ctl(es.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: + OSError(OSLastError()) + # We could fill in the ``fds`` TTable to get the info, but that wouldn't + # be nice for our memory. + result = TSelectorKey(fd: fd, events: {}, data: nil) + + proc esClose(s: PSelector) = + var es = PEpollSelector(s) + if es.epollFD.close() != 0: OSError(OSLastError()) + + proc esSelect(s: PSelector, timeout: int): seq[TReadyInfo] = + result = @[] + var es = PEpollSelector(s) + + var events: array[64, epoll_event] + let evNum = epoll_wait(es.epollFD, addr events[0], 64.cint, timeout.cint) + if evNum < 0: OSError(OSLastError()) + for i in 0 .. 63: + var evSet: set[TEvent] = {} + if (events[i].events and EPOLLIN) == 1: evSet = evSet + {EvRead} + if (events[i].events and EPOLLOUT) == 1: evSet = evSet + {EvWrite} + let selectorKey = TSelectorKey(fd: events[i].data.fd, events: evSet, + data: cast[PObject](events[i].data.thePtr)) + result.add((selectorKey, evSet)) + + proc newEpollSelector*(): PEpollSelector = + new result + result.epollFD = epoll_create(64) + if result.epollFD < 0: + OSError(OSLastError()) + +when isMainModule: + # Select() + import sockets + type + PSockWrapper = ref object of PObject + sock: TSocket + + + var sock = socket() + sock.connect("irc.freenode.net", TPort(6667)) + + var selector = newSelectSelector() + var data = PSockWrapper(sock: sock) + let key = selector.register(sock.getFD.cint, {EvRead, EvWrite}, data) + while true: + let ready = selector.select() + echo ready.len + if ready.len > 0: echo ready[0].repr + + + + + + + + + + \ No newline at end of file From 471c0aa6349b7ad41d70b64ed370106bed9f0d01 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 31 Oct 2013 15:07:14 +0000 Subject: [PATCH 02/72] Epoll now works. --- lib/posix/epoll.nim | 13 ++++---- lib/pure/selectors.nim | 73 +++++++++++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 32 deletions(-) diff --git a/lib/posix/epoll.nim b/lib/posix/epoll.nim index 098838624d..d50394f604 100644 --- a/lib/posix/epoll.nim +++ b/lib/posix/epoll.nim @@ -31,13 +31,14 @@ const EPOLL_CTL_MOD* = 3 # Change file descriptor epoll_event structure. type - epoll_data* {.pure, final.} = object - thePtr*: pointer - fd*: cint - u32*: uint32 - u64*: uint64 + epoll_data* {.importc: "union epoll_data", + header: "", pure, final.} = object # TODO: This is actually a union. + thePtr* {.importc: "ptr".}: pointer # \ + #fd*: cint + #u32*: uint32 + #u64*: uint64 - epoll_event* {.pure, final.} = object + epoll_event* {.importc: "struct epoll_event", header: "", pure, final.} = object events*: uint32 # Epoll events data*: epoll_data # User data variable diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 613f2d57b4..83c158da14 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -29,7 +29,7 @@ type PSelector* = ref object of PObject ## Selector interface. fds*: TTable[cint, TSelectorKey] registerImpl*: proc (s: PSelector, fd: cint, events: set[TEvent], - data: var PObject): TSelectorKey {.nimcall, tags: [FWriteIO].} + data: PObject): TSelectorKey {.nimcall, tags: [FWriteIO].} unregisterImpl*: proc (s: PSelector, fd: cint): TSelectorKey {.nimcall, tags: [FWriteIO].} selectImpl*: proc (s: PSelector, timeout: int): seq[TReadyInfo] {.nimcall, tags: [FReadIO].} closeImpl*: proc (s: PSelector) {.nimcall.} @@ -38,7 +38,7 @@ template initSelector(r: expr) = new r r.fds = initTable[cint, TSelectorKey]() -proc register*(s: PSelector, fd: cint, events: set[TEvent], data: var PObject): +proc register*(s: PSelector, fd: cint, events: set[TEvent], data: PObject): TSelectorKey = if not s.registerImpl.isNil: result = s.registerImpl(s, fd, events, data) @@ -51,10 +51,10 @@ proc unregister*(s: PSelector, fd: cint): TSelectorKey = proc select*(s: PSelector, timeout = 500): seq[TReadyInfo] = ## - ## **Note:** For the ``epoll`` implementation the resulting - ## ``TSelectorKey.events`` will not contain the original events. - ## TODO: This breaks what TSelectorKey means... it's not a key anymore. - ## Rename to TSelectorInfo? + ## The ``events`` field of the returned ``key`` contains the original events + ## for which the ``fd`` was bound. This is contrary to the ``events`` field + ## of the ``TReadyInfo`` tuple which determines which events are ready + ## on the ``fd``. if not s.selectImpl.isNil: result = s.selectImpl(s, timeout) @@ -67,7 +67,7 @@ type PSelectSelector* = ref object of PSelector ## Implementation of select() proc ssRegister(s: PSelector, fd: cint, events: set[TEvent], - data: var PObject): TSelectorKey = + data: PObject): TSelectorKey = if s.fds.hasKey(fd): raise newException(EInvalidValue, "FD already exists in selector.") var sk = TSelectorKey(fd: fd, events: events, data: data) @@ -144,19 +144,29 @@ proc newSelectSelector*(): PSelectSelector = when defined(linux): import epoll type - PEpollSelector = ref object of PSelector + PEpollSelector* = ref object of PSelector epollFD: cint + events: array[64, ptr epoll_event] + + TDataWrapper = object + fd: cint + boundEvents: set[TEvent] ## The events which ``fd`` listens for. + data: PObject ## User object. proc esRegister(s: PSelector, fd: cint, events: set[TEvent], - data: var PObject): TSelectorKey = + data: PObject): TSelectorKey = var es = PEpollSelector(s) var event: epoll_event if EvRead in events: event.events = EPOLLIN if EvWrite in events: event.events = event.events or EPOLLOUT - event.data.fd = fd - event.data.thePtr = addr(data) + + var dw = cast[ptr TDataWrapper](alloc0(sizeof(TDataWrapper))) # TODO: This needs to be dealloc'd + dw.fd = fd + dw.boundEvents = events + dw.data = data + event.data.thePtr = dw if epoll_ctl(es.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: OSError(OSLastError()) @@ -176,47 +186,58 @@ when defined(linux): proc esClose(s: PSelector) = var es = PEpollSelector(s) if es.epollFD.close() != 0: OSError(OSLastError()) + dealloc(addr es.events) # TODO: Test this proc esSelect(s: PSelector, timeout: int): seq[TReadyInfo] = result = @[] var es = PEpollSelector(s) - var events: array[64, epoll_event] - let evNum = epoll_wait(es.epollFD, addr events[0], 64.cint, timeout.cint) + let evNum = epoll_wait(es.epollFD, es.events[0], 64.cint, timeout.cint) if evNum < 0: OSError(OSLastError()) - for i in 0 .. 63: + if evNum == 0: return @[] + for i in 0 .. 0: echo ready[0].repr - + if ready.len > 0: echo ready[0].events + i.inc + if i == 6: + selector.close() + break From 3503f1ca3395dfe64ad302717914218db72b402f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sat, 21 Dec 2013 18:05:19 +0100 Subject: [PATCH 03/72] Normalize whitespace in osproc.nim. --- lib/pure/osproc.nim | 58 ++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 61b940ce83..065d940408 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -109,8 +109,8 @@ proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [FExecIO].} proc startProcess*(command: string, workingDir: string = "", args: openarray[string] = [], - env: PStringTable = nil, - options: set[TProcessOption] = {poStdErrToStdOut}): + env: PStringTable = nil, + options: set[TProcessOption] = {poStdErrToStdOut}): PProcess {.rtl, extern: "nosp$1", tags: [FExecIO, FReadEnv].} ## Starts a process. `Command` is the executable file, `workingDir` is the ## process's working directory. If ``workingDir == ""`` the current directory @@ -157,7 +157,7 @@ proc processID*(p: PProcess): int {.rtl, extern: "nosp$1".} = ## returns `p`'s process ID. return p.id -proc waitForExit*(p: PProcess, timeout: int = -1): int {.rtl, +proc waitForExit*(p: PProcess, timeout: int = -1): int {.rtl, extern: "nosp$1", tags: [].} ## waits for the process to finish and returns `p`'s error code. @@ -167,19 +167,19 @@ proc peekExitCode*(p: PProcess): int {.tags: [].} proc inputStream*(p: PProcess): PStream {.rtl, extern: "nosp$1", tags: [].} ## returns ``p``'s input stream for writing to. ## - ## **Warning**: The returned `PStream` should not be closed manually as it + ## **Warning**: The returned `PStream` should not be closed manually as it ## is closed when closing the PProcess ``p``. proc outputStream*(p: PProcess): PStream {.rtl, extern: "nosp$1", tags: [].} ## returns ``p``'s output stream for reading from. ## - ## **Warning**: The returned `PStream` should not be closed manually as it + ## **Warning**: The returned `PStream` should not be closed manually as it ## is closed when closing the PProcess ``p``. proc errorStream*(p: PProcess): PStream {.rtl, extern: "nosp$1", tags: [].} ## returns ``p``'s error stream for reading from. ## - ## **Warning**: The returned `PStream` should not be closed manually as it + ## **Warning**: The returned `PStream` should not be closed manually as it ## is closed when closing the PProcess ``p``. proc inputHandle*(p: PProcess): TFileHandle {.rtl, extern: "nosp$1", @@ -245,7 +245,7 @@ proc countProcessors*(): int {.rtl, extern: "nosp$1".} = proc execProcesses*(cmds: openArray[string], options = {poStdErrToStdOut, poParentStreams}, - n = countProcessors()): int {.rtl, extern: "nosp$1", + n = countProcessors()): int {.rtl, extern: "nosp$1", tags: [FExecIO, FTime, FReadEnv].} = ## executes the commands `cmds` in parallel. Creates `n` processes ## that execute in parallel. The highest return value of all processes @@ -253,7 +253,7 @@ proc execProcesses*(cmds: openArray[string], when defined(posix): # poParentStreams causes problems on Posix, so we simply disable it: var options = options - {poParentStreams} - + assert n > 0 if n > 1: var q: seq[PProcess] @@ -441,7 +441,7 @@ when defined(Windows) and not defined(useNimRtl): var ee = newWideCString(e) var wwd = newWideCString(wd) success = winlean.CreateProcessW(nil, - tmp, nil, nil, 1, NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT, + tmp, nil, nil, 1, NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT, ee, wwd, SI, ProcInfo) else: success = winlean.CreateProcessA(nil, @@ -495,7 +495,7 @@ when defined(Windows) and not defined(useNimRtl): proc peekExitCode(p: PProcess): int = var b = waitForSingleObject(p.FProcessHandle, 50) == WAIT_TIMEOUT if b: result = -1 - else: + else: var res: int32 discard GetExitCodeProcess(p.FProcessHandle, res) return res @@ -538,13 +538,13 @@ when defined(Windows) and not defined(useNimRtl): result = -1 discard CloseHandle(Process) - proc select(readfds: var seq[PProcess], timeout = 500): int = + proc select(readfds: var seq[PProcess], timeout = 500): int = assert readfds.len <= MAXIMUM_WAIT_OBJECTS var rfds: TWOHandleArray for i in 0..readfds.len()-1: rfds[i] = readfds[i].FProcessHandle - - var ret = waitForMultipleObjects(readfds.len.int32, + + var ret = waitForMultipleObjects(readfds.len.int32, addr(rfds), 0'i32, timeout.int32) case ret of WAIT_TIMEOUT: @@ -595,7 +595,7 @@ elif not defined(useNimRtl): result[i] = cast[cstring](alloc(x.len+1)) copyMem(result[i], addr(x[0]), x.len+1) inc(i) - + proc startProcess(command: string, workingDir: string = "", args: openarray[string] = [], @@ -609,23 +609,23 @@ elif not defined(useNimRtl): if pipe(p_stdin) != 0'i32 or pipe(p_stdout) != 0'i32 or pipe(p_stderr) != 0'i32: OSError(OSLastError()) - + var pid: TPid when defined(posix_spawn) and not defined(useFork): var attr: Tposix_spawnattr var fops: Tposix_spawn_file_actions - template chck(e: expr) = + template chck(e: expr) = if e != 0'i32: OSError(OSLastError()) chck posix_spawn_file_actions_init(fops) chck posix_spawnattr_init(attr) - + var mask: Tsigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) chck posix_spawnattr_setpgroup(attr, 0'i32) - + chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or POSIX_SPAWN_SETSIGMASK or POSIX_SPAWN_SETPGROUP) @@ -640,7 +640,7 @@ elif not defined(useNimRtl): chck posix_spawn_file_actions_adddup2(fops, p_stdout[writeIdx], 2) else: chck posix_spawn_file_actions_adddup2(fops, p_stderr[writeIdx], 2) - + var e = if env == nil: EnvToCStringArray() else: ToCStringArray(env) var a: cstringArray var res: cint @@ -659,7 +659,7 @@ elif not defined(useNimRtl): chck res else: - + Pid = fork() if Pid < 0: OSError(OSLastError()) if pid == 0: @@ -790,13 +790,13 @@ elif not defined(useNimRtl): proc execCmd(command: string): int = result = csystem(command) - proc createFdSet(fd: var TFdSet, s: seq[PProcess], m: var int) = + proc createFdSet(fd: var TFdSet, s: seq[PProcess], m: var int) = FD_ZERO(fd) - for i in items(s): + for i in items(s): m = max(m, int(i.outHandle)) FD_SET(cint(i.outHandle), fd) - - proc pruneProcessSet(s: var seq[PProcess], fd: var TFdSet) = + + proc pruneProcessSet(s: var seq[PProcess], fd: var TFdSet) = var i = 0 var L = s.len while i < L: @@ -807,26 +807,26 @@ elif not defined(useNimRtl): inc(i) setLen(s, L) - proc select(readfds: var seq[PProcess], timeout = 500): int = + proc select(readfds: var seq[PProcess], timeout = 500): int = var tv: TTimeVal tv.tv_sec = 0 tv.tv_usec = timeout * 1000 - + var rd: TFdSet var m = 0 createFdSet((rd), readfds, m) - + if timeout != -1: result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv))) else: result = int(select(cint(m+1), addr(rd), nil, nil, nil)) - + pruneProcessSet(readfds, (rd)) proc execCmdEx*(command: string, options: set[TProcessOption] = { poStdErrToStdOut, poUseShell}): tuple[ - output: TaintedString, + output: TaintedString, exitCode: int] {.tags: [FExecIO, FReadIO].} = ## a convenience proc that runs the `command`, grabs all its output and ## exit code and returns both. From bdb5c4ad35812d1b065f8cfb3cabf1bf6ab6f4dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sat, 21 Dec 2013 18:15:11 +0100 Subject: [PATCH 04/72] Introduce poEvalCommand, poUsePath, fix remaining quoting issues. - poUsePath is now an alias for poUseShell. - poEvalCommand should be used when shell evaluation is really needed. It passes `command` directly to shell/winapi. Requires `args` parameter to be empty. --- lib/posix/posix.nim | 3 +- lib/pure/osproc.nim | 120 +++++++++++++++++++++++--------------------- 2 files changed, 63 insertions(+), 60 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 806c255ee9..b0324b780b 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -2085,6 +2085,7 @@ proc execv*(a1: cstring, a2: cstringArray): cint {.importc, header: "" proc execve*(a1: cstring, a2, a3: cstringArray): cint {. importc, header: "".} proc execvp*(a1: cstring, a2: cstringArray): cint {.importc, header: "".} +proc execvpe*(a1: cstring, a2: cstringArray, a3: cstringArray): cint {.importc, header: "".} proc fchown*(a1: cint, a2: Tuid, a3: Tgid): cint {.importc, header: "".} proc fchdir*(a1: cint): cint {.importc, header: "".} proc fdatasync*(a1: cint): cint {.importc, header: "".} @@ -2565,5 +2566,3 @@ proc poll*(a1: ptr Tpollfd, a2: Tnfds, a3: int): cint {. proc realpath*(name, resolved: CString): CString {. importc: "realpath", header: "".} - - diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 065d940408..0e6ff04cf9 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -13,7 +13,7 @@ include "system/inclrtl" import - strutils, os, strtabs, streams + strutils, os, strtabs, streams, sequtils when defined(windows): import winlean @@ -36,11 +36,17 @@ type TProcessOption* = enum ## options that can be passed `startProcess` poEchoCmd, ## echo the command before execution - poUseShell, ## use the shell to execute the command; NOTE: This - ## often creates a security hole! + poUsePath, ## Asks system to search for executable using PATH environment + ## variable. + ## On Windows, this is the default. + poEvalCommand, ## Pass `command` directly to the shell, without quoting. + ## Use it only if `command` comes from trused source. poStdErrToStdOut, ## merge stdout and stderr to the stdout stream poParentStreams ## use the parent's streams +template poUseShell*: TProcessOption {.deprecated.} = poUsePath + ## Deprecated alias for poUsePath. + proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote s, so it can be safely passed to Windows API. ## Based on Python's subprocess.list2cmdline @@ -94,12 +100,17 @@ proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = {.error:"quoteShell is not supported on your system".} proc execProcess*(command: string, + args: openarray[string] = [], + env: PStringTable = nil, options: set[TProcessOption] = {poStdErrToStdOut, - poUseShell}): TaintedString {. + poUsePath, + poEvalCommand}): TaintedString {. rtl, extern: "nosp$1", tags: [FExecIO, FReadIO].} ## A convenience procedure that executes ``command`` with ``startProcess`` ## and returns its output as a string. + ## WARNING: this function uses poEvalCommand by default for backward compatibility. + ## Make sure to pass options explicitly. proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [FExecIO].} ## Executes ``command`` and returns its error code. Standard input, output, @@ -127,16 +138,10 @@ proc startProcess*(command: string, ## but ``EOS`` is raised in case of an error. proc startCmd*(command: string, options: set[TProcessOption] = { - poStdErrToStdOut, poUseShell}): PProcess {. - tags: [FExecIO, FReadEnv].} = - ## a simpler version of `startProcess` that parses the command line into - ## program and arguments and then calls `startProcess` with the empty string - ## for `workingDir` and the nil string table for `env`. - var c = parseCmdLine(command) - var a: seq[string] - newSeq(a, c.len-1) # avoid slicing for now (still unstable) - for i in 1 .. c.len-1: a[i-1] = c[i] - result = startProcess(command=c[0], args=a, options=options) + poStdErrToStdOut, poUsePath}): PProcess {. + tags: [FExecIO, FReadEnv], deprecated.} = + ## Deprecated - use `startProcess` directly. + result = startProcess(command=command, options=options + {poEvalCommand}) proc close*(p: PProcess) {.rtl, extern: "nosp$1", tags: [].} ## When the process has finished executing, cleanup related handles @@ -246,7 +251,7 @@ proc countProcessors*(): int {.rtl, extern: "nosp$1".} = proc execProcesses*(cmds: openArray[string], options = {poStdErrToStdOut, poParentStreams}, n = countProcessors()): int {.rtl, extern: "nosp$1", - tags: [FExecIO, FTime, FReadEnv].} = + tags: [FExecIO, FTime, FReadEnv]} = ## executes the commands `cmds` in parallel. Creates `n` processes ## that execute in parallel. The highest return value of all processes ## is returned. @@ -307,13 +312,17 @@ proc select*(readfds: var seq[PProcess], timeout = 500): int when not defined(useNimRtl): proc execProcess(command: string, + args: openarray[string] = [], + env: PStringTable = nil, options: set[TProcessOption] = {poStdErrToStdOut, - poUseShell}): TaintedString = - var p = startCmd(command, options=options) + poUsePath, + poEvalCommand}): TaintedString = + var p = startProcess(command, args=args, env=env, options=options) var outp = outputStream(p) result = TaintedString"" var line = newStringOfCap(120).TaintedString while true: + # FIXME: converts CR-LF to LF. if outp.readLine(line): result.string.add(line.string) result.string.add("\n") @@ -427,8 +436,9 @@ when defined(Windows) and not defined(useNimRtl): result.errHandle = TFileHandle(si.hStdError) var cmdl: cstring - when false: # poUseShell in options: - cmdl = buildCommandLine(getEnv("COMSPEC"), @["/c", command] & args) + if poEvalCommand in options: + cmdl = command + assert args.len == 0 else: cmdl = buildCommandLine(command, args) var wd: cstring = nil @@ -455,7 +465,6 @@ when defined(Windows) and not defined(useNimRtl): FileClose(si.hStdError) if e != nil: dealloc(e) - dealloc(cmdl) if success == 0: OSError(lastError) # Close the handle now so anyone waiting is woken: discard closeHandle(procInfo.hThread) @@ -561,22 +570,7 @@ elif not defined(useNimRtl): readIdx = 0 writeIdx = 1 - proc addCmdArgs(command: string, args: openarray[string]): string = - result = quoteShell(command) - for i in 0 .. high(args): - add(result, " ") - add(result, quoteShell(args[i])) - - proc toCStringArray(b, a: openarray[string]): cstringArray = - result = cast[cstringArray](alloc0((a.len + b.len + 1) * sizeof(cstring))) - for i in 0..high(b): - result[i] = cast[cstring](alloc(b[i].len+1)) - copyMem(result[i], cstring(b[i]), b[i].len+1) - for i in 0..high(a): - result[i+b.len] = cast[cstring](alloc(a[i].len+1)) - copyMem(result[i+b.len], cstring(a[i]), a[i].len+1) - - proc ToCStringArray(t: PStringTable): cstringArray = + proc envToCStringArray(t: PStringTable): cstringArray = result = cast[cstringArray](alloc0((t.len + 1) * sizeof(cstring))) var i = 0 for key, val in pairs(t): @@ -585,7 +579,7 @@ elif not defined(useNimRtl): copyMem(result[i], addr(x[0]), x.len+1) inc(i) - proc EnvToCStringArray(): cstringArray = + proc envToCStringArray(): cstringArray = var counter = 0 for key, val in envPairs(): inc counter result = cast[cstringArray](alloc0((counter + 1) * sizeof(cstring))) @@ -610,6 +604,21 @@ elif not defined(useNimRtl): pipe(p_stderr) != 0'i32: OSError(OSLastError()) + var sys_command: string + var sys_args_raw: seq[string] + if poEvalCommand in options: + sys_command = "/bin/sh" + sys_args_raw = @[sys_command, "-c", command] + assert args.len == 0 + else: + sys_command = command + sys_args_raw = @[command] + for arg in args.items: + sys_args_raw.add arg + + var sys_args = allocCStringArray(sys_args_raw) + finally: deallocCStringArray(sys_args) + var pid: TPid when defined(posix_spawn) and not defined(useFork): var attr: Tposix_spawnattr @@ -641,19 +650,15 @@ elif not defined(useNimRtl): else: chck posix_spawn_file_actions_adddup2(fops, p_stderr[writeIdx], 2) - var e = if env == nil: EnvToCStringArray() else: ToCStringArray(env) - var a: cstringArray + var sys_env = if env == nil: EnvToCStringArray() else: EnvToCStringArray(env) var res: cint + # This is incorrect! if workingDir.len > 0: os.setCurrentDir(workingDir) - if poUseShell notin options: - a = toCStringArray([extractFilename(command)], args) - res = posix_spawn(pid, command, fops, attr, a, e) + if poUsePath in options: + res = posix_spawnp(pid, sys_command, fops, attr, sys_args, sys_env) else: - var x = addCmdArgs(command, args) - a = toCStringArray(["sh", "-c"], [x]) - res = posix_spawn(pid, "/bin/sh", fops, attr, a, e) - deallocCStringArray(a) - deallocCStringArray(e) + res = posix_spawn(pid, sys_command, fops, attr, sys_args, sys_env) + deallocCStringArray(sys_env) discard posix_spawn_file_actions_destroy(fops) discard posix_spawnattr_destroy(attr) chck res @@ -680,19 +685,18 @@ elif not defined(useNimRtl): if setpgid(0, 0) == -1: quit("setpgid call failed: " & $strerror(errno)) if workingDir.len > 0: os.setCurrentDir(workingDir) - if poUseShell notin options: - var a = toCStringArray([extractFilename(command)], args) - if env == nil: - discard execv(command, a) + + if env == nil: + if poUsePath in options: + discard execvp(sys_command, sys_args) else: - discard execve(command, a, ToCStringArray(env)) + discard execv(sys_command, sys_args) else: - var x = addCmdArgs(command, args) - var a = toCStringArray(["sh", "-c"], [x]) - if env == nil: - discard execv("/bin/sh", a) + var c_env = envToCStringArray(env) + if poUsePath in options: + discard execvpe(sys_command, sys_args, c_env) else: - discard execve("/bin/sh", a, ToCStringArray(env)) + discard execve(sys_command, sys_args, c_env) # too risky to raise an exception here: quit("execve call failed: " & $strerror(errno)) # Parent process. Copy process information. @@ -825,7 +829,7 @@ elif not defined(useNimRtl): proc execCmdEx*(command: string, options: set[TProcessOption] = { - poStdErrToStdOut, poUseShell}): tuple[ + poStdErrToStdOut, poUsePath}): tuple[ output: TaintedString, exitCode: int] {.tags: [FExecIO, FReadIO].} = ## a convenience proc that runs the `command`, grabs all its output and From d63eca96e5976f9125f89909b1e2db02b69654da Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Wed, 22 Jan 2014 23:01:50 +0100 Subject: [PATCH 05/72] Adds AllChars constant to strutils. --- lib/pure/strutils.nim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index de8dc5e51d..b63224cec4 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -45,6 +45,16 @@ const NewLines* = {'\13', '\10'} ## the set of characters a newline terminator can start with + AllChars* = {'\x00'..'\xFF'} + ## A set with all the possible characters. Not very useful by its own, you + ## can use it to create *inverted* sets to make the ``find()`` proc find + ## **invalid** characters in strings. Example: + ## + ## .. code-block:: nimrod + ## let invalid = AllChars - Digits + ## doAssert "01234".find(invalid) == -1 + ## doAssert "01A34".find(invalid) == 2 + proc toLower*(c: char): char {.noSideEffect, procvar, rtl, extern: "nsuToLowerChar".} = ## Converts `c` into lower case. This works only for the letters A-Z. From 7956737ef1f4819190dc86e0c7b9434ba58a9390 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Sun, 26 Jan 2014 16:39:20 +0100 Subject: [PATCH 06/72] Adds note about manual memory handling to GC doc. --- doc/gc.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/doc/gc.txt b/doc/gc.txt index 13498afaa1..18fb03b6df 100644 --- a/doc/gc.txt +++ b/doc/gc.txt @@ -107,3 +107,20 @@ that up to 100 objects are traversed and freed before it checks again. Thus ``workPackage`` affects the timing granularity and may need to be tweaked in highly specialized environments or for older hardware. + +Keeping track of memory +----------------------- + +If you need to pass around memory allocated by Nimrod to C, you can use the +procs ``GC_ref`` and ``GC_unref`` to mark objects as referenced to avoid them +being freed by the GC. Other useful procs from `system `_ you can +use to keep track of memory are: + +* getTotalMem(): returns the amount of total memory managed by the GC. +* getOccupiedMem(): bytes reserved by the GC and used by objects. +* getFreeMem(): bytes reserved by the GC and not in use. + +In addition to ``GC_ref`` and ``GC_unref`` you can avoid the GC by manually +allocating memory with procs like ``alloc``, ``allocShared``, or +``allocCStringArray``. The GC won't try to free them, you need to call their +respective *dealloc* pairs when you are done with them or they will leak. From 92f7c479b9c8db6fdf5044a61025dd88529ce8b6 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 27 Jan 2014 17:48:39 +0000 Subject: [PATCH 07/72] Fixes #848. --- compiler/types.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/types.nim b/compiler/types.nim index 4a53a84c97..cd703474e6 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -451,7 +451,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = of tyProc: "proc" of tyObject: "object" of tyTuple: "tuple" - else: (internalAssert false; "") + else: (internalAssert(false); "") of tyUserTypeClassInst: let body = t.base result = body.sym.name.s & "[" From 9344e9fbf7d4c056710c152b8e811632914f8043 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 27 Jan 2014 18:20:08 +0000 Subject: [PATCH 08/72] Finished logging module. --- devel/logging.nim | 210 ---------------------------------- doc/lib.txt | 3 + lib/pure/logging.nim | 267 +++++++++++++++++++++++++++++++++++++++++++ web/nimrod.ini | 2 +- 4 files changed, 271 insertions(+), 211 deletions(-) delete mode 100644 devel/logging.nim create mode 100644 lib/pure/logging.nim diff --git a/devel/logging.nim b/devel/logging.nim deleted file mode 100644 index a10478dab4..0000000000 --- a/devel/logging.nim +++ /dev/null @@ -1,210 +0,0 @@ -# -# -# Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements a simple logger. It is based on the following design: -## * Runtime log formating is a bug: Sooner or later every log file is parsed. -## * Keep it simple: If this library does not fullfill your needs, write your -## own. Trying to support every logging feature just leads to bloat. -## -## Format is:: -## -## DEBUG|INFO|... (2009-11-02 00:00:00)? (Component: )? Message -## -## - -import strutils, os, times - -type - TLevel* = enum ## logging level - lvlAll, ## all levels active - lvlDebug, ## debug level (and any above) active - lvlInfo, ## info level (and any above) active - lvlWarn, ## warn level (and any above) active - lvlError, ## error level (and any above) active - lvlFatal, ## fatal level (and any above) active - lvlNone - -const - LevelNames*: array [TLevel, string] = [ - "DEBUG", "DEBUG", "INFO", "WARN", "ERROR", "FATAL", "NONE" - ] - - defaultFmtStr = "" ## default string between log level and message per logger - verboseFmtStr = "$date $time " - -type - TLogger* = object of TObject ## abstract logger; the base type of all loggers - levelThreshold*: TLevel ## only messages of level >= levelThreshold - ## should be processed - fmtStr: string ## = defaultFmtStr by default, see substituteLog for $date etc. - - TConsoleLogger* = object of TLogger ## logger that writes the messages to the - ## console - - TFileLogger* = object of TLogger ## logger that writes the messages to a file - f: TFile - - # TODO: implement rolling log, will produce filename.1, filename.2 etc. - TRollingFileLogger* = object of TFileLogger ## logger that writes the - ## message to a file - maxLines: int # maximum number of lines - curLine : int - baseName: string # initial filename - logFiles: int # how many log files already created, e.g. basename.1, basename.2... - - - - -proc substituteLog*(frmt: string): string = - ## converts $date to the current date - ## converts $time to the current time - ## converts $app to getAppFilename() - ## converts - result = newStringOfCap(frmt.len + 20) - var i = 0 - while i < frmt.len: - if frmt[i] != '$': - result.add(frmt[i]) - inc(i) - else: - inc(i) - var v = "" - var app = getAppFilename() - while frmt[i] in IdentChars: - v.add(toLower(frmt[i])) - inc(i) - case v - of "date": result.add(getDateStr()) - of "time": result.add(getClockStr()) - of "app": result.add(app) - of "appdir": result.add(app.splitFile.dir) - of "appname": result.add(app.splitFile.name) - - - -method log*(L: ref TLogger, level: TLevel, - frmt: string, args: varargs[string, `$`]) = - ## override this method in custom loggers. Default implementation does - ## nothing. - nil - -method log*(L: ref TConsoleLogger, level: TLevel, - frmt: string, args: varargs[string, `$`]) = - Writeln(stdout, LevelNames[level], " ", substituteLog(L.fmtStr), frmt % args) - -method log*(L: ref TFileLogger, level: TLevel, - frmt: string, args: varargs[string, `$`]) = - Writeln(L.f, LevelNames[level], " ", substituteLog(L.fmtStr), frmt % args) - -proc defaultFilename*(): string = - ## returns the default filename for a logger - var (path, name, ext) = splitFile(getAppFilename()) - result = changeFileExt(path / name & "_" & getDateStr(), "log") - - - - -proc newConsoleLogger*(levelThreshold = lvlAll) : ref TConsoleLogger = - new result - result.fmtStr = defaultFmtStr - result.levelThreshold = levelThreshold - -proc newFileLogger*(filename = defaultFilename(), - mode: TFileMode = fmAppend, - levelThreshold = lvlAll): ref TFileLogger = - new(result) - result.levelThreshold = levelThreshold - result.f = open(filename, mode) - result.fmtStr = defaultFmtStr - -# ------ - -proc readLogLines(logger : ref TRollingFileLogger) = nil - #f.readLine # TODO read all lines, update curLine - - -proc newRollingFileLogger*(filename = defaultFilename(), - mode: TFileMode = fmReadWrite, - levelThreshold = lvlAll, - maxLines = 1000): ref TRollingFileLogger = - new(result) - result.levelThreshold = levelThreshold - result.fmtStr = defaultFmtStr - result.maxLines = maxLines - result.f = open(filename, mode) - result.curLine = 0 - - # TODO count all number files - # count lines in existing filename file - # if >= maxLines then rename to next numbered file and create new file - - #if mode in {fmReadWrite, fmReadWriteExisting}: - # readLogLines(result) - - - -method log*(L: ref TRollingFileLogger, level: TLevel, - frmt: string, args: varargs[string, `$`]) = - # TODO - # if more than maxlines, then set cursor to zero - - Writeln(L.f, LevelNames[level], " ", frmt % args) - -# -------- - -var - level* = lvlAll ## global log filter - handlers*: seq[ref TLogger] = @[] ## handlers with their own log levels - -proc logLoop(level: TLevel, frmt: string, args: varargs[string, `$`]) = - for logger in items(handlers): - if level >= logger.levelThreshold: - log(logger, level, frmt, args) - -template log*(level: TLevel, frmt: string, args: varargs[string, `$`]) = - ## logs a message of the given level - bind logLoop - bind `%` - bind logging.Level - - if level >= logging.Level: - logLoop(level, frmt, args) - -template debug*(frmt: string, args: varargs[string, `$`]) = - ## logs a debug message - log(lvlDebug, frmt, args) - -template info*(frmt: string, args: varargs[string, `$`]) = - ## logs an info message - log(lvlInfo, frmt, args) - -template warn*(frmt: string, args: varargs[string, `$`]) = - ## logs a warning message - log(lvlWarn, frmt, args) - -template error*(frmt: string, args: varargs[string, `$`]) = - ## logs an error message - log(lvlError, frmt, args) - -template fatal*(frmt: string, args: varargs[string, `$`]) = - ## logs a fatal error message - log(lvlFatal, frmt, args) - - -# -------------- - -when isMainModule: - var L = newConsoleLogger() - var fL = newFileLogger("test.log") - fL.fmtStr = verboseFmtStr - handlers.add(L) - handlers.add(fL) - info("hello", []) - - diff --git a/doc/lib.txt b/doc/lib.txt index ba0cb0a905..3214cdae21 100644 --- a/doc/lib.txt +++ b/doc/lib.txt @@ -341,6 +341,9 @@ Miscellaneous * `endians `_ This module contains helpers that deal with different byte orders. +* `logging `_ + This module implements a simple logger. + Database support ---------------- diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim new file mode 100644 index 0000000000..8158cbc2aa --- /dev/null +++ b/lib/pure/logging.nim @@ -0,0 +1,267 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Andreas Rumpf, Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements a simple logger. It has been designed to be as simple +## as possible to avoid bloat, if this library does not fullfill your needs, +## write your own. +## +## Format strings support the following variables which must be prefixed with +## the dollar operator (``$``): +## +## ============ ======================= +## Operator Output +## ============ ======================= +## $date Current date +## $time Current time +## $app ``os.getAppFilename()`` +## ============ ======================= +## +## +## The following example demonstrates logging to three different handlers +## simultaneously: +## +## .. code-block:: nimrod +## +## var L = newConsoleLogger() +## var fL = newFileLogger("test.log", fmtStr = verboseFmtStr) +## var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr) +## handlers.add(L) +## handlers.add(fL) +## handlers.add(rL) +## info("920410:52 accepted") +## warn("4 8 15 16 23 4-- Error") +## error("922044:16 SYSTEM FAILURE") +## fatal("SYSTEM FAILURE SYSTEM FAILURE") + +import strutils, os, times + +type + TLevel* = enum ## logging level + lvlAll, ## all levels active + lvlDebug, ## debug level (and any above) active + lvlInfo, ## info level (and any above) active + lvlWarn, ## warn level (and any above) active + lvlError, ## error level (and any above) active + lvlFatal, ## fatal level (and any above) active + lvlNone ## no levels active + +const + LevelNames*: array [TLevel, string] = [ + "DEBUG", "DEBUG", "INFO", "WARN", "ERROR", "FATAL", "NONE" + ] + + defaultFmtStr* = "" ## default string between log level and message per logger + verboseFmtStr* = "$date $time " + +type + PLogger* = ref object of PObject ## abstract logger; the base type of all loggers + levelThreshold*: TLevel ## only messages of level >= levelThreshold + ## should be processed + fmtStr: string ## = defaultFmtStr by default, see substituteLog for $date etc. + + PConsoleLogger* = ref object of PLogger ## logger that writes the messages to the + ## console + + PFileLogger* = ref object of PLogger ## logger that writes the messages to a file + f: TFile + + PRollingFileLogger* = ref object of PFileLogger ## logger that writes the + ## messages to a file and + ## performs log rotation + maxLines: int # maximum number of lines + curLine : int + baseName: string # initial filename + baseMode: TFileMode # initial file mode + logFiles: int # how many log files already created, e.g. basename.1, basename.2... + +proc substituteLog(frmt: string): string = + ## converts $date to the current date + ## converts $time to the current time + ## converts $app to getAppFilename() + ## converts + result = newStringOfCap(frmt.len + 20) + var i = 0 + while i < frmt.len: + if frmt[i] != '$': + result.add(frmt[i]) + inc(i) + else: + inc(i) + var v = "" + var app = getAppFilename() + while frmt[i] in IdentChars: + v.add(toLower(frmt[i])) + inc(i) + case v + of "date": result.add(getDateStr()) + of "time": result.add(getClockStr()) + of "app": result.add(app) + of "appdir": result.add(app.splitFile.dir) + of "appname": result.add(app.splitFile.name) + +method log*(logger: PLogger, level: TLevel, + frmt: string, args: varargs[string, `$`]) = + ## Override this method in custom loggers. Default implementation does + ## nothing. + nil + +method log*(logger: PConsoleLogger, level: TLevel, + frmt: string, args: varargs[string, `$`]) = + ## Logs to the console using ``logger`` only. + if level >= logger.levelThreshold: + writeln(stdout, LevelNames[level], " ", substituteLog(logger.fmtStr), + frmt % args) + +method log*(logger: PFileLogger, level: TLevel, + frmt: string, args: varargs[string, `$`]) = + ## Logs to a file using ``logger`` only. + if level >= logger.levelThreshold: + writeln(logger.f, LevelNames[level], " ", + substituteLog(logger.fmtStr), frmt % args) + +proc defaultFilename*(): string = + ## Returns the default filename for a logger. + var (path, name, ext) = splitFile(getAppFilename()) + result = changeFileExt(path / name, "log") + +proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr): PConsoleLogger = + ## Creates a new console logger. This logger logs to the console. + new result + result.fmtStr = fmtStr + result.levelThreshold = levelThreshold + +proc newFileLogger*(filename = defaultFilename(), + mode: TFileMode = fmAppend, + levelThreshold = lvlAll, + fmtStr = defaultFmtStr): PFileLogger = + ## Creates a new file logger. This logger logs to a file. + new(result) + result.levelThreshold = levelThreshold + result.f = open(filename, mode) + result.fmtStr = fmtStr + +# ------ + +proc countLogLines(logger: PRollingFileLogger): int = + result = 0 + for line in logger.f.lines(): + result.inc() + +proc countFiles(filename: string): int = + # Example: file.log.1 + result = 0 + let (dir, name, ext) = splitFile(filename) + for kind, path in walkDir(dir): + if kind == pcFile: + let llfn = name & ext & ExtSep + if path.extractFilename.startsWith(llfn): + let numS = path.extractFilename[llfn.len .. -1] + try: + let num = parseInt(numS) + if num > result: + result = num + except EInvalidValue: discard + +proc newRollingFileLogger*(filename = defaultFilename(), + mode: TFileMode = fmReadWrite, + levelThreshold = lvlAll, + fmtStr = defaultFmtStr, + maxLines = 1000): PRollingFileLogger = + ## Creates a new rolling file logger. Once a file reaches ``maxLines`` lines + ## a new log file will be started and the old will be renamed. + new(result) + result.levelThreshold = levelThreshold + result.fmtStr = defaultFmtStr + result.maxLines = maxLines + result.f = open(filename, mode) + result.curLine = 0 + result.baseName = filename + result.baseMode = mode + + result.logFiles = countFiles(filename) + + if mode == fmAppend: + # We need to get a line count because we will be appending to the file. + result.curLine = countLogLines(result) + +proc rotate(logger: PRollingFileLogger) = + let (dir, name, ext) = splitFile(logger.baseName) + for i in countdown(logger.logFiles, 0): + let srcSuff = if i != 0: ExtSep & $i else: "" + moveFile(dir / (name & ext & srcSuff), + dir / (name & ext & ExtSep & $(i+1))) + +method log*(logger: PRollingFileLogger, level: TLevel, + frmt: string, args: varargs[string, `$`]) = + ## Logs to a file using rolling ``logger`` only. + if level >= logger.levelThreshold: + if logger.curLine >= logger.maxLines: + logger.f.close() + rotate(logger) + logger.logFiles.inc + logger.curLine = 0 + logger.f = open(logger.baseName, logger.baseMode) + + writeln(logger.f, LevelNames[level], " ", frmt % args) + logger.curLine.inc + +# -------- + +var + level* = lvlAll ## global log filter + handlers*: seq[PLogger] = @[] ## handlers with their own log levels + +proc logLoop(level: TLevel, frmt: string, args: varargs[string, `$`]) = + for logger in items(handlers): + if level >= logger.levelThreshold: + log(logger, level, frmt, args) + +template log*(level: TLevel, frmt: string, args: varargs[string, `$`]) = + ## Logs a message to all registered handlers at the given level. + bind logLoop + bind `%` + bind logging.Level + + if level >= logging.Level: + logLoop(level, frmt, args) + +template debug*(frmt: string, args: varargs[string, `$`]) = + ## Logs a debug message to all registered handlers. + log(lvlDebug, frmt, args) + +template info*(frmt: string, args: varargs[string, `$`]) = + ## Logs an info message to all registered handlers. + log(lvlInfo, frmt, args) + +template warn*(frmt: string, args: varargs[string, `$`]) = + ## Logs a warning message to all registered handlers. + log(lvlWarn, frmt, args) + +template error*(frmt: string, args: varargs[string, `$`]) = + ## Logs an error message to all registered handlers. + log(lvlError, frmt, args) + +template fatal*(frmt: string, args: varargs[string, `$`]) = + ## Logs a fatal error message to all registered handlers. + log(lvlFatal, frmt, args) + + +# -------------- + +when isMainModule: + var L = newConsoleLogger() + var fL = newFileLogger("test.log", fmtStr = verboseFmtStr) + var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr) + handlers.add(L) + handlers.add(fL) + handlers.add(rL) + for i in 0 .. 25: + info("hello" & $i, []) + + diff --git a/web/nimrod.ini b/web/nimrod.ini index 9af3bc226e..71e36dcdc3 100644 --- a/web/nimrod.ini +++ b/web/nimrod.ini @@ -62,7 +62,7 @@ srcdoc2: "pure/ftpclient;pure/memfiles;pure/subexes;pure/collections/critbits" srcdoc2: "pure/asyncio;pure/actors;core/locks;pure/oids;pure/endians;pure/uri" srcdoc2: "pure/nimprof;pure/unittest;packages/docutils/highlite" srcdoc2: "packages/docutils/rst;packages/docutils/rstast" -srcdoc2: "packages/docutils/rstgen" +srcdoc2: "packages/docutils/rstgen;pure/logging" webdoc: "wrappers/libcurl;pure/md5;wrappers/mysql;wrappers/iup" webdoc: "wrappers/sqlite3;wrappers/postgres;wrappers/tinyc" From 446ffb548126099dd7f62d058ac2aca0c5e65fe1 Mon Sep 17 00:00:00 2001 From: EXetoC Date: Wed, 29 Jan 2014 02:29:25 +0100 Subject: [PATCH 09/72] Punctation -> punctuation. --- lib/packages/docutils/highlite.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index db7a639281..4ca0c79e03 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -19,7 +19,7 @@ type gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber, gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit, gtLongStringLit, gtCharLit, gtEscapeSequence, # escape sequence like \xff - gtOperator, gtPunctation, gtComment, gtLongComment, gtRegularExpression, + gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression, gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler, gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel, gtReference, gtOther @@ -39,7 +39,7 @@ const tokenClassToStr*: array[TTokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", - "EscapeSequence", "Operator", "Punctation", "Comment", "LongComment", + "EscapeSequence", "Operator", "Punctuation", "Comment", "LongComment", "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData", "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink", "Label", "Reference", "Other"] @@ -258,7 +258,7 @@ proc nimNextToken(g: var TGeneralTokenizer) = else: inc(pos) of '(', ')', '[', ']', '{', '}', '`', ':', ',', ';': inc(pos) - g.kind = gtPunctation + g.kind = gtPunctuation of '\0': g.kind = gtEof else: @@ -473,7 +473,7 @@ proc clikeNextToken(g: var TGeneralTokenizer, keywords: openArray[string], else: inc(pos) of '(', ')', '[', ']', '{', '}', ':', ',', ';', '.': inc(pos) - g.kind = gtPunctation + g.kind = gtPunctuation of '\0': g.kind = gtEof else: From b549ca45175ef53e0990a3289b71051a9fe5349f Mon Sep 17 00:00:00 2001 From: EXetoC Date: Wed, 29 Jan 2014 17:35:11 +0100 Subject: [PATCH 10/72] Punctation -> Punctuation; fix remaining typos. --- config/nimdoc.cfg | 2 +- config/nimdoc.tex.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index d47dccb635..63a3c30c4e 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -76,7 +76,7 @@ span.LongStringLit {color: blue} span.CharLit {color: blue} span.EscapeSequence {color: black} span.Operator {color: black} -span.Punctation {color: black} +span.Punctuation {color: black} span.Comment, span.LongComment {font-style:italic; color: green} span.RegularExpression {color: DarkViolet} span.TagStart {color: DarkViolet} diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 8b59f2ee95..599ede345a 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -98,7 +98,7 @@ doc.file = """ \newcommand{\spanCharLit}[1]{#1} \newcommand{\spanEscapeSequence}[1]{#1} \newcommand{\spanOperator}[1]{#1} -\newcommand{\spanPunctation}[1]{#1} +\newcommand{\spanPunctuation}[1]{#1} \newcommand{\spanComment}[1]{\emph{#1}} \newcommand{\spanLongComment}[1]{\emph{#1}} \newcommand{\spanRegularExpression}[1]{#1} From d07f86b1597b17bf13ae51cc7912f378b56aa9f3 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Wed, 29 Jan 2014 23:25:21 +0100 Subject: [PATCH 11/72] Avoids raising exceptions in tryInsertID. --- lib/impure/db_sqlite.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index a3499a6df6..809ee70392 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -148,7 +148,8 @@ proc getValue*(db: TDbConn, query: TSqlQuery, if finalize(stmt) != SQLITE_OK: dbError(db) proc tryInsertID*(db: TDbConn, query: TSqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = + args: varargs[string, `$`]): int64 + {.tags: [FWriteDb], raises: [].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. var q = dbFormat(query, args) @@ -157,7 +158,8 @@ proc tryInsertID*(db: TDbConn, query: TSqlQuery, if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: if step(stmt) == SQLITE_DONE: result = last_insert_rowid(db) - if finalize(stmt) != SQLITE_OK: dbError(db) + if finalize(stmt) != SQLITE_OK: + result = -1 proc insertID*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = From ac0f15379cb0409e47512bcd40af28d1e7816337 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Thu, 30 Jan 2014 02:07:21 -0600 Subject: [PATCH 12/72] added Cartesian product --- lib/pure/algorithm.nim | 32 ++++++++++++++++++++++++++++++++ tests/stdlib/talgorithm.nim | 9 +++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/stdlib/talgorithm.nim diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index df7ae6d176..2620f7c901 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -131,3 +131,35 @@ proc sort*[T](a: var openArray[T], dec(m, s*2) s = s*2 +proc product[T](x: openarray[seq[T]]): seq[seq[T]] = + ## produces the Cartesian product of the array. Warning: complexity + ## may explode. + result = @[] + if x.len == 0: + return + if x.len == 1: + result = @x + return + var + indexes = newSeq[int](x.len) + initial = newSeq[int](x.len) + index = 0 + # replace with newSeq as soon as #853 is fixed + var next: seq[T] = @[] + next.setLen(x.len) + for i in 0..(x.len-1): + initial[i] = len(x[i])-1 + indexes = initial + while true: + while indexes[index] == -1: + indexes[index] = initial[index] + index +=1 + if index == x.len: return + indexes[index] -=1 + for ni, i in indexes: + next[ni] = x[ni][i] + var res: seq[T] + shallowCopy(res, next) + result.add(res) + index = 0 + indexes[index] -=1 diff --git a/tests/stdlib/talgorithm.nim b/tests/stdlib/talgorithm.nim new file mode 100644 index 0000000000..ea57883b0f --- /dev/null +++ b/tests/stdlib/talgorithm.nim @@ -0,0 +1,9 @@ +import unittest + +suite "product": + test "a simple case of one element": + check product(@[@[1,2]]) == @[@[1,2]] + test "two elements": + check product(@[@[1,2], @[3,4]]) == @[@[2,4],@[1,4],@[2,3],@[1,3]] + test "three elements": + check product(@[@[1,2], @[3,4], @[5,6]]) == @[@[2,4,6],@[1,4,6],@[2,3,6],@[1,3,6], @[2,4,5],@[1,4,5],@[2,3,5],@[1,3,5]] From f070edb8b91d07bd4ac8fea82317b906cc5ec1c9 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Thu, 30 Jan 2014 02:09:37 -0600 Subject: [PATCH 13/72] forgot to export product --- lib/pure/algorithm.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 2620f7c901..b71b2c0fce 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -131,7 +131,7 @@ proc sort*[T](a: var openArray[T], dec(m, s*2) s = s*2 -proc product[T](x: openarray[seq[T]]): seq[seq[T]] = +proc product*[T](x: openarray[seq[T]]): seq[seq[T]] = ## produces the Cartesian product of the array. Warning: complexity ## may explode. result = @[] From e01fb17d023b046b3403a4413a637d24a9dc492f Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Thu, 30 Jan 2014 23:55:43 -0600 Subject: [PATCH 14/72] product more robust against empty input --- lib/pure/algorithm.nim | 1 + tests/stdlib/talgorithm.nim | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index b71b2c0fce..921c659de8 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -148,6 +148,7 @@ proc product*[T](x: openarray[seq[T]]): seq[seq[T]] = var next: seq[T] = @[] next.setLen(x.len) for i in 0..(x.len-1): + if len(x[i]) == 0: return initial[i] = len(x[i])-1 indexes = initial while true: diff --git a/tests/stdlib/talgorithm.nim b/tests/stdlib/talgorithm.nim index ea57883b0f..37de1262f3 100644 --- a/tests/stdlib/talgorithm.nim +++ b/tests/stdlib/talgorithm.nim @@ -1,6 +1,11 @@ import unittest +import algorithm suite "product": + test "empty input": + check product[int](newSeq[seq[int]]()) == newSeq[seq[int]]() + test "bit more empty input": + check product[int](@[newSeq[int](), @[], @[]]) == newSeq[seq[int]]() test "a simple case of one element": check product(@[@[1,2]]) == @[@[1,2]] test "two elements": From 2c5a2d07fb3bb25dcb590d61882f88528ea17e91 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Fri, 31 Jan 2014 18:22:27 -0600 Subject: [PATCH 15/72] sets equivalence --- lib/pure/collections/sets.nim | 17 +++++++++++++++++ tests/sets/testequivalence.nim | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 tests/sets/testequivalence.nim diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 7259772aad..e478a2ce19 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -224,3 +224,20 @@ proc toOrderedSet*[A](keys: openArray[A]): TOrderedSet[A] = proc `$`*[A](s: TOrderedSet[A]): string = ## The `$` operator for ordered hash sets. dollarImpl() + +proc `<`*[A](s, t: TSet[A]): bool = + ## Is a a strict subset of b? + s.counter != t.counter and s <= t + +proc `<=`*[A](s, t: TSet[A]): bool = + ## Is a a subset of b? + result = false + if s.counter > t.counter: return + result = true + for item in s: + if not(t.contains(item)): + result = false + return + +proc `==`*[A](s, t: TSet[A]): bool = + s.counter == t.counter and s <= t diff --git a/tests/sets/testequivalence.nim b/tests/sets/testequivalence.nim new file mode 100644 index 0000000000..a1e02fee77 --- /dev/null +++ b/tests/sets/testequivalence.nim @@ -0,0 +1,16 @@ +import unittest +import sets + +suite "sets": + test "equivalent or subset": + check toSet(@[1,2,3]) <= toSet(@[1,2,3,4]) + check toSet(@[1,2,3]) <= toSet(@[1,2,3]) + check(not(toSet(@[1,2,3]) <= toSet(@[1,2]))) + test "strict subset": + check toSet(@[1,2,3]) <= toSet(@[1,2,3,4]) + check(not(toSet(@[1,2,3]) < toSet(@[1,2,3]))) + check(not(toSet(@[1,2,3]) < toSet(@[1,2]))) + test "==": + check(not(toSet(@[1,2,3]) == toSet(@[1,2,3,4]))) + check toSet(@[1,2,3]) == toSet(@[1,2,3]) + check(not(toSet(@[1,2,3]) == toSet(@[1,2]))) From d8d93218fa509b8980d5ecb4b8af4e70e9f4926e Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 1 Feb 2014 11:56:21 +0100 Subject: [PATCH 16/72] bugfix: object constructor doesn't allow 'distinct' types --- compiler/semexprs.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 6e2d777fb0..432443dc69 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1709,8 +1709,8 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = result = n result.typ = t result.kind = nkObjConstr - t = skipTypes(t, abstractInst) - if t.kind == tyRef: t = skipTypes(t.sons[0], abstractInst) + t = skipTypes(t, {tyGenericInst}) + if t.kind == tyRef: t = skipTypes(t.sons[0], {tyGenericInst}) if t.kind != tyObject: localError(n.info, errGenerated, "object constructor needs an object type") return From 6d62503e5d8d195e3da7d9cb30782410a5d3c3ea Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 1 Feb 2014 13:39:40 +0100 Subject: [PATCH 17/72] documented new symbol binding rules for templates --- doc/manual.txt | 131 ++++++++++++++++++++++--------------------------- 1 file changed, 59 insertions(+), 72 deletions(-) diff --git a/doc/manual.txt b/doc/manual.txt index faf62dcee5..260f0807af 100644 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -12,6 +12,8 @@ Nimrod Manual user to one/some of the other players, but the total amount seems to remain pretty much constant for a given task. -- Ran + + About this document =================== @@ -1479,7 +1481,7 @@ But it seems all this boilerplate code needs to be repeated for the ``TEuro`` currency. This can be solved with templates_. .. code-block:: nimrod - template Additive(typ: typedesc): stmt = + template additive(typ: typedesc): stmt = proc `+` *(x, y: typ): typ {.borrow.} proc `-` *(x, y: typ): typ {.borrow.} @@ -1487,26 +1489,26 @@ currency. This can be solved with templates_. proc `+` *(x: typ): typ {.borrow.} proc `-` *(x: typ): typ {.borrow.} - template Multiplicative(typ, base: typedesc): stmt = + template multiplicative(typ, base: typedesc): stmt = proc `*` *(x: typ, y: base): typ {.borrow.} proc `*` *(x: base, y: typ): typ {.borrow.} proc `div` *(x: typ, y: base): typ {.borrow.} proc `mod` *(x: typ, y: base): typ {.borrow.} - template Comparable(typ: typedesc): stmt = + template comparable(typ: typedesc): stmt = proc `<` * (x, y: typ): bool {.borrow.} proc `<=` * (x, y: typ): bool {.borrow.} proc `==` * (x, y: typ): bool {.borrow.} - template DefineCurrency(typ, base: expr): stmt = + template defineCurrency(typ, base: expr): stmt = type typ* = distinct base - Additive(typ) - Multiplicative(typ, base) - Comparable(typ) + additive(typ) + multiplicative(typ, base) + comparable(typ) - DefineCurrency(TDollar, int) - DefineCurrency(TEuro, int) + defineCurrency(TDollar, int) + defineCurrency(TEuro, int) Void type @@ -3440,13 +3442,41 @@ A symbol can be forced to be open by a `mixin`:idx: declaration: .. code-block:: nimrod proc create*[T](): ref T = - # there is no overloaded 'mixin' here, so we need to state that it's an + # there is no overloaded 'init' here, so we need to state that it's an # open symbol explicitly: mixin init new result init result +Bind statement +-------------- + +The `bind`:idx: statement is the counterpart to the ``mixin`` statement. It +can be used to explicitly declare identifiers that should be bound early (i.e. +the identifiers should be looked up in the scope of the template/generic +definition): + +.. code-block:: nimrod + # Module A + var + lastId = 0 + + template genId*: expr = + bind lastId + inc(lastId) + lastId + +.. code-block:: nimrod + # Module B + import A + + echo genId() + +But a ``bind`` is rarely useful because symbol binding from the definition +scope is the default. + + Templates ========= @@ -3506,28 +3536,6 @@ receive undeclared identifiers: declareInt(x) # valid -Scoping in templates --------------------- - -The template body does not open a new scope. To open a new scope a ``block`` -statement can be used: - -.. code-block:: nimrod - template declareInScope(x: expr, t: typedesc): stmt {.immediate.} = - var x: t - - template declareInNewScope(x: expr, t: typedesc): stmt {.immediate.} = - # open a new scope: - block: - var x: t - - declareInScope(a, int) - a = 42 # works, `a` is known here - - declareInNewScope(b, int) - b = 42 # does not work, `b` is unknown - - Passing a code block to a template ---------------------------------- @@ -3538,26 +3546,28 @@ special ``:`` syntax: .. code-block:: nimrod template withFile(f, fn, mode: expr, actions: stmt): stmt {.immediate.} = - block: - var f: TFile - if open(f, fn, mode): - try: - actions - finally: - close(f) - else: - quit("cannot open: " & fn) + var f: TFile + if open(f, fn, mode): + try: + actions + finally: + close(f) + else: + quit("cannot open: " & fn) withFile(txt, "ttempl3.txt", fmWrite): txt.writeln("line 1") txt.writeln("line 2") In the example the two ``writeln`` statements are bound to the ``actions`` -parameter. +parameter. -**Note:** The symbol binding rules for templates might change! -Symbol binding within templates happens after template instantiation: +Symbol binding in templates +--------------------------- + +A template is a `hygienic`:idx: macro and so opens a new scope. Most symbols are +bound from the definition scope of the template: .. code-block:: nimrod # Module A @@ -3572,34 +3582,11 @@ Symbol binding within templates happens after template instantiation: # Module B import A - echo genId() # Error: undeclared identifier: 'lastId' + echo genId() # Works as 'lastId' has been bound in 'genId's defining scope +As in generics symbol binding can be influenced via ``mixin`` or ``bind`` +statements. -Bind statement --------------- - -Exporting a template is a often a leaky abstraction as it can depend on -symbols that are not visible from a client module. However, to compensate for -this case, a `bind`:idx: statement can be used: It declares all identifiers -that should be bound early (i.e. when the template is parsed): - -.. code-block:: nimrod - # Module A - var - lastId = 0 - - template genId*: expr = - bind lastId - inc(lastId) - lastId - -.. code-block:: nimrod - # Module B - import A - - echo genId() # Works - -A ``bind`` statement can also be used in generics for the same purpose. Identifier construction @@ -3942,13 +3929,13 @@ Static params can also appear in the signatures of generic types: type Matrix[M,N: static[int]; T: Number] = array[0..(M*N - 1), T] - # Please, note how `Number` is just a type constraint here, while + # Note how `Number` is just a type constraint here, while # `static[int]` requires us to supply a compile-time int value AffineTransform2D[T] = Matrix[3, 3, T] AffineTransform3D[T] = Matrix[4, 4, T] - AffineTransform3D[float] # OK + AffineTransform3D[float] # OK AffineTransform2D[string] # Error, `string` is not a `Number` From f9c2ec8d9238afaff285b2ba286f8c5534b05eec Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 1 Feb 2014 14:57:52 +0000 Subject: [PATCH 18/72] Fixes wrong slurp path in tester. --- tests/tester.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tester.nim b/tests/tester.nim index 0e125b1bbb..9f9da6bfed 100644 --- a/tests/tester.nim +++ b/tests/tester.nim @@ -193,8 +193,8 @@ const Test results From 9f29bb8d9ee01ead64e8af89d471f6fcb67b9712 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 1 Feb 2014 16:07:44 -0600 Subject: [PATCH 19/72] corrected docs and tests --- lib/pure/collections/sets.nim | 4 ++-- tests/sets/testequivalence.nim | 22 +++++++++------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index e478a2ce19..e6ab617e50 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -226,11 +226,11 @@ proc `$`*[A](s: TOrderedSet[A]): string = dollarImpl() proc `<`*[A](s, t: TSet[A]): bool = - ## Is a a strict subset of b? + ## Is s a strict subset of t? s.counter != t.counter and s <= t proc `<=`*[A](s, t: TSet[A]): bool = - ## Is a a subset of b? + ## Is s a subset of t? result = false if s.counter > t.counter: return result = true diff --git a/tests/sets/testequivalence.nim b/tests/sets/testequivalence.nim index a1e02fee77..8a83e2a213 100644 --- a/tests/sets/testequivalence.nim +++ b/tests/sets/testequivalence.nim @@ -1,16 +1,12 @@ import unittest import sets -suite "sets": - test "equivalent or subset": - check toSet(@[1,2,3]) <= toSet(@[1,2,3,4]) - check toSet(@[1,2,3]) <= toSet(@[1,2,3]) - check(not(toSet(@[1,2,3]) <= toSet(@[1,2]))) - test "strict subset": - check toSet(@[1,2,3]) <= toSet(@[1,2,3,4]) - check(not(toSet(@[1,2,3]) < toSet(@[1,2,3]))) - check(not(toSet(@[1,2,3]) < toSet(@[1,2]))) - test "==": - check(not(toSet(@[1,2,3]) == toSet(@[1,2,3,4]))) - check toSet(@[1,2,3]) == toSet(@[1,2,3]) - check(not(toSet(@[1,2,3]) == toSet(@[1,2]))) +doAssert(toSet(@[1,2,3]) <= toSet(@[1,2,3,4]), "equivalent or subset") +doAssert(toSet(@[1,2,3]) <= toSet(@[1,2,3]), "equivalent or subset") +doAssert((not(toSet(@[1,2,3]) <= toSet(@[1,2]))), "equivalent or subset") +doAssert(toSet(@[1,2,3]) <= toSet(@[1,2,3,4]), "strict subset") +doAssert((not(toSet(@[1,2,3]) < toSet(@[1,2,3]))), "strict subset") +doAssert((not(toSet(@[1,2,3]) < toSet(@[1,2]))), "strict subset") +doAssert((not(toSet(@[1,2,3]) == toSet(@[1,2,3,4]))), "==") +doAssert(toSet(@[1,2,3]) == toSet(@[1,2,3]), "==") +doAssert((not(toSet(@[1,2,3]) == toSet(@[1,2]))), "==") From 68d8cd1301b301b4ebfa41dd7ae8714e28cbe721 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 1 Feb 2014 23:56:32 +0100 Subject: [PATCH 20/72] case consistency for evalffi --- compiler/ccgexprs.nim | 2 +- compiler/evalffi.nim | 56 +++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index ba543039e1..be47ac0c4a 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -343,7 +343,7 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = of tyPtr, tyPointer, tyChar, tyBool, tyEnum, tyCString, tyInt..tyUInt64, tyRange, tyVar: linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src)) - else: internalError("genAssignment(" & $ty.kind & ')') + else: internalError("genAssignment: " & $ty.kind) proc getDestLoc(p: BProc, d: var TLoc, typ: PType) = if d.k == locNone: getTemp(p, typ, d) diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 74f0663f3c..54be0ccb2a 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2014 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -102,7 +102,7 @@ proc mapCallConv(cc: TCallingConvention, info: TLineInfo): TABI = of ccStdCall: result = when defined(windows): STDCALL else: DEFAULT_ABI of ccCDecl: result = DEFAULT_ABI else: - GlobalError(info, "cannot map calling convention to FFI") + globalError(info, "cannot map calling convention to FFI") template rd(T, p: expr): expr {.immediate.} = (cast[ptr T](p))[] template wr(T, p, v: expr) {.immediate.} = (cast[ptr T](p))[] = v @@ -164,7 +164,7 @@ proc packObject(x: PNode, typ: PType, res: pointer) = let field = getField(typ.n, i) pack(it, field.typ, res +! field.offset) else: - GlobalError(x.info, "cannot pack unnamed tuple") + globalError(x.info, "cannot pack unnamed tuple") const maxPackDepth = 20 var packRecCheck = 0 @@ -193,7 +193,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = of 4: awr(int32, v.intVal.int32) of 8: awr(int64, v.intVal.int64) else: - GlobalError(v.info, "cannot map value to FFI (tyEnum, tySet)") + globalError(v.info, "cannot map value to FFI (tyEnum, tySet)") of tyFloat: awr(float, v.floatVal) of tyFloat32: awr(float32, v.floatVal) of tyFloat64: awr(float64, v.floatVal) @@ -207,7 +207,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = elif v.kind in {nkStrLit..nkTripleStrLit}: awr(cstring, cstring(v.strVal)) else: - GlobalError(v.info, "cannot map pointer/proc value to FFI") + globalError(v.info, "cannot map pointer/proc value to FFI") of tyPtr, tyRef, tyVar: if v.kind == nkNilLit: # nothing to do since the memory is 0 initialized anyway @@ -217,7 +217,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = else: if packRecCheck > maxPackDepth: packRecCheck = 0 - GlobalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) + globalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) inc packRecCheck pack(v.sons[0], typ.sons[0], res +! sizeof(pointer)) dec packRecCheck @@ -233,7 +233,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = of tyDistinct, tyGenericInst: pack(v, typ.sons[0], res) else: - GlobalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) + globalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) proc unpack(x: pointer, typ: PType, n: PNode): PNode @@ -243,7 +243,7 @@ proc unpackObjectAdd(x: pointer, n, result: PNode) = for i in countup(0, sonsLen(n) - 1): unpackObjectAdd(x, n.sons[i], result) of nkRecCase: - GlobalError(result.info, "case objects cannot be unpacked") + globalError(result.info, "case objects cannot be unpacked") of nkSym: var pair = newNodeI(nkExprColonExpr, result.info, 2) pair.sons[0] = n @@ -262,14 +262,14 @@ proc unpackObject(x: pointer, typ: PType, n: PNode): PNode = result = newNode(nkPar) result.typ = typ if typ.n.isNil: - InternalError("cannot unpack unnamed tuple") + internalError("cannot unpack unnamed tuple") unpackObjectAdd(x, typ.n, result) else: result = n if result.kind notin {nkObjConstr, nkPar}: - GlobalError(n.info, "cannot map value from FFI") + globalError(n.info, "cannot map value from FFI") if typ.n.isNil: - GlobalError(n.info, "cannot unpack unnamed tuple") + globalError(n.info, "cannot unpack unnamed tuple") for i in countup(ord(n.kind == nkObjConstr), sonsLen(n) - 1): var it = n.sons[i] if it.kind == nkExprColonExpr: @@ -288,7 +288,7 @@ proc unpackArray(x: pointer, typ: PType, n: PNode): PNode = else: result = n if result.kind != nkBracket: - GlobalError(n.info, "cannot map value from FFI") + globalError(n.info, "cannot map value from FFI") let baseSize = typ.sons[1].getSize for i in 0 .. < result.len: result.sons[i] = unpack(x +! i * baseSize, typ.sons[1], result.sons[i]) @@ -312,7 +312,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = #echo "expected ", k, " but got ", result.kind #debug result return newNodeI(nkExceptBranch, n.info) - #GlobalError(n.info, "cannot map value from FFI") + #globalError(n.info, "cannot map value from FFI") result.field = v template setNil() = @@ -337,19 +337,19 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = of tyInt16: awi(nkInt16Lit, rd(int16, x)) of tyInt32: awi(nkInt32Lit, rd(int32, x)) of tyInt64: awi(nkInt64Lit, rd(int64, x)) - of tyUInt: awi(nkUIntLit, rd(uint, x).biggestInt) - of tyUInt8: awi(nkUInt8Lit, rd(uint8, x).biggestInt) - of tyUInt16: awi(nkUInt16Lit, rd(uint16, x).biggestInt) - of tyUInt32: awi(nkUInt32Lit, rd(uint32, x).biggestInt) - of tyUInt64: awi(nkUInt64Lit, rd(uint64, x).biggestInt) + of tyUInt: awi(nkUIntLit, rd(uint, x).BiggestInt) + of tyUInt8: awi(nkUInt8Lit, rd(uint8, x).BiggestInt) + of tyUInt16: awi(nkUInt16Lit, rd(uint16, x).BiggestInt) + of tyUInt32: awi(nkUInt32Lit, rd(uint32, x).BiggestInt) + of tyUInt64: awi(nkUInt64Lit, rd(uint64, x).BiggestInt) of tyEnum: case typ.getSize - of 1: awi(nkIntLit, rd(uint8, x).biggestInt) - of 2: awi(nkIntLit, rd(uint16, x).biggestInt) - of 4: awi(nkIntLit, rd(int32, x).biggestInt) - of 8: awi(nkIntLit, rd(int64, x).biggestInt) + of 1: awi(nkIntLit, rd(uint8, x).BiggestInt) + of 2: awi(nkIntLit, rd(uint16, x).BiggestInt) + of 4: awi(nkIntLit, rd(int32, x).BiggestInt) + of 8: awi(nkIntLit, rd(int64, x).BiggestInt) else: - GlobalError(n.info, "cannot map value from FFI (tyEnum, tySet)") + globalError(n.info, "cannot map value from FFI (tyEnum, tySet)") of tyFloat: awf(nkFloatLit, rd(float, x)) of tyFloat32: awf(nkFloat32Lit, rd(float32, x)) of tyFloat64: awf(nkFloat64Lit, rd(float64, x)) @@ -374,7 +374,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = n.sons[0] = unpack(p, typ.sons[0], n.sons[0]) result = n else: - GlobalError(n.info, "cannot map value from FFI " & typeToString(typ)) + globalError(n.info, "cannot map value from FFI " & typeToString(typ)) of tyObject, tyTuple: result = unpackObject(x, typ, n) of tyArray, tyArrayConstr: @@ -391,7 +391,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = result = unpack(x, typ.sons[0], n) else: # XXX what to do with 'array' here? - GlobalError(n.info, "cannot map value from FFI " & typeToString(typ)) + globalError(n.info, "cannot map value from FFI " & typeToString(typ)) proc fficast*(x: PNode, destTyp: PType): PNode = if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyPointer, @@ -414,7 +414,7 @@ proc fficast*(x: PNode, destTyp: PType): PNode = dealloc a proc callForeignFunction*(call: PNode): PNode = - InternalAssert call.sons[0].kind == nkPtrLit + internalAssert call.sons[0].kind == nkPtrLit var cif: TCif var sig: TParamList @@ -422,12 +422,12 @@ proc callForeignFunction*(call: PNode): PNode = for i in 1..call.len-1: sig[i-1] = mapType(call.sons[i].typ) if sig[i-1].isNil: - GlobalError(call.info, "cannot map FFI type") + globalError(call.info, "cannot map FFI type") let typ = call.sons[0].typ if prep_cif(cif, mapCallConv(typ.callConv, call.info), cuint(call.len-1), mapType(typ.sons[0]), sig) != OK: - GlobalError(call.info, "error in FFI call") + globalError(call.info, "error in FFI call") var args: TArgList let fn = cast[pointer](call.sons[0].intVal) From 31f3034c3a93a7f056863db51ede65d2ebf66db6 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 1 Feb 2014 23:57:09 +0100 Subject: [PATCH 21/72] case consistency for evalffi --- compiler/main.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/main.nim b/compiler/main.nim index cdea7b5ca9..f6d11d960a 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -135,7 +135,7 @@ proc interactivePasses = #setTarget(osNimrodVM, cpuNimrodVM) initDefines() defineSymbol("nimrodvm") - when hasFFI: DefineSymbol("nimffi") + when hasFFI: defineSymbol("nimffi") registerPass(verbosePass) registerPass(semPass) registerPass(evalPass) @@ -324,7 +324,7 @@ proc mainCommand* = wantMainModule() when hasTinyCBackend: extccomp.setCC("tcc") - CommandCompileToC() + commandCompileToC() else: rawMessage(errInvalidCommandX, command) of "js", "compiletojs": @@ -450,7 +450,8 @@ proc mainCommand* = echo " tries : ", gCacheTries echo " misses: ", gCacheMisses echo " int tries: ", gCacheIntTries - echo " efficiency: ", formatFloat(1-(gCacheMisses.float/gCacheTries.float), ffDecimal, 3) + echo " efficiency: ", formatFloat(1-(gCacheMisses.float/gCacheTries.float), + ffDecimal, 3) when SimiluateCaasMemReset: resetMemory() From 0b8f68def0130ca381c2b2f16cc9b62918e32301 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 1 Feb 2014 23:58:20 +0100 Subject: [PATCH 22/72] tstringinterp almost working --- compiler/vm.nim | 16 ++++++-- compiler/vmdef.nim | 2 + compiler/vmgen.nim | 94 ++++++++++++++++++++++++++++------------------ 3 files changed, 72 insertions(+), 40 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 26aab30680..bc5320d9d6 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -337,7 +337,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode = asgnRef(c.globals.sons[instr.regBx-wordExcess-1], regs[ra]) of opcWrGlobal: asgnComplex(c.globals.sons[instr.regBx-wordExcess-1], regs[ra]) - of opcLdArr: + of opcLdArr, opcLdArrRef: # a = b[c] let rb = instr.regB let rc = instr.regC @@ -348,7 +348,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode = assert regs[rb].kind != nkMetaNode let src = regs[rb] if src.kind notin {nkEmpty..nkNilLit} and idx <% src.len: - asgnComplex(regs[ra], src.sons[idx]) + if instr.opcode == opcLdArrRef and false: + # XXX activate when seqs are fixed + asgnRef(regs[ra], src.sons[idx]) + else: + asgnComplex(regs[ra], src.sons[idx]) else: stackTrace(c, tos, pc, errIndexOutOfBounds) of opcLdStrIdx: @@ -379,9 +383,15 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode = # a = b.c let rb = instr.regB let rc = instr.regC - # XXX this creates a wrong alias #Message(c.debug[pc], warnUser, $regs[rb].safeLen & " " & $rc) asgnComplex(regs[ra], regs[rb].sons[rc]) + of opcLdObjRef: + # a = b.c + let rb = instr.regB + let rc = instr.regC + # XXX activate when seqs are fixed + asgnComplex(regs[ra], regs[rb].sons[rc]) + #asgnRef(regs[ra], regs[rb].sons[rc]) of opcWrObj: # a.b = c let rb = instr.regB diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 480c7f31b9..87159c8136 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -34,9 +34,11 @@ type opcAsgnComplex, opcLdArr, # a = b[c] + opcLdArrRef, opcWrArr, # a[b] = c opcWrArrRef, opcLdObj, # a = b.c + opcLdObjRef, opcWrObj, # a.b = c opcWrObjRef, opcAddr, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index a41e60e7d0..ff479b74a3 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2013 Andreas Rumpf +# (c) Copyright 2014 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -13,9 +13,18 @@ import unsigned, strutils, ast, astalgo, types, msgs, renderer, vmdef, trees, intsets, rodread, magicsys, options +from os import splitFile + when hasFFI: import evalffi +type + TGenFlag = enum gfNone, gfAddrOf + TGenFlags = set[TGenFlag] + +proc debugInfo(info: TLineInfo): string = + result = info.toFilename.splitFile.name & ":" & $info.line + proc codeListing(c: PCtx, result: var string, start=0) = # first iteration: compute all necessary labels: var jumpTargets = initIntSet() @@ -44,7 +53,7 @@ proc codeListing(c: PCtx, result: var string, start=0) = else: result.addf("\t$#\tr$#, $#", ($opc).substr(3), x.regA, x.regBx-wordExcess) result.add("\t#") - result.add(toFileLine(c.debug[i])) + result.add(debugInfo(c.debug[i])) result.add("\n") inc i @@ -190,20 +199,20 @@ template withBlock(labl: PSym; body: stmt) {.immediate, dirty.} = body popBlock(c, oldLen) -proc gen(c: PCtx; n: PNode; dest: var TDest) -proc gen(c: PCtx; n: PNode; dest: TRegister) = +proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) +proc gen(c: PCtx; n: PNode; dest: TRegister; flags: TGenFlags = {}) = var d: TDest = dest - gen(c, n, d) + gen(c, n, d, flags) internalAssert d == dest -proc gen(c: PCtx; n: PNode) = +proc gen(c: PCtx; n: PNode; flags: TGenFlags = {}) = var tmp: TDest = -1 - gen(c, n, tmp) + gen(c, n, tmp, flags) #if n.typ.isEmptyType: InternalAssert tmp < 0 -proc genx(c: PCtx; n: PNode): TRegister = +proc genx(c: PCtx; n: PNode; flags: TGenFlags = {}): TRegister = var tmp: TDest = -1 - gen(c, n, tmp) + gen(c, n, tmp, flags) internalAssert tmp >= 0 result = TRegister(tmp) @@ -477,8 +486,8 @@ proc genNew(c: PCtx; n: PNode) = proc genNewSeq(c: PCtx; n: PNode) = let dest = if needsAsgnPatch(n.sons[1]): c.getTemp(n.sons[1].typ) else: c.genx(n.sons[1]) - c.gABx(n, opcNewSeq, dest, c.genType(n.sons[1].typ.skipTypes(abstractVar))) let tmp = c.genx(n.sons[2]) + c.gABx(n, opcNewSeq, dest, c.genType(n.sons[1].typ.skipTypes(abstractVar))) c.gABx(n, opcNewSeq, tmp, 0) c.freeTemp(tmp) c.genAsgnPatch(n.sons[1], dest) @@ -528,6 +537,14 @@ proc genBinaryStmt(c: PCtx; n: PNode; opc: TOpcode) = c.gABC(n, opc, dest, tmp, 0) c.freeTemp(tmp) +proc genBinaryStmtVar(c: PCtx; n: PNode; opc: TOpcode) = + let + dest = c.genx(n.sons[1], {gfAddrOf}) + tmp = c.genx(n.sons[2]) + c.gABC(n, opc, dest, tmp, 0) + #c.genAsgnPatch(n.sons[1], dest) + c.freeTemp(tmp) + proc genUnaryStmt(c: PCtx; n: PNode; opc: TOpcode) = let tmp = c.genx(n.sons[1]) c.gABC(n, opc, tmp, 0, 0) @@ -754,13 +771,13 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) = c.freeTempRange(x, n.len-1) of mAppendStrCh: unused(n, dest) - genBinaryStmt(c, n, opcAddStrCh) + genBinaryStmtVar(c, n, opcAddStrCh) of mAppendStrStr: unused(n, dest) - genBinaryStmt(c, n, opcAddStrStr) + genBinaryStmtVar(c, n, opcAddStrStr) of mAppendSeqElem: unused(n, dest) - genBinaryStmt(c, n, opcAddSeqElem) + genBinaryStmtVar(c, n, opcAddSeqElem) of mParseExprToAst: genUnaryABC(c, n, dest, opcParseExprToAst) of mParseStmtToAst: @@ -890,12 +907,14 @@ proc skipDeref(n: PNode): PNode = else: result = n -proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = +proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; + flags: TGenFlags) = # a nop for certain types + let flags = if opc == opcAddr: flags+{gfAddrOf} else: flags if unneededIndirection(n.sons[0]): - gen(c, n.sons[0], dest) + gen(c, n.sons[0], dest, flags) else: - let tmp = c.genx(n.sons[0]) + let tmp = c.genx(n.sons[0], flags) if dest < 0: dest = c.getTemp(n.typ) gABC(c, n, opc, dest, tmp) c.freeTemp(tmp) @@ -1026,26 +1045,27 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest) = cannotEval(n) #InternalError(n.info, s.name.s & " " & $s.position) -proc genAccess(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = - let a = c.genx(n.sons[0]) - let b = c.genx(n.sons[1]) +proc genAccess(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; + flags: TGenFlags) = + let a = c.genx(n.sons[0], flags) + let b = c.genx(n.sons[1], {}) if dest < 0: dest = c.getTemp(n.typ) - c.gABC(n, opc, dest, a, b) + c.gABC(n, (if gfAddrOf in flags: succ(opc) else: opc), dest, a, b) c.freeTemp(a) c.freeTemp(b) -proc genObjAccess(c: PCtx; n: PNode; dest: var TDest) = - genAccess(c, n, dest, opcLdObj) +proc genObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = + genAccess(c, n, dest, opcLdObj, flags) -proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest) = +proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = # XXX implement field checks! - genAccess(c, n.sons[0], dest, opcLdObj) + genAccess(c, n.sons[0], dest, opcLdObj, flags) -proc genArrAccess(c: PCtx; n: PNode; dest: var TDest) = +proc genArrAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = if n.sons[0].typ.skipTypes(abstractVarRange).kind in {tyString, tyCString}: - genAccess(c, n, dest, opcLdStrIdx) + genAccess(c, n, dest, opcLdStrIdx, {}) else: - genAccess(c, n, dest, opcLdArr) + genAccess(c, n, dest, opcLdArr, flags) proc getNullValue*(typ: PType, info: TLineInfo): PNode proc getNullValueAux(obj: PNode, result: PNode) = @@ -1222,7 +1242,7 @@ proc genTupleConstr(c: PCtx, n: PNode, dest: var TDest) = proc genProc*(c: PCtx; s: PSym): int -proc gen(c: PCtx; n: PNode; dest: var TDest) = +proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = case n.kind of nkSym: let s = n.sym @@ -1271,11 +1291,11 @@ proc gen(c: PCtx; n: PNode; dest: var TDest) = of nkAsgn, nkFastAsgn: unused(n, dest) genAsgn(c, n.sons[0], n.sons[1], n.kind == nkAsgn) - of nkDotExpr: genObjAccess(c, n, dest) - of nkCheckedFieldExpr: genCheckedObjAccess(c, n, dest) - of nkBracketExpr: genArrAccess(c, n, dest) - of nkDerefExpr, nkHiddenDeref: genAddrDeref(c, n, dest, opcDeref) - of nkAddr, nkHiddenAddr: genAddrDeref(c, n, dest, opcAddr) + of nkDotExpr: genObjAccess(c, n, dest, flags) + of nkCheckedFieldExpr: genCheckedObjAccess(c, n, dest, flags) + of nkBracketExpr: genArrAccess(c, n, dest, flags) + of nkDerefExpr, nkHiddenDeref: genAddrDeref(c, n, dest, opcDeref, flags) + of nkAddr, nkHiddenAddr: genAddrDeref(c, n, dest, opcAddr, flags) of nkWhenStmt, nkIfStmt, nkIfExpr: genIf(c, n, dest) of nkCaseStmt: genCase(c, n, dest) of nkWhileStmt: @@ -1298,7 +1318,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest) = of nkStmtListExpr: let L = n.len-1 for i in 0 .. Date: Sun, 2 Feb 2014 00:00:17 +0100 Subject: [PATCH 23/72] 'discard' instead of 'nil' for system.nim --- lib/system.nim | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 09e44a45a9..2acb989c5f 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2333,29 +2333,29 @@ when not defined(JS): #and not defined(NimrodVM): elif defined(JS): # Stubs: - proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = nil + proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = discard - proc GC_disable() = nil - proc GC_enable() = nil - proc GC_fullCollect() = nil - proc GC_setStrategy(strategy: TGC_Strategy) = nil - proc GC_enableMarkAndSweep() = nil - proc GC_disableMarkAndSweep() = nil + proc GC_disable() = discard + proc GC_enable() = discard + proc GC_fullCollect() = discard + proc GC_setStrategy(strategy: TGC_Strategy) = discard + proc GC_enableMarkAndSweep() = discard + proc GC_disableMarkAndSweep() = discard proc GC_getStatistics(): string = return "" proc getOccupiedMem(): int = return -1 proc getFreeMem(): int = return -1 proc getTotalMem(): int = return -1 - proc dealloc(p: pointer) = nil - proc alloc(size: int): pointer = nil - proc alloc0(size: int): pointer = nil - proc realloc(p: Pointer, newsize: int): pointer = nil + proc dealloc(p: pointer) = discard + proc alloc(size: int): pointer = discard + proc alloc0(size: int): pointer = discard + proc realloc(p: Pointer, newsize: int): pointer = discard - proc allocShared(size: int): pointer = nil - proc allocShared0(size: int): pointer = nil - proc deallocShared(p: pointer) = nil - proc reallocShared(p: pointer, newsize: int): pointer = nil + proc allocShared(size: int): pointer = discard + proc allocShared0(size: int): pointer = discard + proc deallocShared(p: pointer) = discard + proc reallocShared(p: pointer, newsize: int): pointer = discard when defined(JS): include "system/jssys" @@ -2490,11 +2490,11 @@ proc staticRead*(filename: string): string {.magic: "Slurp".} ## ``slurp`` is an alias for ``staticRead``. proc gorge*(command: string, input = ""): string {. - magic: "StaticExec".} = nil + magic: "StaticExec".} = discard ## This is an alias for ``staticExec``. proc staticExec*(command: string, input = ""): string {. - magic: "StaticExec".} = nil + magic: "StaticExec".} = discard ## Executes an external process at compile-time. ## if `input` is not an empty string, it will be passed as a standard input ## to the executed program. @@ -2561,7 +2561,7 @@ proc instantiationInfo*(index = -1, fullPaths = false): tuple[ ## $pos.line, astToStr(code)] ## assert false, "A test expecting failure succeeded?" ## except exception: - ## nil + ## discard ## ## proc tester(pos: int): int = ## let From 70eff919cd6fa7a0f1443125ba29706328117046 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 00:00:56 +0100 Subject: [PATCH 24/72] only 1 argument allowed for command expressions --- compiler/parser.nim | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 3765557b9d..4497e360aa 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -672,12 +672,14 @@ proc primarySuffix(p: var TParser, r: PNode): PNode = let a = result result = newNodeP(nkCommand, p) addSon(result, a) - while p.tok.tokType != tkEof: - let a = parseExpr(p) - addSon(result, a) - if p.tok.tokType != tkComma: break - getTok(p) - optInd(p, a) + addSon result, parseExpr(p) + when false: + while p.tok.tokType != tkEof: + let a = parseExpr(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) if p.tok.tokType == tkDo: parseDoBlocks(p, result) else: @@ -1103,7 +1105,9 @@ proc parseExprStmt(p: var TParser): PNode = #| doBlocks #| / macroColon #| ))? + inc p.inPragma var a = simpleExpr(p) + dec p.inPragma if p.tok.tokType == tkEquals: getTok(p) optInd(p, result) From e647428539f829b1e761b85739da1af9f1625a2b Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 00:01:33 +0100 Subject: [PATCH 25/72] case consistency for dynlib.nim --- lib/pure/dynlib.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index a64b7f1386..3ed00fdb29 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -14,15 +14,15 @@ type TLibHandle* = pointer ## a handle to a dynamically loaded library -proc LoadLib*(path: string): TLibHandle +proc loadLib*(path: string): TLibHandle ## loads a library from `path`. Returns nil if the library could not ## be loaded. -proc LoadLib*(): TLibHandle +proc loadLib*(): TLibHandle ## gets the handle from the current executable. Returns nil if the ## library could not be loaded. -proc UnloadLib*(lib: TLibHandle) +proc unloadLib*(lib: TLibHandle) ## unloads the library `lib` proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} = @@ -60,9 +60,9 @@ when defined(posix): proc dlsym(lib: TLibHandle, name: cstring): pointer {. importc, header: "".} - proc LoadLib(path: string): TLibHandle = return dlopen(path, RTLD_NOW) - proc LoadLib(): TLibHandle = return dlopen(nil, RTLD_NOW) - proc UnloadLib(lib: TLibHandle) = dlclose(lib) + proc loadLib(path: string): TLibHandle = return dlopen(path, RTLD_NOW) + proc loadLib(): TLibHandle = return dlopen(nil, RTLD_NOW) + proc unloadLib(lib: TLibHandle) = dlclose(lib) proc symAddr(lib: TLibHandle, name: cstring): pointer = return dlsym(lib, name) @@ -78,14 +78,14 @@ elif defined(windows) or defined(dos): proc FreeLibrary(lib: THINSTANCE) {.importc, header: "", stdcall.} proc winLoadLibrary(path: cstring): THINSTANCE {. importc: "LoadLibraryA", header: "", stdcall.} - proc GetProcAddress(lib: THINSTANCE, name: cstring): pointer {. + proc getProcAddress(lib: THINSTANCE, name: cstring): pointer {. importc: "GetProcAddress", header: "", stdcall.} - proc LoadLib(path: string): TLibHandle = + proc loadLib(path: string): TLibHandle = result = cast[TLibHandle](winLoadLibrary(path)) - proc LoadLib(): TLibHandle = + proc loadLib(): TLibHandle = result = cast[TLibHandle](winLoadLibrary(nil)) - proc UnloadLib(lib: TLibHandle) = FreeLibrary(cast[THINSTANCE](lib)) + proc unloadLib(lib: TLibHandle) = FreeLibrary(cast[THINSTANCE](lib)) proc symAddr(lib: TLibHandle, name: cstring): pointer = result = GetProcAddress(cast[THINSTANCE](lib), name) From 7fe340e7bc309be4751e9719f9966afd1ef566cb Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 00:02:08 +0100 Subject: [PATCH 26/72] year 2014 --- copying.txt | 2 +- readme.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/copying.txt b/copying.txt index 0ff6b7d876..4041ca0272 100644 --- a/copying.txt +++ b/copying.txt @@ -1,7 +1,7 @@ =============================================================================== Nimrod -- a Compiler for Nimrod. http://nimrod-code.org/ -Copyright (C) 2004-2013 Andreas Rumpf. All rights reserved. +Copyright (C) 2004-2014 Andreas Rumpf. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/readme.md b/readme.md index 8d42c66db8..3eaef0b355 100644 --- a/readme.md +++ b/readme.md @@ -61,5 +61,5 @@ allowing you to create commercial applications. Read copying.txt for more details. -Copyright (c) 2004-2013 Andreas Rumpf. +Copyright (c) 2004-2014 Andreas Rumpf. All rights reserved. From 43d1181c0465587f0b354a314b676c1daaec79be Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 00:02:39 +0100 Subject: [PATCH 27/72] year 2014 --- readme.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.txt b/readme.txt index 8d42c66db8..3eaef0b355 100644 --- a/readme.txt +++ b/readme.txt @@ -61,5 +61,5 @@ allowing you to create commercial applications. Read copying.txt for more details. -Copyright (c) 2004-2013 Andreas Rumpf. +Copyright (c) 2004-2014 Andreas Rumpf. All rights reserved. From 4b47bee65407a1ebde5e804cd77fc242c3fa7152 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 00:02:57 +0100 Subject: [PATCH 28/72] deactivated debug output --- compiler/vmgen.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index ff479b74a3..e0ff5b2350 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1480,9 +1480,9 @@ proc genProc(c: PCtx; s: PSym): int = c.gABC(body, opcEof, eofInstr.regA) c.optimizeJumps(result) s.offset = c.prc.maxSlots - if s.name.s == "concatStyleInterpolation": - c.echoCode(result) - echo renderTree(body) + #if s.name.s == "concatStyleInterpolation": + # c.echoCode(result) + # echo renderTree(body) c.prc = oldPrc else: c.prc.maxSlots = s.offset From 3b5c0b27ce2578eceb97128239112653d592707f Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 00:03:30 +0100 Subject: [PATCH 29/72] case consistency for excpt.nim --- lib/system/excpt.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index a3f6669d42..e50ba7b9fa 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -23,7 +23,7 @@ else: proc MessageBoxA(hWnd: cint, lpText, lpCaption: cstring, uType: int): int32 {. header: "", nodecl.} - proc writeToStdErr(msg: CString) = + proc writeToStdErr(msg: cstring) = discard MessageBoxA(0, msg, nil, 0) proc showErrorMessage(data: cstring) = From a5098f1fda974ec8420f3a2f6691077e4aa93484 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 00:03:57 +0100 Subject: [PATCH 30/72] fixed cases --- tests/macros/tstringinterp.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/macros/tstringinterp.nim b/tests/macros/tstringinterp.nim index f030213e0f..a500ed56ec 100644 --- a/tests/macros/tstringinterp.nim +++ b/tests/macros/tstringinterp.nim @@ -9,7 +9,7 @@ proc concat(strings: varargs[string]): string = result = newString(0) for s in items(strings): result.add(s) -template ProcessInterpolations(e: expr) = +template processInterpolations(e: expr) = var s = e[1].strVal for f in interpolatedFragments(s): case f.kind @@ -35,7 +35,7 @@ macro formatStyleInterpolation(e: expr): expr = proc addDollar() = formatString.add("$$") - ProcessInterpolations(e) + processInterpolations(e) result = parseExpr("\"x\" % [y]") result[1].strVal = formatString @@ -50,7 +50,7 @@ macro concatStyleInterpolation(e: expr): expr = proc addExpr(e: PNimrodNode) = args.add(e) proc addDollar() = args.add(newStrLitNode"$") - ProcessInterpolations(e) + processInterpolations(e) result = newCall("concat", args) From 38697234f66f98fad953085412b2ce218521427e Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 00:35:26 +0100 Subject: [PATCH 31/72] mark and sweep GC compiles with --cs:partial --- lib/system/gc_ms.nim | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 2e35969857..e78a4e5cde 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2013 Andreas Rumpf +# (c) Copyright 2014 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -59,11 +59,11 @@ var gch {.rtlThreadVar.}: TGcHeap when not defined(useNimRtl): - InstantiateForRegion(gch.region) + instantiateForRegion(gch.region) template acquire(gch: TGcHeap) = when hasThreadSupport and hasSharedHeap: - AcquireSys(HeapLock) + acquireSys(HeapLock) template release(gch: TGcHeap) = when hasThreadSupport and hasSharedHeap: @@ -90,7 +90,7 @@ proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging result = usrToCell(c).typ -proc unsureAsgnRef(dest: ppointer, src: pointer) {.inline.} = +proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline.} = dest[] = src proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = @@ -114,10 +114,10 @@ when BitsPerPage mod (sizeof(int)*8) != 0: # forward declarations: proc collectCT(gch: var TGcHeap) -proc IsOnStack*(p: pointer): bool {.noinline.} +proc isOnStack*(p: pointer): bool {.noinline.} proc forAllChildren(cell: PCell, op: TWalkOp) proc doOperation(p: pointer, op: TWalkOp) -proc forAllChildrenAux(dest: Pointer, mt: PNimType, op: TWalkOp) +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: TWalkOp) # we need the prototype here for debugging purposes proc prepareDealloc(cell: PCell) = @@ -162,19 +162,19 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: TWalkOp) = if m != nil: forAllSlotsAux(dest, m, op) of nkNone: sysAssert(false, "forAllSlotsAux") -proc forAllChildrenAux(dest: Pointer, mt: PNimType, op: TWalkOp) = +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: TWalkOp) = var d = cast[TAddress](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: - case mt.Kind + case mt.kind of tyRef, tyString, tySequence: # leaf: - doOperation(cast[ppointer](d)[], op) + doOperation(cast[PPointer](d)[], op) of tyObject, tyTuple: forAllSlotsAux(dest, mt.node, op) of tyArray, tyArrayConstr, tyOpenArray: for i in 0..(mt.size div mt.base.size)-1: forAllChildrenAux(cast[pointer](d +% i *% mt.base.size), mt.base, op) - else: nil + else: discard proc forAllChildren(cell: PCell, op: TWalkOp) = gcAssert(cell != nil, "forAllChildren: 1") @@ -184,7 +184,7 @@ proc forAllChildren(cell: PCell, op: TWalkOp) = if marker != nil: marker(cellToUsr(cell), op.int) else: - case cell.typ.Kind + case cell.typ.kind of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: @@ -194,7 +194,7 @@ proc forAllChildren(cell: PCell, op: TWalkOp) = for i in 0..s.len-1: forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +% GenericSeqSize), cell.typ.base, op) - else: nil + else: discard proc rawNewObj(typ: PNimType, size: int, gch: var TGcHeap): pointer = # generates a new object and sets its reference counter to 0 @@ -466,7 +466,7 @@ else: sp = sp +% sizeof(pointer)*8 # last few entries: while sp <=% max: - gcMark(gch, cast[ppointer](sp)[]) + gcMark(gch, cast[PPointer](sp)[]) sp = sp +% sizeof(pointer) # ---------------------------------------------------------------------------- @@ -505,7 +505,7 @@ when not defined(useNimRtl): else: dec(gch.recGcLock) - proc GC_setStrategy(strategy: TGC_Strategy) = nil + proc GC_setStrategy(strategy: TGC_Strategy) = discard proc GC_enableMarkAndSweep() = gch.cycleThreshold = InitialThreshold From d29aa4c5ac6950a9b8c53bedeb9dd0dd9b4f64a2 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 00:41:04 +0100 Subject: [PATCH 32/72] new tester produces json output --- koch.nim | 5 +- tests/specials.nim | 238 ------------------- tests/testament/backend.nim | 12 +- tests/testament/htmlgen.nim | 19 ++ tests/testament/tester.nim | 15 +- tests/tester.nim | 457 ------------------------------------ todo.txt | 5 +- 7 files changed, 30 insertions(+), 721 deletions(-) delete mode 100644 tests/specials.nim delete mode 100644 tests/tester.nim diff --git a/koch.nim b/koch.nim index 35a86a597c..4d2b3bfb71 100644 --- a/koch.nim +++ b/koch.nim @@ -266,8 +266,9 @@ proc tests(args: string) = # we compile the tester with taintMode:on to have a basic # taint mode test :-) exec "nimrod cc --taintMode:on tests/testament/tester" - exec quoteShell(getCurrentDir() / "tests/testament/tester".exe) & " " & - (args|"all") + let tester = quoteShell(getCurrentDir() / "tests/testament/tester".exe) + exec tester & " " & (args|"all") + exec tester & " html" proc temp(args: string) = var output = "compiler" / "nimrod".exe diff --git a/tests/specials.nim b/tests/specials.nim deleted file mode 100644 index 9ced66bbb9..0000000000 --- a/tests/specials.nim +++ /dev/null @@ -1,238 +0,0 @@ -# -# -# Nimrod Tester -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Include for the tester that contains test suites that test special features -## of the compiler. - -# included from tester.nim -# ---------------- ROD file tests --------------------------------------------- - -const - rodfilesDir = "tests/rodfiles" - nimcacheDir = rodfilesDir / "nimcache" - -proc delNimCache() = - try: - removeDir(nimcacheDir) - except EOS: - echo "[Warning] could not delete: ", nimcacheDir - -proc runRodFiles(r: var TResults, options: string) = - template test(filename: expr): stmt = - runSingleTest(r, rodfilesDir / filename, options) - - delNimCache() - - # test basic recompilation scheme: - test "hallo" - test "hallo" - # test incremental type information: - test "hallo2" - delNimCache() - - # test type converters: - test "aconv" - test "bconv" - delNimCache() - - # test G, A, B example from the documentation; test init sections: - test "deada" - test "deada2" - delNimCache() - - # test method generation: - test "bmethods" - test "bmethods2" - delNimCache() - - # test generics: - test "tgeneric1" - test "tgeneric2" - delNimCache() - -proc compileRodFiles(r: var TResults, options: string) = - template test(filename: expr): stmt = - compileSingleTest(r, rodfilesDir / filename, options) - - delNimCache() - # test DLL interfacing: - test "gtkex1" - test "gtkex2" - delNimCache() - -# --------------------- DLL generation tests ---------------------------------- - -proc safeCopyFile(src, dest: string) = - try: - copyFile(src, dest) - except EOS: - echo "[Warning] could not copy: ", src, " to ", dest - -proc runBasicDLLTest(c, r: var TResults, options: string) = - compileSingleTest c, "lib/nimrtl.nim", options & " --app:lib -d:createNimRtl" - compileSingleTest c, "tests/dll/server.nim", - options & " --app:lib -d:useNimRtl" - - when defined(Windows): - # windows looks in the dir of the exe (yay!): - var nimrtlDll = DynlibFormat % "nimrtl" - safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) - else: - # posix relies on crappy LD_LIBRARY_PATH (ugh!): - var libpath = getenv"LD_LIBRARY_PATH".string - if peg"\i '/nimrod' (!'/')* '/lib'" notin libpath: - echo "[Warning] insufficient LD_LIBRARY_PATH" - var serverDll = DynlibFormat % "server" - safeCopyFile("tests/dll" / serverDll, "lib" / serverDll) - - runSingleTest r, "tests/dll/client.nim", options & " -d:useNimRtl" - -proc runDLLTests(r: var TResults, options: string) = - # dummy compile result: - var c = initResults() - - runBasicDLLTest c, r, options - runBasicDLLTest c, r, options & " -d:release" - runBasicDLLTest c, r, options & " --gc:boehm" - runBasicDLLTest c, r, options & " -d:release --gc:boehm" - -proc compileDLLTests(r: var TResults, options: string) = - # dummy run result: - var c = initResults() - - runBasicDLLTest r, c, options - runBasicDLLTest r, c, options & " -d:release" - runBasicDLLTest r, c, options & " --gc:boehm" - runBasicDLLTest r, c, options & " -d:release --gc:boehm" - -# ------------------------------ GC tests ------------------------------------- - -proc runGcTests(r: var TResults, options: string) = - template test(filename: expr): stmt = - runSingleTest(r, "tests/gc" / filename, options) - runSingleTest(r, "tests/gc" / filename, options & " -d:release") - runSingleTest(r, "tests/gc" / filename, options & - " -d:release -d:useRealtimeGC") - runSingleTest(r, "tests/gc" / filename, options & - " --gc:markAndSweep") - runSingleTest(r, "tests/gc" / filename, options & - " -d:release --gc:markAndSweep") - - test "gcbench" - test "gcleak" - test "gcleak2" - test "gctest" - test "gcleak3" - test "weakrefs" - test "cycleleak" - test "closureleak" - -# ------------------------- threading tests ----------------------------------- - -proc runThreadTests(r: var TResults, options: string) = - template test(filename: expr): stmt = - runSingleTest(r, "tests/threads" / filename, options) - runSingleTest(r, "tests/threads" / filename, options & " -d:release") - runSingleTest(r, "tests/threads" / filename, options & " --tlsEmulation:on") - - test "tactors" - test "tactors2" - test "threadex" - # deactivated because output capturing still causes problems sometimes: - #test "trecursive_actor" - #test "threadring" - #test "tthreadanalysis" - #test "tthreadsort" - -proc rejectThreadTests(r: var TResults, options: string) = - rejectSingleTest(r, "tests/threads/tthreadanalysis2", options) - rejectSingleTest(r, "tests/threads/tthreadanalysis3", options) - rejectSingleTest(r, "tests/threads/tthreadheapviolation1", options) - -# ------------------------- IO tests ------------------------------------------ - -proc runIOTests(r: var TResults, options: string) = - # We need readall_echo to be compiled for this test to run. - # dummy compile result: - var c = initResults() - compileSingleTest(c, "tests/system/helpers/readall_echo", options) - runSingleTest(r, "tests/system/io", options) - -# ------------------------- debugger tests ------------------------------------ - -proc compileDebuggerTests(r: var TResults, options: string) = - compileSingleTest(r, "tools/nimgrep", options & - " --debugger:on") - -# ------------------------- JS tests ------------------------------------------ - -proc runJsTests(r: var TResults, options: string) = - template test(filename: expr): stmt = - runSingleTest(r, filename, options & " -d:nodejs", targetJS) - runSingleTest(r, filename, options & " -d:nodejs -d:release", targetJS) - - for t in os.walkFiles("tests/js/t*.nim"): - test(t) - for testfile in ["texceptions", "texcpt1", "texcsub", "tfinally", - "tfinally2", "tfinally3", "tactiontable", "tmultim1", - "tmultim3", "tmultim4"]: - test "tests/run/" & testfile & ".nim" - -# ------------------------- register special tests here ----------------------- -proc runSpecialTests(r: var TResults, options: string) = - runRodFiles(r, options) - #runDLLTests(r, options) - runGCTests(r, options) - runThreadTests(r, options & " --threads:on") - runIOTests(r, options) - - for t in os.walkFiles("tests/patterns/t*.nim"): - runSingleTest(r, t, options) - for t in ["lib/packages/docutils/highlite"]: - runSingleTest(r, t, options) - -proc rejectSpecialTests(r: var TResults, options: string) = - rejectThreadTests(r, options) - -proc findMainFile(dir: string): string = - # finds the file belonging to ".nimrod.cfg"; if there is no such file - # it returns the some ".nim" file if there is only one: - const cfgExt = ".nimrod.cfg" - result = "" - var nimFiles = 0 - for kind, file in os.walkDir(dir): - if kind == pcFile: - if file.endsWith(cfgExt): return file[.. -(cfgExt.len+1)] & ".nim" - elif file.endsWith(".nim"): - if result.len == 0: result = file - inc nimFiles - if nimFiles != 1: result.setlen(0) - -proc compileManyLoc(r: var TResults, options: string) = - for kind, dir in os.walkDir("tests/manyloc"): - if kind == pcDir: - let mainfile = findMainFile(dir) - if mainfile != ".nim": - compileSingleTest(r, mainfile, options) - -proc compileSpecialTests(r: var TResults, options: string) = - compileRodFiles(r, options) - - compileSingleTest(r, "compiler/c2nim/c2nim.nim", options) - compileSingleTest(r, "compiler/pas2nim/pas2nim.nim", options) - - compileDLLTests(r, options) - compileDebuggerTests(r, options) - - compileManyLoc(r, options) - - #var given = callCompiler("nimrod i", "nimrod i", options) - #r.addResult("nimrod i", given.msg, if given.err: reFailure else: reSuccess) - #if not given.err: inc(r.passed) - diff --git a/tests/testament/backend.nim b/tests/testament/backend.nim index bc1f92eba7..5199bb9d6f 100644 --- a/tests/testament/backend.nim +++ b/tests/testament/backend.nim @@ -49,10 +49,10 @@ proc createDb() = # """, []) type - MachineId = distinct int64 + MachineId* = distinct int64 CommitId = distinct int64 -proc `$`(id: MachineId): string {.borrow.} +proc `$`*(id: MachineId): string {.borrow.} proc `$`(id: CommitId): string {.borrow.} var @@ -61,7 +61,7 @@ var proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip -proc getMachine: MachineId = +proc getMachine*(db: TDbConn): MachineId = var name = "hostname"() if name.len == 0: name = when defined(posix): getenv"HOSTNAME".string @@ -76,7 +76,7 @@ proc getMachine: MachineId = result = db.insertId(sql"insert into Machine(name, os, cpu) values (?,?,?)", name, system.hostOS, system.hostCPU).MachineId -proc getCommit: CommitId = +proc getCommit(db: TDbConn): CommitId = const commLen = "commit ".len let hash = "git log -n 1"()[commLen..commLen+10] let branch = "git symbolic-ref --short HEAD"() @@ -115,7 +115,7 @@ proc open*() = db = open(connection="testament.db", user="testament", password="", database="testament") createDb() - thisMachine = getMachine() - thisCommit = getCommit() + thisMachine = getMachine(db) + thisCommit = getCommit(db) proc close*() = close(db) diff --git a/tests/testament/htmlgen.nim b/tests/testament/htmlgen.nim index bc2d8bd37e..eb674a171a 100644 --- a/tests/testament/htmlgen.nim +++ b/tests/testament/htmlgen.nim @@ -159,3 +159,22 @@ proc generateHtml*(filename: string, commit: int) = outfile.write(HtmlEnd) close(db) close(outfile) + +proc generateJson*(filename: string, commit: int) = + const selRow = """select count(*), + sum(result = 'reSuccess'), + sum(result = 'reIgnored') + from TestResult + where [commit] = ? and machine = ? + order by category""" + var db = open(connection="testament.db", user="testament", password="", + database="testament") + let lastCommit = db.getCommit(commit) + + var outfile = open(filename, fmWrite) + + let data = db.getRow(sql(selRow), lastCommit, $backend.getMachine(db)) + + outfile.writeln("""{"total": $#, "passed": $#, "skipped": $#}""" % data) + close(db) + close(outfile) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 54a6de2d0b..fac97cf2ab 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -208,20 +208,6 @@ proc makeTest(test, options: string, cat: Category, action = actionCompile, include categories -proc toJson(res: TResults): PJsonNode = - result = newJObject() - result["total"] = newJInt(res.total) - result["passed"] = newJInt(res.passed) - result["skipped"] = newJInt(res.skipped) - -proc outputJson(reject, compile, run: TResults) = - var doc = newJObject() - doc["reject"] = toJson(reject) - doc["compile"] = toJson(compile) - doc["run"] = toJson(run) - var s = pretty(doc) - writeFile(jsonFile, s) - # proc runCaasTests(r: var TResults) = # for test, output, status, mode in caasTestsRunner(): # r.addResult(test, "", output & "-> " & $mode, @@ -259,6 +245,7 @@ proc main() = var commit = 0 discard parseInt(p.cmdLineRest.string, commit) generateHtml(resultsFile, commit) + generateJson(jsonFile, commit) else: quit usage diff --git a/tests/tester.nim b/tests/tester.nim deleted file mode 100644 index 0e125b1bbb..0000000000 --- a/tests/tester.nim +++ /dev/null @@ -1,457 +0,0 @@ -# -# -# Nimrod Tester -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This program verifies Nimrod against the testcases. - -import - parseutils, strutils, pegs, os, osproc, streams, parsecfg, browsers, json, - marshal, cgi, parseopt #, caas - -const - cmdTemplate = r"nimrod cc --hints:on $# $#" - resultsFile = "testresults.html" - jsonFile = "testresults.json" - Usage = "usage: tester [--print] " & - "reject|compile|run|" & - "merge|special|rodfiles| [nimrod options]\n" & - " or: tester test|comp|rej singleTest" - -type - TTestAction = enum - actionCompile, actionRun, actionReject - TResultEnum = enum - reNimrodcCrash, # nimrod compiler seems to have crashed - reMsgsDiffer, # error messages differ - reFilesDiffer, # expected and given filenames differ - reLinesDiffer, # expected and given line numbers differ - reOutputsDiffer, - reExitcodesDiffer, - reInvalidPeg, - reCodegenFailure, - reCodeNotFound, - reExeNotFound, - reIgnored, # test is ignored - reSuccess # test was successful - - TTarget = enum - targetC, targetCpp, targetObjC, targetJS - - TSpec = object - action: TTestAction - file, cmd: string - outp: string - line, exitCode: int - msg: string - ccodeCheck: string - err: TResultEnum - substr: bool - TResults = object - total, passed, skipped: int - data: string - -# ----------------------- Spec parser ---------------------------------------- - -when not defined(parseCfgBool): - # candidate for the stdlib: - proc parseCfgBool(s: string): bool = - case normalize(s) - of "y", "yes", "true", "1", "on": result = true - of "n", "no", "false", "0", "off": result = false - else: raise newException(EInvalidValue, "cannot interpret as a bool: " & s) - -proc extractSpec(filename: string): string = - const tripleQuote = "\"\"\"" - var x = readFile(filename).string - var a = x.find(tripleQuote) - var b = x.find(tripleQuote, a+3) - # look for """ only in the first section - if a >= 0 and b > a and a < 40: - result = x.substr(a+3, b-1).replace("'''", tripleQuote) - else: - #echo "warning: file does not contain spec: " & filename - result = "" - -when not defined(nimhygiene): - {.pragma: inject.} - -template parseSpecAux(fillResult: stmt) {.immediate.} = - var ss = newStringStream(extractSpec(filename)) - var p {.inject.}: TCfgParser - open(p, ss, filename, 1) - while true: - var e {.inject.} = next(p) - case e.kind - of cfgEof: break - of cfgSectionStart, cfgOption, cfgError: - echo ignoreMsg(p, e) - of cfgKeyValuePair: - fillResult - close(p) - -proc parseSpec(filename: string): TSpec = - result.file = filename - result.msg = "" - result.outp = "" - result.ccodeCheck = "" - result.cmd = cmdTemplate - parseSpecAux: - case normalize(e.key) - of "action": - case e.value.normalize - of "compile": result.action = actionCompile - of "run": result.action = actionRun - of "reject": result.action = actionReject - else: echo ignoreMsg(p, e) - of "file": result.file = e.value - of "line": discard parseInt(e.value, result.line) - of "output": result.outp = e.value - of "outputsub": - result.outp = e.value - result.substr = true - of "exitcode": - discard parseInt(e.value, result.exitCode) - of "errormsg", "msg": result.msg = e.value - of "disabled": - if parseCfgBool(e.value): result.err = reIgnored - of "cmd": result.cmd = e.value - of "ccodecheck": result.ccodeCheck = e.value - else: echo ignoreMsg(p, e) - -# ---------------------------------------------------------------------------- - -let - pegLineError = - peg"{[^(]*} '(' {\d+} ', ' \d+ ') ' ('Error'/'Warning') ':' \s* {.*}" - pegOtherError = peg"'Error:' \s* {.*}" - pegSuccess = peg"'Hint: operation successful'.*" - pegOfInterest = pegLineError / pegOtherError - -proc callCompiler(cmdTemplate, filename, options: string): TSpec = - let c = parseCmdLine(cmdTemplate % [options, filename]) - var p = startProcess(command=c[0], args=c[1.. -1], - options={poStdErrToStdOut, poUseShell}) - let outp = p.outputStream - var suc = "" - var err = "" - var x = newStringOfCap(120) - while outp.readLine(x.TaintedString) or running(p): - if x =~ pegOfInterest: - # `err` should contain the last error/warning message - err = x - elif x =~ pegSuccess: - suc = x - close(p) - result.msg = "" - result.file = "" - result.outp = "" - result.line = -1 - if err =~ pegLineError: - result.file = extractFilename(matches[0]) - result.line = parseInt(matches[1]) - result.msg = matches[2] - elif err =~ pegOtherError: - result.msg = matches[0] - elif suc =~ pegSuccess: - result.err = reSuccess - -proc initResults: TResults = - result.total = 0 - result.passed = 0 - result.skipped = 0 - result.data = "" - -proc readResults(filename: string): TResults = - result = marshal.to[TResults](readFile(filename).string) - -proc writeResults(filename: string, r: TResults) = - writeFile(filename, $$r) - -proc `$`(x: TResults): string = - result = ("Tests passed: $1 / $3
\n" & - "Tests skipped: $2 / $3
\n") % - [$x.passed, $x.skipped, $x.total] - -proc colorResult(r: TResultEnum): string = - case r - of reIgnored: result = "ignored" - of reSuccess: result = "yes" - else: result = "no" - -const - TableHeader4 = "" & - "\n" - TableHeader3 = "
TestExpectedGivenSuccess
" & - "\n" - TableFooter = "
TestGivenSuccess
\n" - HtmlBegin = """ - - Test results - - - - """ - - HtmlEnd = "" - -proc td(s: string): string = - result = s.substr(0, 200).XMLEncode - -proc addResult(r: var TResults, test, expected, given: string, - success: TResultEnum) = - r.data.addf("$#$#$#$#\n", [ - XMLEncode(test), td(expected), td(given), success.colorResult]) - -proc addResult(r: var TResults, test, given: string, - success: TResultEnum) = - r.data.addf("$#$#$#\n", [ - XMLEncode(test), td(given), success.colorResult]) - -proc listResults(reject, compile, run: TResults) = - var s = HtmlBegin - s.add("

Tests to Reject

\n") - s.add($reject) - s.add(TableHeader4 & reject.data & TableFooter) - s.add("


Tests to Compile

\n") - s.add($compile) - s.add(TableHeader3 & compile.data & TableFooter) - s.add("


Tests to Run

\n") - s.add($run) - s.add(TableHeader4 & run.data & TableFooter) - s.add(HtmlEnd) - writeFile(resultsFile, s) - -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: string) = - if strip(expected.msg) notin strip(given.msg): - r.addResult(test, expected.msg, given.msg, reMsgsDiffer) - elif extractFilename(expected.file) != extractFilename(given.file) and - "internal error:" notin expected.msg: - r.addResult(test, expected.file, given.file, reFilesDiffer) - elif expected.line != given.line and expected.line != 0: - r.addResult(test, $expected.line, $given.line, reLinesDiffer) - else: - r.addResult(test, expected.msg, given.msg, reSuccess) - inc(r.passed) - -proc rejectSingleTest(r: var TResults, test, options: string) = - let test = test.addFileExt(".nim") - var t = extractFilename(test) - inc(r.total) - echo t - var expected = parseSpec(test) - if expected.err == reIgnored: - r.addResult(t, "", "", reIgnored) - inc(r.skipped) - else: - var given = callCompiler(expected.cmd, test, options) - cmpMsgs(r, expected, given, t) - -proc reject(r: var TResults, dir, options: string) = - ## handle all the tests that the compiler should reject - for test in os.walkFiles(dir / "t*.nim"): rejectSingleTest(r, test, options) - -proc codegenCheck(test, check, ext: string, given: var TSpec) = - if check.len > 0: - try: - let (path, name, ext2) = test.splitFile - echo path / "nimcache" / name.changeFileExt(ext) - let contents = readFile(path / "nimcache" / name.changeFileExt(ext)).string - if contents.find(check.peg) < 0: - given.err = reCodegenFailure - except EInvalidValue: - given.err = reInvalidPeg - except EIO: - given.err = reCodeNotFound - -proc codegenChecks(test: string, expected: TSpec, given: var TSpec) = - codegenCheck(test, expected.ccodeCheck, ".c", given) - -proc compile(r: var TResults, pattern, options: string) = - for test in os.walkFiles(pattern): - let t = extractFilename(test) - echo t - inc(r.total) - let expected = parseSpec(test) - if expected.err == reIgnored: - r.addResult(t, "", reIgnored) - inc(r.skipped) - else: - var given = callCompiler(expected.cmd, test, options) - if given.err == reSuccess: - codegenChecks(test, expected, given) - r.addResult(t, given.msg, given.err) - if given.err == reSuccess: inc(r.passed) - -proc compileSingleTest(r: var TResults, test, options: string) = - # does not extract the spec because the file is not supposed to have any - let test = test.addFileExt(".nim") - let t = extractFilename(test) - inc(r.total) - echo t - let given = callCompiler(cmdTemplate, test, options) - r.addResult(t, given.msg, given.err) - if given.err == reSuccess: inc(r.passed) - -proc runSingleTest(r: var TResults, test, options: string, target: TTarget) = - var test = test.addFileExt(".nim") - var t = extractFilename(test) - echo t - inc(r.total) - var expected = parseSpec(test) - if expected.err == reIgnored: - r.addResult(t, "", "", reIgnored) - inc(r.skipped) - else: - var given = callCompiler(expected.cmd, test, options) - if given.err != reSuccess: - r.addResult(t, "", given.msg, given.err) - else: - var exeFile: string - if target == targetC: - exeFile = changeFileExt(test, ExeExt) - else: - let (dir, file, ext) = splitFile(test) - exeFile = dir / "nimcache" / file & ".js" - - if existsFile(exeFile): - var (buf, exitCode) = execCmdEx( - (if target==targetJS: "node " else: "") & exeFile) - if exitCode != expected.ExitCode: - r.addResult(t, "exitcode: " & $expected.ExitCode, - "exitcode: " & $exitCode, reExitCodesDiffer) - else: - if strip(buf.string) != strip(expected.outp): - if not (expected.substr and expected.outp in buf.string): - given.err = reOutputsDiffer - if given.err == reSuccess: - codeGenChecks(test, expected, given) - if given.err == reSuccess: inc(r.passed) - r.addResult(t, expected.outp, buf.string, given.err) - else: - r.addResult(t, expected.outp, "executable not found", reExeNotFound) - -proc runSingleTest(r: var TResults, test, options: string) = - runSingleTest(r, test, options, targetC) - -proc run(r: var TResults, dir, options: string) = - for test in os.walkFiles(dir / "t*.nim"): runSingleTest(r, test, options) - -include specials - -proc compileExample(r: var TResults, pattern, options: string) = - for test in os.walkFiles(pattern): compileSingleTest(r, test, options) - -proc toJson(res: TResults): PJsonNode = - result = newJObject() - result["total"] = newJInt(res.total) - result["passed"] = newJInt(res.passed) - result["skipped"] = newJInt(res.skipped) - -proc outputJSON(reject, compile, run: TResults) = - var doc = newJObject() - doc["reject"] = toJson(reject) - doc["compile"] = toJson(compile) - doc["run"] = toJson(run) - var s = pretty(doc) - writeFile(jsonFile, s) - -# proc runCaasTests(r: var TResults) = -# for test, output, status, mode in caasTestsRunner(): -# r.addResult(test, "", output & "-> " & $mode, -# if status: reSuccess else: reOutputsDiffer) - -proc main() = - os.putenv "NIMTEST_NO_COLOR", "1" - os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" - - const - compileJson = "compile.json" - runJson = "run.json" - rejectJson = "reject.json" - - var optPrintResults = false - var p = initOptParser() - p.next() - if p.kind == cmdLongoption: - case p.key.string - of "print": optPrintResults = true - else: quit usage - p.next() - if p.kind != cmdArgument: quit usage - var action = p.key.string.normalize - p.next() - var r = initResults() - case action - of "reject": - reject(r, "tests/reject", p.cmdLineRest.string) - rejectSpecialTests(r, p.cmdLineRest.string) - writeResults(rejectJson, r) - of "compile": - compile(r, "tests/compile/t*.nim", p.cmdLineRest.string) - compile(r, "tests/ccg/t*.nim", p.cmdLineRest.string) - compile(r, "tests/js.nim", p.cmdLineRest.string) - compileExample(r, "lib/pure/*.nim", p.cmdLineRest.string) - compileExample(r, "examples/*.nim", p.cmdLineRest.string) - compileExample(r, "examples/gtk/*.nim", p.cmdLineRest.string) - compileExample(r, "examples/talk/*.nim", p.cmdLineRest.string) - compileSpecialTests(r, p.cmdLineRest.string) - writeResults(compileJson, r) - of "run": - run(r, "tests/run", p.cmdLineRest.string) - runSpecialTests(r, p.cmdLineRest.string) - writeResults(runJson, r) - of "special": - runSpecialTests(r, p.cmdLineRest.string) - # runCaasTests(r) - writeResults(runJson, r) - of "rodfiles": - runRodFiles(r, p.cmdLineRest.string) - writeResults(runJson, r) - of "js": - if existsFile(runJSon): - r = readResults(runJson) - runJsTests(r, p.cmdLineRest.string) - writeResults(runJson, r) - of "merge": - var rejectRes = readResults(rejectJson) - var compileRes = readResults(compileJson) - var runRes = readResults(runJson) - listResults(rejectRes, compileRes, runRes) - outputJSON(rejectRes, compileRes, runRes) - of "dll": - runDLLTests r, p.cmdLineRest.string - of "gc": - runGCTests(r, p.cmdLineRest.string) - of "test": - if p.kind != cmdArgument: quit usage - var testFile = p.key.string - p.next() - runSingleTest(r, testFile, p.cmdLineRest.string) - of "comp", "rej": - if p.kind != cmdArgument: quit usage - var testFile = p.key.string - p.next() - if peg"'/reject/'" in testFile or action == "rej": - rejectSingleTest(r, testFile, p.cmdLineRest.string) - elif peg"'/compile/'" in testFile or action == "comp": - compileSingleTest(r, testFile, p.cmdLineRest.string) - else: - runSingleTest(r, testFile, p.cmdLineRest.string) - else: - quit usage - - if optPrintResults: echo r, r.data - -if paramCount() == 0: - quit usage -main() - diff --git a/todo.txt b/todo.txt index d0aec9c8cc..bad9373a14 100644 --- a/todo.txt +++ b/todo.txt @@ -1,14 +1,11 @@ version 0.9.4 ============= -- better debugging support for writes to locations -- document new templating symbol binding rules -- fix eval in macros.nim - Bugs ==== +- fix eval in macros.nim - new VM: - implement overflow checking - bug: 'type T = ref T' not recognized as illegal recursion From b96aa59aefb5aaf24e839e4419d536a9556953a9 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 01:07:02 +0100 Subject: [PATCH 33/72] merge issue #707 by hand --- doc/manual.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual.txt b/doc/manual.txt index 260f0807af..baa0f33668 100644 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -4672,6 +4672,10 @@ These rules ensure that the construction is tied to a variable and can easily be destructed at its scope exit. Later versions of the language will improve the support of destructors. +Be aware that destructors are not called for objects allocated with ``new``. +This may change in future versions of language, but for now use +the ``finalizer`` parameter to ``new``. + delegator pragma ---------------- From 38db9371935f482fb5205124030a3ec25c8ff2d5 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 01:22:31 +0100 Subject: [PATCH 34/72] merged #811 against devel --- lib/pure/os.nim | 2 +- lib/pure/osproc.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 1f42d0d58e..8cb3919a7a 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1013,7 +1013,7 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## the process has finished. To execute a program without having a ## shell involved, use the `execProcess` proc of the `osproc` ## module. - result = c_system(command) + result = c_system(command) shr 8 # Environment handling cannot be put into RTL, because the ``envPairs`` # iterator depends on ``environment``. diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 676707abbc..0714da28e0 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -788,7 +788,7 @@ elif not defined(useNimRtl): proc csystem(cmd: cstring): cint {.nodecl, importc: "system".} proc execCmd(command: string): int = - result = csystem(command) + result = csystem(command) shr 8 proc createFdSet(fd: var TFdSet, s: seq[PProcess], m: var int) = FD_ZERO(fd) From 435610e0afd1fc50d550c48e49f44fc5e9162bd2 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 01:38:19 +0100 Subject: [PATCH 35/72] merged #823 against devel --- lib/pure/sockets.nim | 7 ++++--- lib/wrappers/openssl.nim | 24 ++++++++++++++---------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index fd64031181..76d37879b6 100644 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -311,7 +311,8 @@ when defined(ssl): newCTX.SSLCTXSetVerify(SSLVerifyNone, nil) if newCTX == nil: SSLError() - + + discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY) newCTX.loadCertificates(certFile, keyFile) return PSSLContext(newCTX) @@ -1291,14 +1292,14 @@ proc readLine*(socket: TSocket, line: var TaintedString, timeout = -1) {. var c: char discard waitFor(socket, waited, timeout, 1, "readLine") var n = recv(socket, addr(c), 1) - if n < 0: osError(osLastError()) + if n < 0: socket.socketError() elif n == 0: return if c == '\r': discard waitFor(socket, waited, timeout, 1, "readLine") n = peekChar(socket, c) if n > 0 and c == '\L': discard recv(socket, addr(c), 1) - elif n <= 0: osError(osLastError()) + elif n <= 0: socket.socketError() addNLIfEmpty() return elif c == '\L': diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index af72d04ebf..90c398dceb 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -268,14 +268,22 @@ proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSS proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} -proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLSSLName, importc.} +when not defined(windows): + proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, + dynlib: DLLSSLName, importc.} proc CRYPTO_malloc_init*() = when not defined(windows): CRYPTO_set_mem_functions(alloc, realloc, dealloc) -when True: - nil +proc SSL_CTX_ctrl*(ctx: PSSL_CTX, cmd: cInt, larg: int, parg: pointer): int{. + cdecl, dynlib: DLLSSLName, importc.} + +proc SSLCTXSetMode*(ctx: PSSL_CTX, mode: int): int = + result = SSL_CTX_ctrl(ctx, SSL_CTRL_MODE, mode, nil) + +when true: + discard else: proc SslCtxSetCipherList*(arg0: PSSL_CTX, str: cstring): cInt{.cdecl, dynlib: DLLSSLName, importc.} @@ -288,7 +296,6 @@ else: proc SslCTXCtrl*(ctx: PSSL_CTX, cmd: cInt, larg: int, parg: Pointer): int{. cdecl, dynlib: DLLSSLName, importc.} - proc SSLCTXSetMode*(ctx: PSSL_CTX, mode: int): int proc SSLSetMode*(s: PSSL, mode: int): int proc SSLCTXGetMode*(ctx: PSSL_CTX): int proc SSLGetMode*(s: PSSL): int @@ -417,15 +424,12 @@ else: enc: cInt){.cdecl, dynlib: DLLUtilName, importc.} # implementation - proc SSLCTXSetMode(ctx: PSSL_CTX, mode: int): int = - Result = SslCTXCtrl(ctx, SSL_CTRL_MODE, mode, nil) - proc SSLSetMode(s: PSSL, mode: int): int = - Result = SSLctrl(s, SSL_CTRL_MODE, mode, nil) + result = SSLctrl(s, SSL_CTRL_MODE, mode, nil) proc SSLCTXGetMode(ctx: PSSL_CTX): int = - Result = SSLCTXctrl(ctx, SSL_CTRL_MODE, 0, nil) + result = SSLCTXctrl(ctx, SSL_CTRL_MODE, 0, nil) proc SSLGetMode(s: PSSL): int = - Result = SSLctrl(s, SSL_CTRL_MODE, 0, nil) + result = SSLctrl(s, SSL_CTRL_MODE, 0, nil) From 6fda10aed0287d50ebc3c6192d0dd64e710a1f72 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 1 Feb 2014 18:39:39 -0600 Subject: [PATCH 36/72] added test specs for the tester --- tests/sets/testequivalence.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/sets/testequivalence.nim b/tests/sets/testequivalence.nim index 8a83e2a213..7c5d9e3e90 100644 --- a/tests/sets/testequivalence.nim +++ b/tests/sets/testequivalence.nim @@ -1,3 +1,6 @@ +discard """ + output: '''''' +""" import unittest import sets From eeae68f9e445154bc9c0e965904478de8ef24ab4 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 01:57:56 +0100 Subject: [PATCH 37/72] added the parts of #541 that I like --- doc/tut1.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/tut1.txt b/doc/tut1.txt index 1ba3253c8e..b70f40f4af 100644 --- a/doc/tut1.txt +++ b/doc/tut1.txt @@ -1321,6 +1321,30 @@ In this example ``$`` is applied to any argument that is passed to the parameter ``a``. Note that ``$`` applied to strings is a nop. +Slices +------ + +Slices look similar to subranges types in syntax but are used in a different +context. A slice is just an object of type TSlice which contains two bounds, +`a` and `b`. By itself a slice is not very useful, but other collection types +define operators which accept TSlice objects to define ranges. + +.. code-block:: nimrod + + var + a = "Nimrod is a progamming language" + b = "Slices are useless." + + echo a[10..15] # --> 'a prog' + b[11.. -2] = "useful" + echo b # --> 'Slices are useful.' + +In the previous example slices are used to modify a part of a string, and even +a negative index is used. The slice's bounds can hold any value supported by +their type, but it is the proc using the slice object which defines what values +are accepted. + + Tuples ------ From 69bc0779378bf3684d2138d2b6c0a423ae56f2c6 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 2 Feb 2014 01:01:12 +0000 Subject: [PATCH 38/72] Fixes docgen for logging module. --- lib/pure/logging.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index 8158cbc2aa..284384b37e 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -106,10 +106,10 @@ proc substituteLog(frmt: string): string = of "appname": result.add(app.splitFile.name) method log*(logger: PLogger, level: TLevel, - frmt: string, args: varargs[string, `$`]) = + frmt: string, args: varargs[string, `$`]) {.raises: [EBase], tags: [FTime, FWriteIO, FReadIO].} = ## Override this method in custom loggers. Default implementation does ## nothing. - nil + discard method log*(logger: PConsoleLogger, level: TLevel, frmt: string, args: varargs[string, `$`]) = From 798295a9b6611468bf195de602bb0de41d91d1ac Mon Sep 17 00:00:00 2001 From: EXetoC Date: Sun, 2 Feb 2014 02:22:42 +0100 Subject: [PATCH 39/72] A first attempt at fixing the MongoDB modules. --- lib/impure/db_mongo.nim | 4 ++-- lib/wrappers/mongo.nim | 52 +++++++++++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/lib/impure/db_mongo.nim b/lib/impure/db_mongo.nim index d012f677f8..dc8a808f24 100644 --- a/lib/impure/db_mongo.nim +++ b/lib/impure/db_mongo.nim @@ -58,7 +58,7 @@ proc open*(host: string = defaultHost, port: int = defaultPort): TDbConn {. ## be established. init(result) - let x = connect(result, host, port.cint) + let x = client(result, host, port.cint) if x != 0'i32: dbError(result, "cannot open: " & host) @@ -119,7 +119,7 @@ proc insertId*(db: var TDbConn, namespace: string, data: PJsonNode): TOid {. ## the generated OID for the ``_id`` field. result = genOid() var x = jsonToBSon(data, result) - insert(db, namespace, x) + insert(db, namespace, x, nil) destroy(x) proc insert*(db: var TDbConn, namespace: string, data: PJsonNode) {. diff --git a/lib/wrappers/mongo.nim b/lib/wrappers/mongo.nim index 6673e8ddf7..098b4f4d3f 100644 --- a/lib/wrappers/mongo.nim +++ b/lib/wrappers/mongo.nim @@ -109,11 +109,12 @@ type cur*: cstring dataSize*: cint finished*: TBsonBool - stack*: array[0..32 - 1, cint] + ownsData*: TBsonBool + err*: cint + stackSize*: cint stackPos*: cint - err*: cint ## Bitfield representing errors or warnings on this buffer - errstr*: cstring ## A string representation of the most recent error - ## or warning. + stackPtr*: ptr csize + stack*: array[0..32 - 1, csize] TDate* = int64 @@ -141,6 +142,7 @@ proc print*(TBson: cstring, depth: cint) {.stdcall, importc: "bson_print_raw", dynlib: bsondll.} ## Print a string representation of a BSON object up to `depth`. + proc data*(b: var TBson): cstring{.stdcall, importc: "bson_data", dynlib: bsondll.} ## Return a pointer to the raw buffer stored by this bson object. @@ -590,19 +592,30 @@ type hosts*: ptr THostPort ## List of host/ports given by the replica set name*: cstring ## Name of the replica set. primary_connected*: TBsonBool ## Primary node connection status. + + TWriteConcern*{.pure, final.} = object ## mongo_write_concern + w*: cint + wtimeout*: cint + j*: cint + fsync*: cint + mode*: cstring + cmd*: TBSon TMongo*{.pure, final.} = object ## mongo - primary*: ptr THostPort ## Primary connection info. - replset*: ptr TReplSet ## replset object if connected to a replica set. - sock*: cint ## Socket file descriptor. - flags*: cint ## Flags on this connection object. - conn_timeout_ms*: cint ## Connection timeout in milliseconds. - op_timeout_ms*: cint ## Read and write timeout in milliseconds. - connected*: TBsonBool ## Connection status. - err*: TError ## Most recent driver error code. - errstr*: array[0..128 - 1, char] ## String version of most recent driver error code. - lasterrcode*: cint ## getlasterror code given by the server on error. - lasterrstr*: cstring ## getlasterror string generated by server. + primary*: ptr THostPort ## Primary connection info. + replset*: ptr TReplSet ## replset object if connected to a replica set. + sock*: cint ## Socket file descriptor. + flags*: cint ## Flags on this connection object. + conn_timeout_ms*: cint ## Connection timeout in milliseconds. + op_timeout_ms*: cint ## Read and write timeout in milliseconds. + max_bson_size*: cint ## Largest BSON object allowed on this connection. + connected*: TBsonBool ## Connection status. + write_concern*: TWriteConcern ## The default write concern. + err*: TError ## Most recent driver error code. + errcode*: cint ## Most recent errno or WSAGetLastError(). + errstr*: array[0..128 - 1, char] ## String version of most recent driver error code. + lasterrcode*: cint ## getlasterror code given by the server on error. + lasterrstr*: array[0..128 - 1, char] ## getlasterror string generated by server. TCursor*{.pure, final.} = object ## cursor reply*: ptr TReply ## reply is owned by cursor @@ -654,7 +667,11 @@ proc init*(conn: var TMongo){.stdcall, importc: "mongo_init", dynlib: mongodll.} proc connect*(conn: var TMongo, host: cstring = defaultHost, port: cint = defaultPort): cint {.stdcall, - importc: "mongo_connect", dynlib: mongodll.} + importc: "mongo_connect", dynlib: mongodll, deprecated.} + ## Connect to a single MongoDB server. +proc client*(conn: var TMongo, host: cstring = defaultHost, + port: cint = defaultPort): cint {.stdcall, + importc: "mongo_client", dynlib: mongodll.} ## Connect to a single MongoDB server. proc replsetInit*(conn: var TMongo, name: cstring){.stdcall, @@ -714,7 +731,8 @@ proc destroy*(conn: var TMongo){.stdcall, importc: "mongo_destroy", ## You must always call this function when finished with the connection ## object. -proc insert*(conn: var TMongo, ns: cstring, data: var TBson): cint{.stdcall, +proc insert*(conn: var TMongo, ns: cstring, data: var TBson, + custom_write_concern: ptr TWriteConcern): cint{.stdcall, importc: "mongo_insert", dynlib: mongodll, discardable.} ## Insert a BSON document into a MongoDB server. This function ## will fail if the supplied BSON struct is not UTF-8 or if From 7cd5e79464c13b6a8047bdc3d58f0ddc4069431b Mon Sep 17 00:00:00 2001 From: Clay Sweetser Date: Sat, 1 Feb 2014 20:56:34 -0500 Subject: [PATCH 40/72] Added symlink procs --- lib/pure/os.nim | 74 +++++++++++++++++++++++++++++++++++++++-- lib/windows/winlean.nim | 18 +++++++++- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 8cb3919a7a..4bdb0e7e7e 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -287,12 +287,18 @@ proc osLastError*(): TOSErrorCode = result = TOSErrorCode(errno) {.pop.} -proc unixToNativePath*(path: string): string {. +proc unixToNativePath*(path: string, drive=""): string {. noSideEffect, rtl, extern: "nos$1".} = ## Converts an UNIX-like path to a native one. ## ## On an UNIX system this does nothing. Else it converts ## '/', '.', '..' to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + when defined(unix): result = path else: @@ -300,7 +306,10 @@ proc unixToNativePath*(path: string): string {. if path[0] == '/': # an absolute path when doslike: - result = r"C:\" + if drive != "": + result = drive & ":" & DirSep + else: + result = $DirSep elif defined(macos): result = "" # must not start with ':' else: @@ -387,6 +396,21 @@ proc existsDir*(dir: string): bool {.rtl, extern: "nos$1", tags: [FReadDir].} = var res: TStat return stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) +proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", + tags: [FReadDir].} = + ## Returns true iff the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + when defined(windows): + when useWinUnicode: + wrapUnary(a, GetFileAttributesW, link) + else: + var a = GetFileAttributesA(link) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 + else: + var res: TStat + return lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) + proc fileExists*(filename: string): bool {.inline.} = ## Synonym for existsFile existsFile(filename) @@ -1221,6 +1245,8 @@ iterator walkDir*(dir: string): tuple[kind: TPathComponent, path: string] {. if not skipFindData(f): if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: k = pcDir + if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) yield (k, dir / extractFilename(getFilename(f))) if findNextFile(h, f) == 0'i32: break findClose(h) @@ -1245,6 +1271,10 @@ iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string {. tags: [FReadDir].} = ## walks over the directory `dir` and yields for each file in `dir`. The ## full path for each file is returned. + ## **Warning**: + ## Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## ## Walking is recursive. `filter` controls the behaviour of the iterator: ## ## --------------------- --------------------------------------------- @@ -1336,6 +1366,46 @@ proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", copyDir(path, dest / noSource) else: discard +proc createSymlink*(src, dest: string) = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a lonk + ## + ## **Warning**: + ## Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators). + when defined(Windows): + let flag = dirExists(src).int32 + when useWinUnicode: + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if CreateSymbolicLinkW(wDst, wSrc, flag) == 0 or GetLastError() != 0: + osError(osLastError()) + else: + if CreateSymbolicLinkA(dest, src, flag) == 0 or GetLastError() != 0: + osError(osLastError()) + else: + if symlink(src, dest) != 0: + OSError(OSLastError()) + +proc createHardlink*(src, dest: string) = + ## Create a hard link at `dest` which points to the item specified + ## by `src`. + ## + ## **Warning**: Most OS's restrict the creation of hard links to + ## root users (administrators) . + when defined(Windows): + when useWinUnicode: + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createHardLinkW(wDst, wSrc, nil) == 0: + OSError(OSLastError()) + else: + if createHardLinkA(dest, src, nil) == 0: + OSError(OSLastError()) + else: + if link(src, dest) != 0: + OSError(OSLastError()) + proc parseCmdLine*(c: string): seq[string] {. noSideEffect, rtl, extern: "nos$1".} = ## Splits a command line into several components; diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 91c6495cec..3aec2bd525 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -204,7 +204,22 @@ else: proc getModuleFileNameA*(handle: THandle, buf: CString, size: int32): int32 {. importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.} - + +when useWinUnicode: + proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: wideCString, + flags: DWORD): int32 {. + importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall.} + proc createHardLinkW*(lpFileName, lpExistingFileName: wideCString, + security: Pointer=nil): int32 {. + importc:"CreateHardLinkW", dynlib: "kernel32", stdcall.} +else: + proc createSymbolicLinkA*(lpSymlinkFileName, lpTargetFileName: cstring, + flags: DWORD): int32 {. + importc:"CreateSymbolicLinkA", dynlib: "kernel32", stdcall.} + proc createHardLinkA*(lpFileName, lpExistingFileName: cstring, + security: Pointer=nil): int32 {. + importc:"CreateHardLinkA", dynlib: "kernel32", stdcall.} + const FILE_ATTRIBUTE_ARCHIVE* = 32'i32 FILE_ATTRIBUTE_COMPRESSED* = 2048'i32 @@ -212,6 +227,7 @@ const FILE_ATTRIBUTE_DIRECTORY* = 16'i32 FILE_ATTRIBUTE_HIDDEN* = 2'i32 FILE_ATTRIBUTE_READONLY* = 1'i32 + FILE_ATTRIBUTE_REPARSE_POINT* = 1024'i32 FILE_ATTRIBUTE_SYSTEM* = 4'i32 FILE_ATTRIBUTE_TEMPORARY* = 256'i32 From 4b94aba6de5da797c339dc6645e72b1070b92c31 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 03:03:38 +0100 Subject: [PATCH 41/72] fixes #735 --- lib/pure/smtp.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index 6a3f652795..88afeb5895 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -66,13 +66,15 @@ proc checkReply(smtp: var TSMTP, reply: string) = if not line.string.startswith(reply): quitExcpt(smtp, "Expected " & reply & " reply, got: " & line.string) +const compiledWithSsl = defined(ssl) + proc connect*(address: string, port = 25, ssl = false, debug = false): TSMTP = ## Establishes a connection with a SMTP server. ## May fail with EInvalidReply or with a socket error. result.sock = socket() if ssl: - when defined(ssl): + when compiledWithSsl: let ctx = newContext(verifyMode = CVerifyNone) ctx.wrapSocket(result.sock) else: From 47e4f9698cca68a96fbffa6acc0c6d39a6f6420b Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 10:00:10 +0100 Subject: [PATCH 42/72] fixes #844 --- compiler/semdata.nim | 2 +- compiler/semexprs.nim | 4 ++-- compiler/semstmts.nim | 10 ++++++---- tests/exprs/tstmtexprs.nim | 12 ++++++++++++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 980abb865e..8b97f36859 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -42,7 +42,7 @@ type TExprFlag* = enum efLValue, efWantIterator, efInTypeof, efWantStmt, efDetermineType, - efAllowDestructor + efAllowDestructor, efWantValue TExprFlags* = set[TExprFlag] PContext* = ref TContext diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index a384c41fdc..6c76795780 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -35,7 +35,7 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result.typ = errorType(c) proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = - result = semExpr(c, n, flags) + result = semExpr(c, n, flags+{efWantValue}) if result.isNil or result.kind == nkEmpty: # do not produce another redundant error message: #raiseRecoverableError("") @@ -1993,7 +1993,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result = semStaticExpr(c, n) of nkAsgn: result = semAsgn(c, n) of nkBlockStmt, nkBlockExpr: result = semBlock(c, n) - of nkStmtList, nkStmtListExpr: result = semStmtList(c, n) + of nkStmtList, nkStmtListExpr: result = semStmtList(c, n, flags) of nkRaiseStmt: result = semRaise(c, n) of nkVarSection: result = semVarOrLet(c, n, skVar) of nkLetSection: result = semVarOrLet(c, n, skLet) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index caa719c7e1..25353fdd7b 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1206,7 +1206,7 @@ proc usesResult(n: PNode): bool = for c in n: if usesResult(c): return true -proc semStmtList(c: PContext, n: PNode): PNode = +proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode = # these must be last statements in a block: const LastBlockStmts = {nkRaiseStmt, nkReturnStmt, nkBreakStmt, nkContinueStmt} @@ -1247,12 +1247,14 @@ proc semStmtList(c: PContext, n: PNode): PNode = if n.sons[i].typ == enforceVoidContext or usesResult(n.sons[i]): voidContext = true n.typ = enforceVoidContext - if i != last or voidContext or c.inTypeClass > 0: + if i == last and efWantValue in flags: + n.typ = n.sons[i].typ + if not isEmptyType(n.typ): n.kind = nkStmtListExpr + elif i != last or voidContext or c.inTypeClass > 0: discardCheck(c, n.sons[i]) else: n.typ = n.sons[i].typ - if not isEmptyType(n.typ): - n.kind = nkStmtListExpr + if not isEmptyType(n.typ): n.kind = nkStmtListExpr case n.sons[i].kind of nkVarSection, nkLetSection: let (outer, inner) = insertDestructors(c, n.sons[i]) diff --git a/tests/exprs/tstmtexprs.nim b/tests/exprs/tstmtexprs.nim index 497a2f6d05..8149ec4b88 100644 --- a/tests/exprs/tstmtexprs.nim +++ b/tests/exprs/tstmtexprs.nim @@ -69,3 +69,15 @@ proc semiProblem() = if false: echo "aye"; echo "indeed" semiProblem() + + +# bug #844 + +import json +proc parseResponse(): PJsonNode = + result = % { "key1": % { "key2": % "value" } } + for key, val in result["key1"]: + var excMsg = key & "(" + if (var n=result["key2"]; n != nil): + excMsg &= n.str + raise newException(ESynch, excMsg) From 7196c7637e00581f94ca9c02782f34ef7d480e2f Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 10:07:29 +0100 Subject: [PATCH 43/72] bootstraps again --- lib/pure/os.nim | 20 ++++++++++---------- lib/windows/winlean.nim | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 4bdb0e7e7e..245a8446bb 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -402,9 +402,9 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", ## regardless of whether the link points to a directory or file. when defined(windows): when useWinUnicode: - wrapUnary(a, GetFileAttributesW, link) + wrapUnary(a, getFileAttributesW, link) else: - var a = GetFileAttributesA(link) + var a = getFileAttributesA(link) if a != -1'i32: result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 else: @@ -1378,14 +1378,14 @@ proc createSymlink*(src, dest: string) = when useWinUnicode: var wSrc = newWideCString(src) var wDst = newWideCString(dest) - if CreateSymbolicLinkW(wDst, wSrc, flag) == 0 or GetLastError() != 0: - osError(osLastError()) + if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: + osError(osLastError()) else: - if CreateSymbolicLinkA(dest, src, flag) == 0 or GetLastError() != 0: - osError(osLastError()) + if createSymbolicLinkA(dest, src, flag) == 0 or getLastError() != 0: + osError(osLastError()) else: if symlink(src, dest) != 0: - OSError(OSLastError()) + osError(osLastError()) proc createHardlink*(src, dest: string) = ## Create a hard link at `dest` which points to the item specified @@ -1398,13 +1398,13 @@ proc createHardlink*(src, dest: string) = var wSrc = newWideCString(src) var wDst = newWideCString(dest) if createHardLinkW(wDst, wSrc, nil) == 0: - OSError(OSLastError()) + osError(osLastError()) else: if createHardLinkA(dest, src, nil) == 0: - OSError(OSLastError()) + osError(osLastError()) else: if link(src, dest) != 0: - OSError(OSLastError()) + osError(osLastError()) proc parseCmdLine*(c: string): seq[string] {. noSideEffect, rtl, extern: "nos$1".} = diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 3aec2bd525..ee5fe06470 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -206,18 +206,18 @@ else: importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.} when useWinUnicode: - proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: wideCString, + proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, flags: DWORD): int32 {. importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall.} - proc createHardLinkW*(lpFileName, lpExistingFileName: wideCString, - security: Pointer=nil): int32 {. + proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, + security: pointer=nil): int32 {. importc:"CreateHardLinkW", dynlib: "kernel32", stdcall.} else: proc createSymbolicLinkA*(lpSymlinkFileName, lpTargetFileName: cstring, flags: DWORD): int32 {. importc:"CreateSymbolicLinkA", dynlib: "kernel32", stdcall.} proc createHardLinkA*(lpFileName, lpExistingFileName: cstring, - security: Pointer=nil): int32 {. + security: pointer=nil): int32 {. importc:"CreateHardLinkA", dynlib: "kernel32", stdcall.} const From c30f6cfcf11cf5e61d708db476d7a6fcb62aab23 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Feb 2014 14:34:37 +0100 Subject: [PATCH 44/72] cleaned up command expressions --- compiler/parser.nim | 11 +++++------ doc/grammar.txt | 3 +-- doc/manual.txt | 18 +++++++++++++----- tests/parser/tcommand_as_expr.nim | 15 ++++++++++----- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 4497e360aa..7c740559c3 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -642,8 +642,7 @@ proc primarySuffix(p: var TParser, r: PNode): PNode = #| | '.' optInd ('type' | 'addr' | symbol) generalizedLit? #| | '[' optInd indexExprList optPar ']' #| | '{' optInd indexExprList optPar '}' - #| | &( '`'|IDENT|literal|'cast') expr ^+ ',' # command syntax - #| (doBlock | macroColon)? + #| | &( '`'|IDENT|literal|'cast') expr # command syntax result = r while p.tok.indent < 0: case p.tok.tokType @@ -680,10 +679,10 @@ proc primarySuffix(p: var TParser, r: PNode): PNode = if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) - if p.tok.tokType == tkDo: - parseDoBlocks(p, result) - else: - result = parseMacroColon(p, result) + if p.tok.tokType == tkDo: + parseDoBlocks(p, result) + else: + result = parseMacroColon(p, result) break else: break diff --git a/doc/grammar.txt b/doc/grammar.txt index 54c2217d87..63e898e110 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -59,8 +59,7 @@ primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks? | '.' optInd ('type' | 'addr' | symbol) generalizedLit? | '[' optInd indexExprList optPar ']' | '{' optInd indexExprList optPar '}' - | &( '`'|IDENT|literal|'cast') expr ^+ ',' # command syntax - (doBlock | macroColon)? + | &( '`'|IDENT|literal|'cast') expr # command syntax condExpr = expr colcom expr optInd ('elif' expr colcom expr optInd)* 'else' colcom expr diff --git a/doc/manual.txt b/doc/manual.txt index baa0f33668..2ccd4c1e9b 100644 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -1315,12 +1315,9 @@ Examples: .. code-block:: nimrod - type - TCallback = proc (x: int) {.cdecl.} + proc printItem(x: int) = ... - proc printItem(x: Int) = ... - - proc forEach(c: TCallback) = + proc forEach(c: proc (x: int) {.cdecl.}) = ... forEach(printItem) # this will NOT work because calling conventions differ @@ -2459,6 +2456,17 @@ notation. (Thus an operator can have more than two parameters): assert `*+`(3, 4, 6) == `*`(a, `+`(b, c)) + +Command invocation syntax +------------------------- + +Routines can be invoked without the ``()`` if the call is syntactially +a statement. This `command invocation syntax`:idx: also works for +expressions, but then only a single argument may follow. This restriction +means ``echo f 1, f 2`` is parsed as ``echo(f(1), f(2))`` and not as +``echo(f(1, f(2)))``. + + Closures -------- diff --git a/tests/parser/tcommand_as_expr.nim b/tests/parser/tcommand_as_expr.nim index f6868a2fc7..a9747b0acd 100644 --- a/tests/parser/tcommand_as_expr.nim +++ b/tests/parser/tcommand_as_expr.nim @@ -1,12 +1,17 @@ discard """ - output: "12" + output: '''140 +5-120''' """ +proc optarg(x:int):int = x +proc singlearg(x:int):int = 20*x +echo optarg 1, singlearg 2 + + proc foo(x: int): int = x-1 proc foo(x, y: int): int = x-y -let x = foo 7.foo, # comment here - foo(1, foo 8) -# 12 = 6 - -6 -echo x +let x = optarg foo 7.foo +let y = singlearg foo(1, foo 8) +echo x, y From 700f32e81470c16ff65bc83d256ad1368f05e4ad Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Sun, 26 Jan 2014 20:15:29 +0100 Subject: [PATCH 45/72] Adds convenience mapIt templates. --- lib/pure/collections/sequtils.nim | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 3993f1ccc7..b2f72ee146 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -276,6 +276,38 @@ template foldr*(sequence, operation: expr): expr = result = operation result +template mapIt*(seq1, typ, pred: expr): expr = + ## Convenience template around the ``map`` proc to reduce typing. + ## + ## The template injects the ``it`` variable which you can use directly in an + ## expression. You also need to pass as `typ` the type of the expression, + ## since the new returned sequence can have a different type than the + ## original. Example: + ## + ## .. code-block:: nimrod + ## let + ## nums = @[1, 2, 3, 4] + ## strings = nums.mapIt(string, $(4 * it)) + var result {.gensym.}: seq[typ] = @[] + for it {.inject.} in items(seq1): + result.add(pred) + result + +template mapIt*(varSeq, pred: expr) = + ## Convenience template around the mutable ``map`` proc to reduce typing. + ## + ## The template injects the ``it`` variable which you can use directly in an + ## expression. The expression has to return the same type as the sequence you + ## are mutating. Example: + ## + ## .. code-block:: nimrod + ## var nums = @[1, 2, 3, 4] + ## nums.mapIt(it * 3) + ## assert nums[0] + nums[3] == 15 + for i in 0 .. Date: Sun, 2 Feb 2014 17:59:17 +0100 Subject: [PATCH 46/72] Mentions {.exportc.} limits. Refs #826. --- doc/manual.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual.txt b/doc/manual.txt index 2ccd4c1e9b..b85c49e037 100644 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -5172,9 +5172,9 @@ the same feature under the same name. Exportc pragma -------------- The `exportc`:idx: pragma provides a means to export a type, a variable, or a -procedure to C. The optional argument is a string containing the C identifier. -If the argument is missing, the C name is the Nimrod -identifier *exactly as spelled*: +procedure to C. Enums and constants can't be exported. The optional argument +is a string containing the C identifier. If the argument is missing, the C +name is the Nimrod identifier *exactly as spelled*: .. code-block:: Nimrod proc callme(formatstr: cstring) {.exportc: "callMe", varargs.} From 52d4e07146f29f9a0af29e99a94ef4f5a1fd1cbf Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Sun, 2 Feb 2014 23:28:17 +0100 Subject: [PATCH 47/72] Adds utimes() importc to posix module. --- lib/posix/posix.nim | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index c3d5b5939e..41260b36fc 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -2566,3 +2566,15 @@ proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {. proc realpath*(name, resolved: cstring): cstring {. importc: "realpath", header: "".} + +proc utimes*(path: cstring, times: ptr array [2, Ttimeval]): int {. + importc: "utimes", header: "".} + ## Sets file access and modification times. + ## + ## Pass the filename and an array of times to set the access and modification + ## times respectively. If you pass nil as the array both attributes will be + ## set to the current time. + ## + ## Returns zero on success. + ## + ## For more information read http://www.unix.com/man-page/posix/3/utimes/. From db031e2b4741f7238433bc8b20beb12da5431b80 Mon Sep 17 00:00:00 2001 From: ReneSac Date: Mon, 3 Feb 2014 01:00:01 -0200 Subject: [PATCH 48/72] Expandeded part about different proc call syntaxes. --- doc/manual.txt | 74 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/doc/manual.txt b/doc/manual.txt index b85c49e037..da229d169b 100644 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -2457,14 +2457,80 @@ notation. (Thus an operator can have more than two parameters): assert `*+`(3, 4, 6) == `*`(a, `+`(b, c)) +Method call syntax +------------------ + +For object oriented programming, the syntax ``obj.method(args)`` can be used +instead of ``method(obj, args)``. The parentheses can be omitted if there are no +remaining arguments: ``obj.len`` (instead of ``len(obj)``). + +This `method call syntax`:idx: is not restricted to objects, it can be used +to supply any type of first argument for procedures: + +.. code-block:: nimrod + + echo("abc".len) # is the same as echo(len("abc")) + echo("abc".toUpper()) + echo({'a', 'b', 'c'}.card) + stdout.writeln("Hallo") # the same as writeln(stdout, "Hallo") + +Another way to look at the method call syntax is that it provides the missing +postfix notation. + + +Properties +---------- +Nimrod has no need for *get-properties*: Ordinary get-procedures that are called +with the *method call syntax* achieve the same. But setting a value is +different; for this a special setter syntax is needed: + +.. code-block:: nimrod + + type + TSocket* = object of TObject + FHost: int # cannot be accessed from the outside of the module + # the `F` prefix is a convention to avoid clashes since + # the accessors are named `host` + + proc `host=`*(s: var TSocket, value: int) {.inline.} = + ## setter of hostAddr + s.FHost = value + + proc host*(s: TSocket): int {.inline.} = + ## getter of hostAddr + return s.FHost + + var + s: TSocket + s.host = 34 # same as `host=`(s, 34) + + Command invocation syntax ------------------------- -Routines can be invoked without the ``()`` if the call is syntactially +Routines can be invoked without the ``()`` if the call is syntatically a statement. This `command invocation syntax`:idx: also works for expressions, but then only a single argument may follow. This restriction means ``echo f 1, f 2`` is parsed as ``echo(f(1), f(2))`` and not as -``echo(f(1, f(2)))``. +``echo(f(1, f(2)))``. The method call syntax may be used to provide one +more argument in this case: + +.. code-block:: nimrod + proc optarg(x:int, y:int = 0):int = x + y + proc singlearg(x:int):int = 20*x + + echo optarg 1, " ", singlearg 2 # prints "1 40" + + let fail = optarg 1, optarg 8 # Wrong. Too many arguments for a command call + let x = optarg(1, optarg 8) # traditional procedure call with 2 arguments + let y = 1.optarg optarg 8 # same thing as above, w/o the parenthesis + assert x == y + +The command invocation syntax also can't have complex expressions as arguments. +For example: (`anonymous procs`_), ``if``, ``case`` or ``try``. The (`do +notation`_) is limited, but usable for a single proc (see the example in the +corresponding section). Function calls with no arguments still needs () to +distinguish between a call and the function itself as a first class value. Closures @@ -2479,6 +2545,7 @@ the closure and its enclosing scope (i.e. any modifications made to them are visible in both places). The closure environment may be allocated on the heap or on the stack if the compiler determines that this would be safe. + Anonymous Procs --------------- @@ -2505,6 +2572,9 @@ calls can use the ``do`` keyword: .. code-block:: nimrod sort(cities) do (x,y: string) -> int: cmp(x.len, y.len) + # Less parenthesis using the method plus command syntax: + cities = cities.map do (x:string) -> string: + "City of " & x ``do`` is written after the parentheses enclosing the regular proc params. The proc expression represented by the do block is appended to them. From 300c0376b5a85233dea9b1c9414a60ba585751d1 Mon Sep 17 00:00:00 2001 From: ReneSac Date: Mon, 3 Feb 2014 01:06:19 -0200 Subject: [PATCH 49/72] Fix optarg() and added two more tests. One for 'do notation' in a single function in an expression, another the trick of using the method call syntax to pass two parameters. --- tests/parser/tcommand_as_expr.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/parser/tcommand_as_expr.nim b/tests/parser/tcommand_as_expr.nim index a9747b0acd..3c02f8cb3d 100644 --- a/tests/parser/tcommand_as_expr.nim +++ b/tests/parser/tcommand_as_expr.nim @@ -1,9 +1,11 @@ discard """ output: '''140 -5-120''' +5-120-120 +359''' """ +import math -proc optarg(x:int):int = x +proc optarg(x:int, y:int = 0):int = x + 3 * y proc singlearg(x:int):int = 20*x echo optarg 1, singlearg 2 @@ -13,5 +15,9 @@ proc foo(x, y: int): int = x-y let x = optarg foo 7.foo let y = singlearg foo(1, foo 8) +let z = singlearg 1.foo foo 8 + +echo x, y, z -echo x, y +let a = [2,4,8].map do (d:int) -> int: d + 1 +echo a[0], a[1], a[2] \ No newline at end of file From 5bd3ec0bfb77a1a7342a183876e45c333b804345 Mon Sep 17 00:00:00 2001 From: ReneSac Date: Mon, 3 Feb 2014 01:12:27 -0200 Subject: [PATCH 50/72] Remove spurious import added in the last commit. --- tests/parser/tcommand_as_expr.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parser/tcommand_as_expr.nim b/tests/parser/tcommand_as_expr.nim index 3c02f8cb3d..22c49ab3f2 100644 --- a/tests/parser/tcommand_as_expr.nim +++ b/tests/parser/tcommand_as_expr.nim @@ -3,7 +3,7 @@ discard """ 5-120-120 359''' """ -import math +#import math proc optarg(x:int, y:int = 0):int = x + 3 * y proc singlearg(x:int):int = 20*x From 99352c1e4c4889b3b1050ee496818fe25ca7d171 Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 3 Feb 2014 17:35:03 +0100 Subject: [PATCH 51/72] macro tests almost green --- compiler/semstmts.nim | 3 +++ compiler/types.nim | 3 ++- compiler/vm.nim | 6 ++++-- compiler/vmgen.nim | 31 +++++++++++++++++-------------- lib/core/macros.nim | 28 ++++++++++++++-------------- tests/macros/tdumpast.nim | 2 +- tests/macros/tdumpast2.nim | 2 +- tests/macros/tmacrogenerics.nim | 6 ++---- tests/macros/tmacrotypes.nim | 4 ++++ tests/macros/tmemit.nim | 2 +- todo.txt | 9 ++++++++- 11 files changed, 57 insertions(+), 39 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 25353fdd7b..d3cc1a12e6 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -972,6 +972,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, else: s = n[namePos].sym typeIsDetermined = s.typ == nil + s.ast = n + s.scope = c.currentScope + # if typeIsDetermined: assert phase == stepCompileBody # else: assert phase == stepDetermineType # before compiling the proc body, set as current the scope diff --git a/compiler/types.nim b/compiler/types.nim index cd703474e6..812cd4e93a 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -47,7 +47,8 @@ proc equalParams*(a, b: PNode): TParamsEquality # returns whether the parameter lists of the procs a, b are exactly the same proc isOrdinalType*(t: PType): bool proc enumHasHoles*(t: PType): bool -const +# XXX it is WRONG to include tyTypeDesc here as that might not have any child! +const abstractPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyDistinct, tyOrdinal, tyConst, tyMutable, tyTypeDesc} abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, diff --git a/compiler/vm.nim b/compiler/vm.nim index aec76f307e..61881a8977 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1047,7 +1047,9 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode = # XXX only supports 'name' for now; we can use regC to encode the # type trait operation decodeB(nkStrLit) - let typ = regs[rb].sym.typ.skipTypes({tyTypeDesc}) + var typ = regs[rb].typ + internalAssert typ != nil + while typ.kind == tyTypeDesc and typ.len > 0: typ = typ.sons[0] regs[ra].strVal = typ.typeToString(preferExported) of opcGlobalOnce: let rb = instr.regBx @@ -1178,7 +1180,7 @@ proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode = # doesn't end up in the parameter: #InternalAssert tos.slots.len >= L # return value: - tos.slots[0] = newNodeIT(nkNilLit, n.info, sym.typ.sons[0]) + tos.slots[0] = newNodeIT(nkEmpty, n.info, sym.typ.sons[0]) # setup parameters: for i in 1 .. < min(tos.slots.len, L): tos.slots[i] = setupMacroParam(n.sons[i]) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index e0ff5b2350..be32f990f0 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -109,7 +109,7 @@ proc patch(c: PCtx, p: TPosition) = uint32(diff+wordExcess) shl 16'u32).TInstr proc getSlotKind(t: PType): TSlotKind = - case t.skipTypes(abstractRange).kind + case t.skipTypes(abstractRange-{tyTypeDesc}).kind of tyBool, tyChar, tyEnum, tyOrdinal, tyInt..tyInt64, tyUInt..tyUInt64: slotTempInt of tyString, tyCString: @@ -409,7 +409,7 @@ proc genTry(c: PCtx; n: PNode; dest: var TDest) = let endExcept = c.xjmp(it, opcExcept, 0) for j in countup(0, blen - 2): assert(it.sons[j].kind == nkType) - let typ = it.sons[j].typ.skipTypes(abstractPtrs) + let typ = it.sons[j].typ.skipTypes(abstractPtrs-{tyTypeDesc}) c.gABx(it, opcExcept, 0, c.genType(typ)) if blen == 1: # general except section: @@ -479,7 +479,7 @@ proc genNew(c: PCtx; n: PNode) = # we use the ref's base type here as the VM conflates 'ref object' # and 'object' since internally we already have a pointer. c.gABx(n, opcNew, dest, - c.genType(n.sons[1].typ.skipTypes(abstractVar).sons[0])) + c.genType(n.sons[1].typ.skipTypes(abstractVar-{tyTypeDesc}).sons[0])) c.genAsgnPatch(n.sons[1], dest) c.freeTemp(dest) @@ -487,7 +487,8 @@ proc genNewSeq(c: PCtx; n: PNode) = let dest = if needsAsgnPatch(n.sons[1]): c.getTemp(n.sons[1].typ) else: c.genx(n.sons[1]) let tmp = c.genx(n.sons[2]) - c.gABx(n, opcNewSeq, dest, c.genType(n.sons[1].typ.skipTypes(abstractVar))) + c.gABx(n, opcNewSeq, dest, c.genType(n.sons[1].typ.skipTypes( + abstractVar-{tyTypeDesc}))) c.gABx(n, opcNewSeq, tmp, 0) c.freeTemp(tmp) c.genAsgnPatch(n.sons[1], dest) @@ -515,7 +516,7 @@ proc genBinaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = c.freeTemp(tmp2) proc genSetType(c: PCtx; n: PNode; dest: TRegister) = - let t = skipTypes(n.typ, abstractInst) + let t = skipTypes(n.typ, abstractInst-{tyTypeDesc}) if t.kind == tySet: c.gABx(n, opcSetType, dest, c.genType(t)) @@ -746,7 +747,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) = var tmp = c.genx(n.sons[1]) var idx = c.getTemp(getSysType(tyInt)) var typ = n.sons[2].typ - if m == mOf: typ = typ.skipTypes(abstractPtrs) + if m == mOf: typ = typ.skipTypes(abstractPtrs-{tyTypeDesc}) c.gABx(n, opcLdImmInt, idx, c.genType(typ)) c.gABC(n, if m == mOf: opcOf else: opcIs, dest, tmp, idx) c.freeTemp(tmp) @@ -756,7 +757,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) = of mHigh: if dest < 0: dest = c.getTemp(n.typ) let tmp = c.genx(n.sons[1]) - if n.sons[1].typ.skipTypes(abstractVar).kind == tyString: + if n.sons[1].typ.skipTypes(abstractVar-{tyTypeDesc}).kind == tyString: c.gABI(n, opcLenStr, dest, tmp, 1) else: c.gABI(n, opcLenSeq, dest, tmp, 1) @@ -891,7 +892,7 @@ const tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64} proc requiresCopy(n: PNode): bool = - if n.typ.skipTypes(abstractInst).kind in atomicTypes: + if n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind in atomicTypes: result = false elif n.kind in ({nkCurly, nkBracket, nkPar, nkObjConstr}+nkCallKinds): result = false @@ -899,7 +900,7 @@ proc requiresCopy(n: PNode): bool = result = true proc unneededIndirection(n: PNode): bool = - n.typ.skipTypes(abstractInst).kind == tyRef + n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind == tyRef proc skipDeref(n: PNode): PNode = if n.kind in {nkDerefExpr, nkHiddenDeref} and unneededIndirection(n.sons[0]): @@ -920,7 +921,7 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; c.freeTemp(tmp) proc whichAsgnOpc(n: PNode): TOpcode = - case n.typ.skipTypes(abstractRange).kind + case n.typ.skipTypes(abstractRange-{tyTypeDesc}).kind of tyBool, tyChar, tyEnum, tyOrdinal, tyInt..tyInt64, tyUInt..tyUInt64: opcAsgnInt of tyString, tyCString: @@ -932,7 +933,7 @@ proc whichAsgnOpc(n: PNode): TOpcode = else: opcAsgnComplex -proc isRef(t: PType): bool = t.skipTypes(abstractRange).kind == tyRef +proc isRef(t: PType): bool = t.skipTypes(abstractRange-{tyTypeDesc}).kind == tyRef proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode = if isRef(n.typ): succ(opc) else: opc @@ -951,7 +952,8 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) = let dest = c.genx(le.sons[0]) let idx = c.genx(le.sons[1]) let tmp = c.genx(ri) - if le.sons[0].typ.skipTypes(abstractVarRange).kind in {tyString, tyCString}: + if le.sons[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind in { + tyString, tyCString}: c.gABC(le, opcWrStrIdx, dest, idx, tmp) else: c.gABC(le, whichAsgnOpc(le, opcWrArr), dest, idx, tmp) @@ -1062,7 +1064,8 @@ proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = genAccess(c, n.sons[0], dest, opcLdObj, flags) proc genArrAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = - if n.sons[0].typ.skipTypes(abstractVarRange).kind in {tyString, tyCString}: + if n.sons[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind in { + tyString, tyCString}: genAccess(c, n, dest, opcLdStrIdx, {}) else: genAccess(c, n, dest, opcLdArr, flags) @@ -1207,7 +1210,7 @@ proc genSetConstr(c: PCtx, n: PNode, dest: var TDest) = proc genObjConstr(c: PCtx, n: PNode, dest: var TDest) = if dest < 0: dest = c.getTemp(n.typ) - let t = n.typ.skipTypes(abstractRange) + let t = n.typ.skipTypes(abstractRange-{tyTypeDesc}) if t.kind == tyRef: c.gABx(n, opcNew, dest, c.genType(t.sons[0])) else: diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 3b36e31e0b..585ccf8692 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -294,19 +294,6 @@ proc quote*(bl: stmt, op = "``"): PNimrodNode {.magic: "QuoteAst".} ## if not `ex`: ## echo `info` & ": Check failed: " & `expString` -when not defined(booting): - template emit*(e: static[string]): stmt = - ## accepts a single string argument and treats it as nimrod code - ## that should be inserted verbatim in the program - ## Example: - ## - ## .. code-block:: nimrod - ## emit("echo " & '"' & "hello world".toUpper & '"') - ## - macro payload: stmt {.gensym.} = - result = e.parseStmt - payload() - proc expectKind*(n: PNimrodNode, k: TNimrodNodeKind) {.compileTime.} = ## checks that `n` is of kind `k`. If this is not the case, ## compilation aborts with an error message. This is useful for writing @@ -421,7 +408,8 @@ proc lispRepr*(n: PNimrodNode): string {.compileTime.} = of nnkFloatLit..nnkFloat64Lit: add(result, $n.floatVal) of nnkStrLit..nnkTripleStrLit: add(result, $n.strVal) of nnkIdent: add(result, "!\"" & $n.ident & '"') - of nnkSym, nnkNone: assert false + of nnkSym: add(result, $n.symbol) + of nnkNone: assert false else: add(result, lispRepr(n[0])) for j in 1..n.len-1: @@ -745,3 +733,15 @@ proc addIdentIfAbsent*(dest: PNimrodNode, ident: string) {.compiletime.} = else: discard dest.add(ident(ident)) +when not defined(booting): + template emit*(e: static[string]): stmt = + ## accepts a single string argument and treats it as nimrod code + ## that should be inserted verbatim in the program + ## Example: + ## + ## .. code-block:: nimrod + ## emit("echo " & '"' & "hello world".toUpper & '"') + ## + macro payload: stmt {.gensym.} = + result = parseStmt(e) + payload() diff --git a/tests/macros/tdumpast.nim b/tests/macros/tdumpast.nim index 55a964327c..160e4e1940 100644 --- a/tests/macros/tdumpast.nim +++ b/tests/macros/tdumpast.nim @@ -2,7 +2,7 @@ import macros -template plus(a, b: expr): expr = +template plus(a, b: expr): expr {.dirty} = a + b macro call(e: expr): expr = diff --git a/tests/macros/tdumpast2.nim b/tests/macros/tdumpast2.nim index c6eab39a9a..2a7024a017 100644 --- a/tests/macros/tdumpast2.nim +++ b/tests/macros/tdumpast2.nim @@ -7,7 +7,7 @@ proc dumpit(n: PNimrodNode): string {.compileTime.} = result = $n.kind add(result, "(") case n.kind - of nnkEmpty: nil # same as nil node in this representation + of nnkEmpty: discard # same as nil node in this representation of nnkNilLit: add(result, "nil") of nnkCharLit..nnkInt64Lit: add(result, $n.intVal) of nnkFloatLit..nnkFloat64Lit: add(result, $n.floatVal) diff --git a/tests/macros/tmacrogenerics.nim b/tests/macros/tmacrogenerics.nim index 5ae59e0da4..b886f4fa54 100644 --- a/tests/macros/tmacrogenerics.nim +++ b/tests/macros/tmacrogenerics.nim @@ -1,10 +1,8 @@ discard """ file: "tmacrogenerics.nim" msg: ''' -instantiation 1 with int and float -instantiation 2 with float and string -instantiation 3 with string and string -counter: 3 +instantiation 1 with typedesc and typedesc +counter: 1 ''' output: "int\nfloat\nint\nstring" """ diff --git a/tests/macros/tmacrotypes.nim b/tests/macros/tmacrotypes.nim index 7697dba27e..f19aa2ddb8 100644 --- a/tests/macros/tmacrotypes.nim +++ b/tests/macros/tmacrotypes.nim @@ -1,3 +1,7 @@ +discard """ + disabled: true +""" + import macros, typetraits macro checkType(ex, expected: expr): stmt {.immediate.} = diff --git a/tests/macros/tmemit.nim b/tests/macros/tmemit.nim index e4bb2daedc..6fb2f3b659 100644 --- a/tests/macros/tmemit.nim +++ b/tests/macros/tmemit.nim @@ -1,5 +1,5 @@ discard """ - out: '''HELLO WORLD''' + output: '''HELLO WORLD''' """ import macros, strutils diff --git a/todo.txt b/todo.txt index bad9373a14..8bf3ff239a 100644 --- a/todo.txt +++ b/todo.txt @@ -1,11 +1,18 @@ version 0.9.4 ============= +- fix macros\tstringinterp.nim +- test and fix showoff; add debug example to showoff +- test and fix stdlib +- test and fix misc +- fix GC issues +- test C source code generation +- test and fix closures +- test and fix exception handling Bugs ==== -- fix eval in macros.nim - new VM: - implement overflow checking - bug: 'type T = ref T' not recognized as illegal recursion From c69d167fa9e6e3bf5aadd8f5910eaf79b3667e70 Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 3 Feb 2014 19:00:05 +0100 Subject: [PATCH 52/72] updated todo.txt --- todo.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/todo.txt b/todo.txt index 8bf3ff239a..fd424ad5e4 100644 --- a/todo.txt +++ b/todo.txt @@ -9,6 +9,8 @@ version 0.9.4 - test C source code generation - test and fix closures - test and fix exception handling +- implement 'union' and 'bits' pragmas + Bugs ==== From 51076a0be6900602e5fc092e52021370937a85f7 Mon Sep 17 00:00:00 2001 From: Charlie Barto Date: Mon, 3 Feb 2014 23:54:01 -0500 Subject: [PATCH 53/72] added good error for stuff like string[int] or int[int] --- compiler/semtypes.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 408b1b62e0..a434effed2 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -885,6 +885,11 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType = for i in countup(1, sonsLen(n)-1): var elem = semGenericParamInInvokation(c, n.sons[i]) addToResult(elem) + elif s.typ.kind != tyGenericBody: + #we likely got code of the form TypeA[TypeB] where TypeA is + #not generic. + localError(n.info, errNoGenericParamsAllowedForX, s.name.s) + return newOrPrevType(tyError, prev, c) else: internalAssert s.typ.kind == tyGenericBody From 7cfec910576c3a9f3401aae99f9209716d02f689 Mon Sep 17 00:00:00 2001 From: Charlie Barto Date: Tue, 4 Feb 2014 00:07:17 -0500 Subject: [PATCH 54/72] deleted assert that would never fire because it turned into an error message --- compiler/semtypes.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index a434effed2..789ff55f09 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -891,7 +891,6 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType = localError(n.info, errNoGenericParamsAllowedForX, s.name.s) return newOrPrevType(tyError, prev, c) else: - internalAssert s.typ.kind == tyGenericBody var m = newCandidate(c, s, n) matches(c, n, copyTree(n), m) From c097acedd30602fba903ed4ee132b5e5bae91017 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 4 Feb 2014 17:29:34 +0100 Subject: [PATCH 55/72] bugfix: immediate templates are preferred consistently (danger: breaks code) --- compiler/astalgo.nim | 33 +++++++++++++++++++++------- compiler/lookups.nim | 14 +++++++++++- tests/template/tprefer_immediate.nim | 17 ++++++++++++++ todo.txt | 1 - 4 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 tests/template/tprefer_immediate.nim diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 2505bc6879..64c1b717c3 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -561,14 +561,31 @@ proc strTableContains(t: TStrTable, n: PSym): bool = proc strTableRawInsert(data: var TSymSeq, n: PSym) = var h: THash = n.name.h and high(data) - while data[h] != nil: - if data[h] == n: - # allowed for 'export' feature: - #InternalError(n.info, "StrTableRawInsert: " & n.name.s) - return - h = nextTry(h, high(data)) - assert(data[h] == nil) - data[h] = n + if sfImmediate notin n.flags: + # fast path: + while data[h] != nil: + if data[h] == n: + # allowed for 'export' feature: + #InternalError(n.info, "StrTableRawInsert: " & n.name.s) + return + h = nextTry(h, high(data)) + assert(data[h] == nil) + data[h] = n + else: + # slow path; we have to ensure immediate symbols are preferred for + # symbol lookups. + # consider the chain: foo (immediate), foo, bar, bar (immediate) + # then bar (immediate) gets replaced with foo (immediate) and the non + # immediate foo is picked! Thus we need to replace it with the first + # slot that has in fact the same identifier stored in it! + var favPos = -1 + while data[h] != nil: + if data[h] == n: return + if favPos < 0 and data[h].name.id == n.name.id: favPos = h + h = nextTry(h, high(data)) + assert(data[h] == nil) + data[h] = n + if favPos >= 0: swap data[h], data[favPos] proc symTabReplaceRaw(data: var TSymSeq, prevSym: PSym, newSym: PSym) = assert prevSym.name.h == newSym.name.h diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 6dfd259686..c31eb3121d 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -339,4 +339,16 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = n.sons[0].sym.name, o.inSymChoice) if result != nil and result.kind == skStub: loadStub(result) - + +when false: + proc qualifiedLookUpPreferImmediate*(c: PContext, n: PNode, + flags = {checkUndeclared}): PSym = + var o: TOverloadIter + result = initOverloadIter(o, c, n) + var a = result + while a != nil: + if sfImmediate in a.flags: return a + a = nextOverloadIter(o, c, n) + if result == nil and checkUndeclared in flags: + localError(n.info, errUndeclaredIdentifier, n.considerAcc.s) + result = errorSym(c, n) diff --git a/tests/template/tprefer_immediate.nim b/tests/template/tprefer_immediate.nim new file mode 100644 index 0000000000..578f447b02 --- /dev/null +++ b/tests/template/tprefer_immediate.nim @@ -0,0 +1,17 @@ +discard """ + output: '''immediate''' +""" + +# Test that immediate templates are preferred over non-immediate templates + +template foo(a, b: expr) = echo "foo expr" + +template foo(a, b: int) = echo "foo int" +template foo(a, b: float) = echo "foo float" +template foo(a, b: string) = echo "foo string" +template foo(a, b: expr) {.immediate.} = echo "immediate" +template foo(a, b: bool) = echo "foo bool" +template foo(a, b: char) = echo "foo char" + +foo(undeclaredIdentifier, undeclaredIdentifier2) + diff --git a/todo.txt b/todo.txt index fd424ad5e4..6f8726199e 100644 --- a/todo.txt +++ b/todo.txt @@ -22,7 +22,6 @@ Bugs - compilation of niminst takes way too long. looks like a regression - docgen: sometimes effects are listed twice - 'result' is not properly cleaned for NRVO --> use uninit checking instead -- sneaking with qualifiedLookup() is really broken! - blocks can "export" an identifier but the CCG generates {} for them ... - osproc execProcesses can deadlock if all processes fail (as experienced in c++ mode) From d912d1837963b11e1df622c7b41e39827f95599c Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 5 Feb 2014 23:43:13 +0100 Subject: [PATCH 56/72] tyTypeDesc and tyRange always have 1 child; this might be tyNone but it is required for skipTypes --- compiler/ast.nim | 6 +++--- compiler/sem.nim | 4 +++- compiler/semdata.nim | 1 + compiler/semstmts.nim | 4 ++-- compiler/semtypes.nim | 35 +++++++++++++++++++++++------------ compiler/semtypinst.nim | 2 +- compiler/sigmatch.nim | 19 ++++++++++--------- compiler/types.nim | 4 ++-- compiler/vmgen.nim | 23 ++++++++++++----------- tests/macros/tdebugstmt.nim | 29 +++++++++++++++++++++++++++++ todo.txt | 2 +- 11 files changed, 87 insertions(+), 42 deletions(-) create mode 100644 tests/macros/tdebugstmt.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 7138b5f52a..cd002eef1f 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1129,7 +1129,7 @@ proc newType(kind: TTypeKind, owner: PSym): PType = result.align = 2 # default alignment result.id = getID() when debugIds: - RegisterId(result) + registerId(result) #if result.id < 2000 then # MessageOut(typeKindToStr[kind] & ' has id: ' & toString(result.id)) @@ -1166,7 +1166,7 @@ proc copyType(t: PType, owner: PSym, keepId: bool): PType = if keepId: result.id = t.id else: - when debugIds: RegisterId(result) + when debugIds: registerId(result) result.sym = t.sym # backend-info should not be copied proc copySym(s: PSym, keepId: bool = false): PSym = @@ -1177,7 +1177,7 @@ proc copySym(s: PSym, keepId: bool = false): PSym = result.id = s.id else: result.id = getID() - when debugIds: RegisterId(result) + when debugIds: registerId(result) result.flags = s.flags result.magic = s.magic if s.kind == skModule: diff --git a/compiler/sem.nim b/compiler/sem.nim index e89e32f4e0..00ac79716a 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -83,7 +83,9 @@ proc commonType*(x, y: PType): PType = elif a.kind == tyTypeDesc: # turn any concrete typedesc into the abstract typedesc type if a.sons == nil: result = a - else: result = newType(tyTypeDesc, a.owner) + else: + result = newType(tyTypeDesc, a.owner) + rawAddSon(result, newType(tyNone, a.owner)) elif b.kind in {tyArray, tyArrayConstr, tySet, tySequence} and a.kind == b.kind: # check for seq[empty] vs. seq[int] diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 8b97f36859..c9d95e1bf4 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -211,6 +211,7 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType = proc makeTypeSymNode*(c: PContext, typ: PType, info: TLineInfo): PNode = let typedesc = makeTypeDesc(c, typ) + rawAddSon(typedesc, newTypeS(tyNone, c)) let sym = newSym(skType, idAnon, getCurrOwner(), info).linkTo(typedesc) return newSymNode(sym, info) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index d3cc1a12e6..0871b7fb7a 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -740,7 +740,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) = # we fill it out later. For magic generics like 'seq', it won't be filled # so we use tyEmpty instead of nil to not crash for strange conversions # like: mydata.seq - rawAddSon(s.typ, newTypeS(tyEmpty, c)) + rawAddSon(s.typ, newTypeS(tyNone, c)) s.ast = a inc c.inGenericContext var body = semTypeNode(c, a.sons[2], nil) @@ -748,7 +748,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) = if body != nil: body.sym = s body.size = -1 # could not be computed properly - s.typ.sons[sonsLen(s.typ) - 1] = body + s.typ.sons[sonsLen(s.typ) - 1] = body popOwner() closeScope(c) elif a.sons[2].kind != nkEmpty: diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 408b1b62e0..6f2d9ece9d 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -681,12 +681,12 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, of tySequence, tySet, tyArray, tyOpenArray, tyVar, tyPtr, tyRef, tyProc: # XXX: this is a bit strange, but proc(s: seq) - # produces tySequence(tyGenericParam, null). + # produces tySequence(tyGenericParam, tyNone). # This also seems to be true when creating aliases # like: type myseq = distinct seq. # Maybe there is another better place to associate # the seq type class with the seq identifier. - if paramType.kind == tySequence and paramType.lastSon == nil: + if paramType.kind == tySequence and paramType.lastSon.kind == tyNone: let typ = c.newTypeWithSons(tyBuiltInTypeClass, @[newTypeS(paramType.kind, c)]) result = addImplicitGeneric(typ) @@ -1133,15 +1133,26 @@ proc processMagicType(c: PContext, m: PSym) = of mNil: setMagicType(m, tyNil, ptrSize) of mExpr: setMagicType(m, tyExpr, 0) of mStmt: setMagicType(m, tyStmt, 0) - of mTypeDesc: setMagicType(m, tyTypeDesc, 0) + of mTypeDesc: + setMagicType(m, tyTypeDesc, 0) + rawAddSon(m.typ, newTypeS(tyNone, c)) of mVoidType: setMagicType(m, tyEmpty, 0) - of mArray: setMagicType(m, tyArray, 0) - of mOpenArray: setMagicType(m, tyOpenArray, 0) - of mVarargs: setMagicType(m, tyVarargs, 0) - of mRange: setMagicType(m, tyRange, 0) - of mSet: setMagicType(m, tySet, 0) - of mSeq: setMagicType(m, tySequence, 0) - of mOrdinal: setMagicType(m, tyOrdinal, 0) + of mArray: + setMagicType(m, tyArray, 0) + of mOpenArray: + setMagicType(m, tyOpenArray, 0) + of mVarargs: + setMagicType(m, tyVarargs, 0) + of mRange: + setMagicType(m, tyRange, 0) + rawAddSon(m.typ, newTypeS(tyNone, c)) + of mSet: + setMagicType(m, tySet, 0) + of mSeq: + setMagicType(m, tySequence, 0) + of mOrdinal: + setMagicType(m, tyOrdinal, 0) + rawAddSon(m.typ, newTypeS(tyNone, c)) of mPNimrodNode: discard else: localError(m.info, errTypeExpected) @@ -1165,8 +1176,8 @@ proc semGenericParamList(c: PContext, n: PNode, father: PType = nil): PNode = typ = semTypeNode(c, constraint, nil) if typ.kind != tyStatic or typ.len == 0: if typ.kind == tyTypeDesc: - if typ.len == 0: - typ = newTypeS(tyTypeDesc, c) + if typ.sons[0].kind == tyNone: + typ = newTypeWithSons(c, tyTypeDesc, @[newTypeS(tyNone, c)]) else: typ = semGenericConstraints(c, typ) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 1158335a85..a07d912414 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -358,7 +358,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = if lookup != nil: result = lookup if tfUnresolved in t.flags: result = result.base - elif t.sonsLen > 0: + elif t.sons[0].kind != tyNone: result = makeTypeDesc(cl.c, replaceTypeVarsT(cl, t.sons[0])) of tyUserTypeClass: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index d269e9e693..f9200ea0c3 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -562,10 +562,11 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = #if result < isGeneric: result = isNone if result notin {isNone, isGeneric}: result = typeRangeRel(f, a) - elif skipTypes(f, {tyRange}).kind == a.kind: - result = isIntConv - elif isConvertibleToRange(skipTypes(f, {tyRange}), a): - result = isConvertible # a convertible to f + else: + if skipTypes(f, {tyRange}).kind == a.kind: + result = isIntConv + elif isConvertibleToRange(skipTypes(f, {tyRange}), a): + result = isConvertible # a convertible to f of tyInt: result = handleRange(f, a, tyInt8, tyInt32) of tyInt8: result = handleRange(f, a, tyInt8, tyInt8) of tyInt16: result = handleRange(f, a, tyInt8, tyInt16) @@ -636,7 +637,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyOrdinal: if isOrdinalType(a): var x = if a.kind == tyOrdinal: a.sons[0] else: a - if f.sonsLen == 0: + if f.sons[0].kind == tyNone: result = isGeneric else: result = typeRel(c, f.sons[0], x) @@ -736,7 +737,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyGenericInst: let roota = a.skipGenericAlias let rootf = f.skipGenericAlias - if a.kind == tyGenericInst and roota.base == rootf.base : + if a.kind == tyGenericInst and roota.base == rootf.base: for i in 1 .. rootf.sonsLen-2: let ff = rootf.sons[i] let aa = roota.sons[i] @@ -845,7 +846,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = # by a tyTypeDesc params. Unfortunately, this requires more substantial # changes in semtypinst and elsewhere. if a.kind == tyTypeDesc: - if f.sons == nil or f.sons.len == 0: + if f.sonsLen == 0: result = isGeneric else: internalAssert a.sons != nil and a.sons.len > 0 @@ -854,7 +855,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = else: result = isNone else: - if f.sonsLen > 0: + if f.sonsLen > 0 and f.sons[0].kind != tyNone: result = typeRel(c, f.lastSon, a) else: result = isGeneric @@ -883,7 +884,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = var prev = PType(idTableGet(c.bindings, f)) if prev == nil: if a.kind == tyTypeDesc: - if f.sonsLen == 0: + if f.sons[0].kind == tyNone: result = isGeneric else: result = typeRel(c, f.sons[0], a.sons[0]) diff --git a/compiler/types.nim b/compiler/types.nim index 812cd4e93a..db75cd3c0f 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -47,7 +47,7 @@ proc equalParams*(a, b: PNode): TParamsEquality # returns whether the parameter lists of the procs a, b are exactly the same proc isOrdinalType*(t: PType): bool proc enumHasHoles*(t: PType): bool -# XXX it is WRONG to include tyTypeDesc here as that might not have any child! + const abstractPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyDistinct, tyOrdinal, tyConst, tyMutable, tyTypeDesc} @@ -1258,7 +1258,7 @@ proc containsGenericTypeIter(t: PType, closure: PObject): bool = return true if t.kind == tyTypeDesc: - if t.sonsLen == 0: return true + if t.sons[0].kind == tyNone: return true if containsGenericTypeIter(t.base, closure): return true return false diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index be32f990f0..313c43dc44 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -946,6 +946,14 @@ proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) = template isGlobal(s: PSym): bool = sfGlobal in s.flags and s.kind != skForVar +proc setSlot(c: PCtx; v: PSym) = + # XXX generate type initialization here? + if v.position == 0: + v.position = c.prc.maxSlots + c.prc.slots[v.position] = (inUse: true, + kind: if v.kind == skLet: slotFixedLet else: slotFixedVar) + inc c.prc.maxSlots + proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) = case le.kind of nkBracketExpr: @@ -973,8 +981,9 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) = gen(c, ri, tmp) c.gABx(le, whichAsgnOpc(le, opcWrGlobal), tmp, s.position) else: + if s.kind == skForVar and c.mode == emRepl: c.setSlot s internalAssert s.position > 0 or (s.position == 0 and - s.kind in {skParam,skResult,skForVar}) + s.kind in {skParam,skResult}) var dest: TRegister = s.position + ord(s.kind == skParam) gen(c, ri, dest) else: @@ -1035,8 +1044,9 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest) = else: c.gABx(n, opcLdGlobal, dest, s.position) else: + if s.kind == skForVar and c.mode == emRepl: c.setSlot s if s.position > 0 or (s.position == 0 and - s.kind in {skParam,skResult,skForVar}): + s.kind in {skParam,skResult}): if dest < 0: dest = s.position + ord(s.kind == skParam) else: @@ -1045,7 +1055,6 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest) = else: # see tests/t99bott for an example that triggers it: cannotEval(n) - #InternalError(n.info, s.name.s & " " & $s.position) proc genAccess(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; flags: TGenFlags) = @@ -1123,14 +1132,6 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode = result = newNodeIT(nkCurly, info, t) else: internalError("getNullValue: " & $t.kind) -proc setSlot(c: PCtx; v: PSym) = - # XXX generate type initialization here? - if v.position == 0: - v.position = c.prc.maxSlots - c.prc.slots[v.position] = (inUse: true, - kind: if v.kind == skLet: slotFixedLet else: slotFixedVar) - inc c.prc.maxSlots - proc genVarSection(c: PCtx; n: PNode) = for a in n: if a.kind == nkCommentStmt: continue diff --git a/tests/macros/tdebugstmt.nim b/tests/macros/tdebugstmt.nim new file mode 100644 index 0000000000..865dc436a9 --- /dev/null +++ b/tests/macros/tdebugstmt.nim @@ -0,0 +1,29 @@ +discard """ + output: '''a[0]: 42 +a[1]: 45 +x: some string''' +""" + +import macros + +macro debug(n: varargs[expr]): stmt = + # `n` is a Nimrod AST that contains the whole macro invocation + # this macro returns a list of statements: + result = newNimNode(nnkStmtList, n) + # iterate over any argument that is passed to this macro: + for i in 0..n.len-1: + # add a call to the statement list that writes the expression; + # `toStrLit` converts an AST to its string representation: + add(result, newCall("write", newIdentNode("stdout"), toStrLit(n[i]))) + # add a call to the statement list that writes ": " + add(result, newCall("write", newIdentNode("stdout"), newStrLitNode(": "))) + # add a call to the statement list that writes the expressions value: + add(result, newCall("writeln", newIdentNode("stdout"), n[i])) + +var + a: array [0..10, int] + x = "some string" +a[0] = 42 +a[1] = 45 + +debug(a[0], a[1], x) diff --git a/todo.txt b/todo.txt index 6f8726199e..fef073f90d 100644 --- a/todo.txt +++ b/todo.txt @@ -2,7 +2,7 @@ version 0.9.4 ============= - fix macros\tstringinterp.nim -- test and fix showoff; add debug example to showoff +- test and fix showoff - test and fix stdlib - test and fix misc - fix GC issues From 8215a32eddb10bb6dda056adff3d330baae73fc8 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 5 Feb 2014 23:46:55 +0100 Subject: [PATCH 57/72] case consistency for -d:useWinAnsi --- lib/pure/os.nim | 4 ++-- lib/pure/osproc.nim | 4 ++-- lib/windows/winlean.nim | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 245a8446bb..bb70f28b66 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1078,9 +1078,9 @@ when defined(windows): while true: var eend = strEnd(e) add(environment, $e) - e = cast[CString](cast[TAddress](eend)+1) + e = cast[cstring](cast[TAddress](eend)+1) if eend[1] == '\0': break - discard FreeEnvironmentStringsA(env) + discard freeEnvironmentStringsA(env) envComputed = true else: diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 9975bfcb3f..6df85bbc6a 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -455,7 +455,7 @@ when defined(Windows) and not defined(useNimRtl): ee, wwd, si, procInfo) else: success = winlean.createProcessA(nil, - cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, e, wd, SI, ProcInfo) + cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, e, wd, si, procInfo) let lastError = osLastError() if poParentStreams notin options: @@ -534,7 +534,7 @@ when defined(Windows) and not defined(useNimRtl): NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) else: var res = winlean.createProcessA(nil, command, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, SI, ProcInfo) + NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) if res == 0: osError(osLastError()) else: diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index ee5fe06470..5b641185eb 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -195,14 +195,14 @@ else: importc: "GetCurrentDirectoryA", dynlib: "kernel32", stdcall.} proc setCurrentDirectoryA*(lpPathName: cstring): int32 {. importc: "SetCurrentDirectoryA", dynlib: "kernel32", stdcall.} - proc createDirectoryA*(pathName: cstring, security: Pointer=nil): int32 {. + proc createDirectoryA*(pathName: cstring, security: pointer=nil): int32 {. importc: "CreateDirectoryA", dynlib: "kernel32", stdcall.} proc removeDirectoryA*(lpPathName: cstring): int32 {. importc: "RemoveDirectoryA", dynlib: "kernel32", stdcall.} proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA".} - proc getModuleFileNameA*(handle: THandle, buf: CString, size: int32): int32 {. + proc getModuleFileNameA*(handle: THandle, buf: cstring, size: int32): int32 {. importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.} when useWinUnicode: @@ -300,7 +300,7 @@ else: dwFileAttributes: int32): WINBOOL {. stdcall, dynlib: "kernel32", importc: "SetFileAttributesA".} - proc copyFileA*(lpExistingFileName, lpNewFileName: CString, + proc copyFileA*(lpExistingFileName, lpNewFileName: cstring, bFailIfExists: cint): cint {. importc: "CopyFileA", stdcall, dynlib: "kernel32".} From 550b5d9aeac9afda613406e3fe4bc243ea7f78d4 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 6 Feb 2014 00:27:01 +0100 Subject: [PATCH 58/72] stdlib compiles mostly without warnings again --- lib/pure/asyncio.nim | 2 +- lib/pure/htmlparser.nim | 4 ++-- lib/pure/irc.nim | 4 ++-- lib/pure/matchers.nim | 2 +- lib/pure/oids.nim | 2 +- lib/pure/parsecsv.nim | 2 +- lib/pure/parsesql.nim | 6 +++--- lib/pure/xmlparser.nim | 4 ++-- todo.txt | 4 +--- 9 files changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/pure/asyncio.nim b/lib/pure/asyncio.nim index f13cadaa2d..96afc6f4f9 100644 --- a/lib/pure/asyncio.nim +++ b/lib/pure/asyncio.nim @@ -689,5 +689,5 @@ when isMainModule: server.listen() d.register(server) - while d.poll(-1): nil + while d.poll(-1): discard diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 060f0e3866..c38eb70634 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -480,7 +480,7 @@ proc untilElementEnd(x: var TXmlParser, result: PXmlNode, if htmlTag(x.elemName) in {tagOption, tagOptgroup}: errors.add(expected(x, result)) break - else: nil + else: discard result.addNode(parse(x, errors)) of xmlElementEnd: if cmpIgnoreCase(x.elemName, result.tag) == 0: @@ -547,7 +547,7 @@ proc parse(x: var TXmlParser, errors: var seq[string]): PXmlNode = var u = entityToUtf8(x.rawData) if u.len != 0: result = newText(u) next(x) - of xmlEof: nil + of xmlEof: discard proc parseHtml*(s: PStream, filename: string, errors: var seq[string]): PXmlNode = diff --git a/lib/pure/irc.nim b/lib/pure/irc.nim index 750c98516b..c1b519b0b5 100644 --- a/lib/pure/irc.nim +++ b/lib/pure/irc.nim @@ -476,12 +476,12 @@ when isMainModule: var client = irc("amber.tenthbit.net", nick="TestBot1234", joinChans = @["#flood"]) client.connect() - while True: + while true: var event: TIRCEvent if client.poll(event): case event.typ of EvConnected: - nil + discard of EvDisconnected: break of EvMsg: diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index b57e0c45a5..2db7fa6606 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -54,7 +54,7 @@ proc parseInt*(s: string, value: var int, validRange: TSlice[int]) {. try: discard parseutils.parseInt(s, x, 0) except EOverflow: - nil + discard if x in validRange: value = x when isMainModule: diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index fbe0dda958..b3e74d2a1a 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -28,7 +28,7 @@ proc hexbyte*(hex: char): int = of '0'..'9': result = (ord(hex) - ord('0')) of 'a'..'f': result = (ord(hex) - ord('a') + 10) of 'A'..'F': result = (ord(hex) - ord('A') + 10) - else: nil + else: discard proc parseOid*(str: cstring): TOid = ## parses an OID. diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 5970f2090d..4b25babecf 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -149,7 +149,7 @@ proc readRow*(my: var TCsvParser, columns = 0): bool = of '\c': my.bufpos = handleCR(my, my.bufpos) of '\l': my.bufpos = handleLF(my, my.bufpos) else: break - of '\0': nil + of '\0': discard else: error(my, my.bufpos, my.sep & " expected") break diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 31951e9669..3f9686e1ed 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -79,7 +79,7 @@ proc handleHexChar(c: var TSqlLexer, xi: var int) = xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) inc(c.bufpos) else: - nil + discard proc handleOctChar(c: var TSqlLexer, xi: var int) = if c.buf[c.bufpos] in {'0'..'7'}: @@ -373,7 +373,7 @@ proc getOperator(c: var TSqlLexer, tok: var TToken) = of '+': if not trailingPlusMinus and buf[pos+1] notin operators and tok.literal.len > 0: break - of '*', '<', '>', '=': nil + of '*', '<', '>', '=': discard else: break add(tok.literal, buf[pos]) inc(pos) @@ -1120,7 +1120,7 @@ proc rs(n: PSqlNode, s: var string, indent: int, proc ra(n: PSqlNode, s: var string, indent: int) = if n == nil: return case n.kind - of nkNone: nil + of nkNone: discard of nkIdent: if allCharsInSet(n.strVal, {'\33'..'\127'}): s.add(n.strVal) diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 16bbe1455b..8b8bb3b030 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -96,7 +96,7 @@ proc parse(x: var TXmlParser, errors: var seq[string]): PXmlNode = ## &entity; errors.add(errorMsg(x, "unknown entity: " & x.entityName)) next(x) - of xmlEof: nil + of xmlEof: discard proc parseXml*(s: PStream, filename: string, errors: var seq[string]): PXmlNode = @@ -110,7 +110,7 @@ proc parseXml*(s: PStream, filename: string, of xmlElementOpen, xmlElementStart: result = parse(x, errors) break - of xmlComment, xmlWhitespace, xmlSpecial, xmlPI: nil # just skip it + of xmlComment, xmlWhitespace, xmlSpecial, xmlPI: discard # just skip it of xmlError: errors.add(errorMsg(x)) else: diff --git a/todo.txt b/todo.txt index fef073f90d..a9f2e80e5b 100644 --- a/todo.txt +++ b/todo.txt @@ -1,11 +1,9 @@ version 0.9.4 ============= +- fix GC issues - fix macros\tstringinterp.nim - test and fix showoff -- test and fix stdlib -- test and fix misc -- fix GC issues - test C source code generation - test and fix closures - test and fix exception handling From f05acd9612959860cdf55c699e5f097cf6692639 Mon Sep 17 00:00:00 2001 From: superfunc Date: Wed, 5 Feb 2014 16:56:43 -0800 Subject: [PATCH 59/72] Fixed typo's in tutorial 2 --- doc/tut2.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/tut2.txt b/doc/tut2.txt index 581239cc76..6738c55511 100644 --- a/doc/tut2.txt +++ b/doc/tut2.txt @@ -528,7 +528,7 @@ containers: proc newNode*[T](data: T): PBinaryTree[T] = # constructor for a node new(result) - result.dat = data + result.data = data proc add*[T](root: var PBinaryTree[T], n: PBinaryTree[T]) = # insert a node into the tree @@ -569,7 +569,7 @@ containers: var root: PBinaryTree[string] # instantiate a PBinaryTree with ``string`` - add(root, newNode("hallo")) # instantiates ``newNode`` and ``add`` + add(root, newNode("hello")) # instantiates ``newNode`` and ``add`` add(root, "world") # instantiates the second ``add`` proc for str in preorder(root): stdout.writeln(str) From 3be07d842a13cde69aeb5e7908ed62907daef4f7 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 6 Feb 2014 02:41:53 +0100 Subject: [PATCH 60/72] fixes regression: constant fac4 didn't work --- compiler/parser.nim | 23 +++++++++++++++-------- compiler/vm.nim | 5 ++++- compiler/vmgen.nim | 2 +- tests/exprs/tstmtexprs.nim | 7 ++++++- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 7c740559c3..ff3324b471 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -35,6 +35,7 @@ type lex*: TLexer # the lexer that is used for parsing tok*: TToken # the current token inPragma: int + inSemiStmtList: int proc parseAll*(p: var TParser): PNode proc openParser*(p: var TParser, filename: string, inputstream: PLLStream) @@ -455,11 +456,13 @@ proc complexOrSimpleStmt(p: var TParser): PNode proc simpleExpr(p: var TParser, mode = pmNormal): PNode proc semiStmtList(p: var TParser, result: PNode) = + inc p.inSemiStmtList result.add(complexOrSimpleStmt(p)) while p.tok.tokType == tkSemiColon: getTok(p) optInd(p, result) result.add(complexOrSimpleStmt(p)) + dec p.inSemiStmtList result.kind = nkStmtListExpr proc parsePar(p: var TParser): PNode = @@ -1881,14 +1884,18 @@ proc parseStmt(p: var TParser): PNode = parMessage(p, errComplexStmtRequiresInd) result = ast.emptyNode else: - result = newNodeP(nkStmtList, p) - while true: - if p.tok.indent >= 0: parMessage(p, errInvalidIndentation) - let a = simpleStmt(p) - if a.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) - result.add(a) - if p.tok.tokType != tkSemiColon: break - getTok(p) + if p.inSemiStmtList > 0: + result = simpleStmt(p) + if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) + else: + result = newNodeP(nkStmtList, p) + while true: + if p.tok.indent >= 0: parMessage(p, errInvalidIndentation) + let a = simpleStmt(p) + if a.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) + result.add(a) + if p.tok.tokType != tkSemiColon: break + getTok(p) proc parseAll(p: var TParser): PNode = result = newNodeP(nkStmtList, p) diff --git a/compiler/vm.nim b/compiler/vm.nim index 61881a8977..10ac7aaafa 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -14,7 +14,7 @@ import ast except getstr import strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes, unsigned, - parser, vmdeps, idents, trees, renderer, options + parser, vmdeps, idents, trees, renderer, options, transf from semfold import leValueConv, ordinalValToString from evaltempl import evalTemplate @@ -1078,6 +1078,7 @@ proc execute(c: PCtx, start: int): PNode = result = rawExecute(c, start, tos) proc evalStmt*(c: PCtx, n: PNode) = + let n = transformExpr(c.module, n) let start = genStmt(c, n) # execute new instructions; this redundant opcEof check saves us lots # of allocations in 'execute': @@ -1085,6 +1086,7 @@ proc evalStmt*(c: PCtx, n: PNode) = discard execute(c, start) proc evalExpr*(c: PCtx, n: PNode): PNode = + let n = transformExpr(c.module, n) let start = genExpr(c, n) assert c.code[start].opcode != opcEof result = execute(c, start) @@ -1127,6 +1129,7 @@ proc myProcess(c: PPassContext, n: PNode): PNode = const evalPass* = makePass(myOpen, nil, myProcess, myProcess) proc evalConstExprAux(module, prc: PSym, n: PNode, mode: TEvalMode): PNode = + let n = transformExpr(module, n) setupGlobalCtx(module) var c = globalCtx c.mode = mode diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 313c43dc44..d0e8dacf32 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1033,7 +1033,7 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest) = if s.isGlobal: if sfCompileTime in s.flags or c.mode == emRepl: discard - else: + elif s.position == 0: cannotEval(n) if s.position == 0: if sfImportc in s.flags: c.importcSym(n.info, s) diff --git a/tests/exprs/tstmtexprs.nim b/tests/exprs/tstmtexprs.nim index 8149ec4b88..816e58cb1d 100644 --- a/tests/exprs/tstmtexprs.nim +++ b/tests/exprs/tstmtexprs.nim @@ -1,5 +1,6 @@ discard """ - output: '''(bar: bar) + output: '''24 +(bar: bar) 1244 6 abcdefghijklmnopqrstuvwxyz @@ -8,6 +9,10 @@ abcdefghijklmnopqrstuvwxyz import strutils +const fac4 = (var x = 1; for i in 1..4: x *= i; x) + +echo fac4 + when true: proc test(foo: proc (x, y: int): bool) = echo foo(5, 5) From 38e4dfc164404a601e0556936497e7ee97a88fba Mon Sep 17 00:00:00 2001 From: EXetoC Date: Thu, 6 Feb 2014 14:18:44 +0100 Subject: [PATCH 61/72] Export procs that are useful elsewhere. --- lib/pure/times.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 6186fcad8f..de6c4e4fa2 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -211,7 +211,9 @@ proc initInterval*(miliseconds, seconds, minutes, hours, days, months, result.months = months result.years = years -proc isLeapYear(year: int): bool = +proc isLeapYear*(year: int): bool = + ## returns true if ``year`` is a leap year + if year mod 400 == 0: return true elif year mod 100 == 0: @@ -221,7 +223,9 @@ proc isLeapYear(year: int): bool = else: return false -proc getDaysInMonth(month: TMonth, year: int): int = +proc getDaysInMonth*(month: TMonth, year: int): int = + ## gets the amount of days in a ``month`` of a ``year`` + # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month case month of mFeb: result = if isLeapYear(year): 29 else: 28 From 75f232eb6efa1d213ca84c524ee9119da128d6e3 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Thu, 6 Feb 2014 14:55:05 -0600 Subject: [PATCH 62/72] removed unittest from talgorithm --- tests/stdlib/talgorithm.nim | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/stdlib/talgorithm.nim b/tests/stdlib/talgorithm.nim index 37de1262f3..7ab652c82a 100644 --- a/tests/stdlib/talgorithm.nim +++ b/tests/stdlib/talgorithm.nim @@ -1,14 +1,8 @@ -import unittest import algorithm -suite "product": - test "empty input": - check product[int](newSeq[seq[int]]()) == newSeq[seq[int]]() - test "bit more empty input": - check product[int](@[newSeq[int](), @[], @[]]) == newSeq[seq[int]]() - test "a simple case of one element": - check product(@[@[1,2]]) == @[@[1,2]] - test "two elements": - check product(@[@[1,2], @[3,4]]) == @[@[2,4],@[1,4],@[2,3],@[1,3]] - test "three elements": - check product(@[@[1,2], @[3,4], @[5,6]]) == @[@[2,4,6],@[1,4,6],@[2,3,6],@[1,3,6], @[2,4,5],@[1,4,5],@[2,3,5],@[1,3,5]] +doAssert product[int](newSeq[seq[int]]()) == newSeq[seq[int]](), "empty input" +doAssert product[int](@[newSeq[int](), @[], @[]]) == newSeq[seq[int]](), "bit more empty input" +doAssert product(@[@[1,2]]) == @[@[1,2]], "a simple case of one element" +doAssert product(@[@[1,2], @[3,4]]) == @[@[2,4],@[1,4],@[2,3],@[1,3]], "two elements" +doAssert product(@[@[1,2], @[3,4], @[5,6]]) == @[@[2,4,6],@[1,4,6],@[2,3,6],@[1,3,6], @[2,4,5],@[1,4,5],@[2,3,5],@[1,3,5]], "three elements" +doAssert product(@[@[1,2], @[]]) == newSeq[seq[int]](), "two elements, but one empty" From 5498415f3b44048739c9b7116638824713d9c1df Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Thu, 6 Feb 2014 16:11:55 -0600 Subject: [PATCH 63/72] indexBy, which indexes a collection into a hashtable --- lib/pure/collections/tables.nim | 10 ++++++++++ tests/collections/ttables.nim | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/collections/ttables.nim diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 73da274b9e..40ae57b5a5 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -189,6 +189,16 @@ template dollarImpl(): stmt {.dirty.} = proc `$`*[A, B](t: TTable[A, B]): string = ## The `$` operator for hash tables. dollarImpl() + +proc `==`*[A, B](s, t: TTable[A, B]): bool = + s.counter == t.counter and s.data == t.data + +proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): TTable[C, B] = + ## Index the collection with the proc provided. + # TODO: As soon as supported, change collection: A to collection: A[B] + result = initTable[C, B]() + for item in collection: + result[index(item)] = item # ------------------------------ ordered table ------------------------------ diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim new file mode 100644 index 0000000000..f374d5504c --- /dev/null +++ b/tests/collections/ttables.nim @@ -0,0 +1,22 @@ +import tables + +doAssert indexBy(newSeq[int](), proc(x: int):int = x) == initTable[int, int](), "empty int table" + +var tbl1 = initTable[int, int]() +tbl1.add(1,1) +tbl1.add(2,2) +doAssert indexBy(@[1,2], proc(x: int):int = x) == tbl1, "int table" + +type + TElem = object + foo: int + bar: string + +let + elem1 = TElem(foo: 1, bar: "bar") + elem2 = TElem(foo: 2, bar: "baz") + +var tbl2 = initTable[string, TElem]() +tbl2.add("bar", elem1) +tbl2.add("baz", elem2) +doAssert indexBy(@[elem1,elem2], proc(x: TElem): string = x.bar) == tbl2, "element table" From a087f6057e70e8b7c57ec9a20ac7f7815afe9327 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 7 Feb 2014 17:06:20 +0100 Subject: [PATCH 64/72] bugfix: codegen issue that caused getGMTime() to leak memory --- compiler/ccgexprs.nim | 2 +- doc/manual.txt | 2 ++ tests/gc/gcleak4.nim | 2 +- tests/gc/gcleak5.nim | 25 +++++++++++++++++++++++++ tests/testament/categories.nim | 5 +++++ todo.txt | 2 +- 6 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 tests/gc/gcleak5.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index be47ac0c4a..031ab8d704 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1049,7 +1049,7 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = app(tmp2.r, field.loc.r) tmp2.k = locTemp tmp2.t = field.loc.t - tmp2.s = OnHeap + tmp2.s = if isRef: OnHeap else: OnStack tmp2.heapRoot = tmp.r expr(p, it.sons[1], tmp2) if d.k == locNone: diff --git a/doc/manual.txt b/doc/manual.txt index da229d169b..520e4f62e9 100644 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -2231,6 +2231,8 @@ Instead of: Using statement --------------- +**Warning**: The ``using`` statement is highly experimental! + The `using statement`:idx: provides syntactic convenience for procs that heavily use a single contextual parameter. When applied to a variable or a constant, it will instruct Nimrod to automatically consider the used symbol as diff --git a/tests/gc/gcleak4.nim b/tests/gc/gcleak4.nim index bd7bded283..6f2b8a1fe6 100644 --- a/tests/gc/gcleak4.nim +++ b/tests/gc/gcleak4.nim @@ -6,7 +6,7 @@ when defined(GC_setMaxPause): GC_setMaxPause 2_000 type - TExpr = object ## abstract base class for an expression + TExpr = object {.inheritable.} ## abstract base class for an expression PLiteral = ref TLiteral TLiteral = object of TExpr x: int diff --git a/tests/gc/gcleak5.nim b/tests/gc/gcleak5.nim new file mode 100644 index 0000000000..b9131051b0 --- /dev/null +++ b/tests/gc/gcleak5.nim @@ -0,0 +1,25 @@ +discard """ + output: "success" +""" + +import os, times + +proc main = + var i = 0 + for ii in 0..50_000: + #while true: + var t = getTime() + var g = t.getGMTime() + #echo isOnStack(addr g) + + if i mod 100 == 0: + let om = getOccupiedMem() + echo "memory: ", om + if om > 100_000: quit "leak" + + inc(i) + sleep(1) + + echo "success" + +main() diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 5dd8414470..f9f5698bbb 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -123,9 +123,14 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "gcleak2" test "gctest" test "gcleak3" + test "gcleak4" + test "gcleak5" test "weakrefs" test "cycleleak" test "closureleak" + test "refarrayleak" + rest "stackrefleak" + # ------------------------- threading tests ----------------------------------- diff --git a/todo.txt b/todo.txt index a9f2e80e5b..738e9b3fad 100644 --- a/todo.txt +++ b/todo.txt @@ -5,7 +5,7 @@ version 0.9.4 - fix macros\tstringinterp.nim - test and fix showoff - test C source code generation -- test and fix closures +- fix closures - test and fix exception handling - implement 'union' and 'bits' pragmas From c3adc19f471ddddf0cab9a92908dcdbede26b3eb Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 7 Feb 2014 18:49:41 +0100 Subject: [PATCH 65/72] tester compiles again --- tests/gc/gcleak5.nim | 2 +- tests/testament/categories.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gc/gcleak5.nim b/tests/gc/gcleak5.nim index b9131051b0..9e2948729e 100644 --- a/tests/gc/gcleak5.nim +++ b/tests/gc/gcleak5.nim @@ -14,7 +14,7 @@ proc main = if i mod 100 == 0: let om = getOccupiedMem() - echo "memory: ", om + #echo "memory: ", om if om > 100_000: quit "leak" inc(i) diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index f9f5698bbb..442dd12123 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -129,7 +129,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "cycleleak" test "closureleak" test "refarrayleak" - rest "stackrefleak" + test "stackrefleak" # ------------------------- threading tests ----------------------------------- From eedf51e9d11e362fc14bc8b33aca58775420576e Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 8 Feb 2014 16:08:03 +0000 Subject: [PATCH 66/72] Added new sockets modules: asyncio2, net, and sockets2. --- lib/pure/asyncio2.nim | 469 ++++++++++++++++++++++++++++++++++++++++ lib/pure/net.nim | 40 ++++ lib/pure/sockets2.nim | 202 +++++++++++++++++ lib/windows/winlean.nim | 77 +++++++ 4 files changed, 788 insertions(+) create mode 100644 lib/pure/asyncio2.nim create mode 100644 lib/pure/net.nim create mode 100644 lib/pure/sockets2.nim diff --git a/lib/pure/asyncio2.nim b/lib/pure/asyncio2.nim new file mode 100644 index 0000000000..38fa934526 --- /dev/null +++ b/lib/pure/asyncio2.nim @@ -0,0 +1,469 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import os, oids, tables, strutils + +import winlean + +import sockets2, net + +## Asyncio2 +## -------- +## +## This module implements a brand new asyncio module based on Futures. +## IOCP is used under the hood on Windows and the selectors module is used for +## other operating systems. + +# -- Futures + +type + PFuture*[T] = ref object + value: T + finished: bool + error: ref EBase + cb: proc (future: PFuture[T]) {.closure.} + +proc newFuture*[T](): PFuture[T] = + ## Creates a new future. + new(result) + result.finished = false + +proc complete*[T](future: PFuture[T], val: T) = + ## Completes ``future`` with value ``val``. + assert(not future.finished) + assert(future.error == nil) + future.value = val + future.finished = true + if future.cb != nil: + future.cb(future) + +proc fail*[T](future: PFuture[T], error: ref EBase) = + ## Completes ``future`` with ``error``. + assert(not future.finished) + future.finished = true + future.error = error + if future.cb != nil: + future.cb(future) + +proc `callback=`*[T](future: PFuture[T], + cb: proc (future: PFuture[T]) {.closure.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + future.cb = cb + if future.finished: + future.cb(future) + +proc read*[T](future: PFuture[T]): T = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``EInvalidValue`` exception. + ## + ## If the result of the future is an error then that error will be raised. + if future.finished: + if future.error != nil: raise future.error + return future.value + else: + # TODO: Make a custom exception type for this? + raise newException(EInvalidValue, "Future still in progress.") + +proc finished*[T](future: PFuture[T]): bool = + ## Determines whether ``future`` has completed. + ## + ## ``True`` may indicate an error or a value. Use ``hasError`` to distinguish. + future.finished + +proc failed*[T](future: PFuture[T]): bool = + ## Determines whether ``future`` completed with an error. + future.error != nil + +when defined(windows): + type + TCompletionKey = dword + + TCompletionData* = object + sock: TSocketHandle + cb: proc (sock: TSocketHandle, errcode: TOSErrorCode) {.closure.} + + PDispatcher* = ref object + ioPort: THandle + + TCustomOverlapped = object + Internal*: DWORD + InternalHigh*: DWORD + Offset*: DWORD + OffsetHigh*: DWORD + hEvent*: THANDLE + data*: TCompletionData + + PCustomOverlapped = ptr TCustomOverlapped + + proc newDispatcher*(): PDispatcher = + ## Creates a new Dispatcher instance. + new result + result.ioPort = CreateIOCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) + + proc register*(p: PDispatcher, sock: TSocketHandle) = + ## Registers ``sock`` with the dispatcher ``p``. + if CreateIOCompletionPort(sock.THandle, p.ioPort, + cast[TCompletionKey](sock), 1) == 0: + OSError(OSLastError()) + + proc poll*(p: PDispatcher, timeout = 500) = + ## Waits for completion events and processes them. + let llTimeout = + if timeout == -1: winlean.INFINITE + else: timeout.int32 + var lpNumberOfBytesTransferred: DWORD + var lpCompletionKey: ULONG + var lpOverlapped: POverlapped + let res = GetQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred, + addr lpCompletionKey, addr lpOverlapped, llTimeout).bool + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + var customOverlapped = cast[PCustomOverlapped](lpOverlapped) + if res: + assert customOverlapped.data.sock == lpCompletionKey.TSocketHandle + + customOverlapped.data.cb(customOverlapped.data.sock, TOSErrorCode(-1)) + dealloc(customOverlapped) + else: + let errCode = OSLastError() + if lpOverlapped != nil: + assert customOverlapped.data.sock == lpCompletionKey.TSocketHandle + dealloc(customOverlapped) + customOverlapped.data.cb(customOverlapped.data.sock, errCode) + else: + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + discard + else: OSError(errCode) + + var connectExPtr: pointer = nil + var acceptExPtr: pointer = nil + var getAcceptExSockAddrsPtr: pointer = nil + + proc initPointer(s: TSocketHandle, func: var pointer, guid: var TGUID): bool = + # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c + var bytesRet: DWord + func = nil + result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, + sizeof(TGUID).dword, addr func, sizeof(pointer).DWORD, + addr bytesRet, nil, nil) == 0 + + proc initAll() = + let dummySock = socket() + if not initPointer(dummySock, connectExPtr, WSAID_CONNECTEX): + OSError(OSLastError()) + if not initPointer(dummySock, acceptExPtr, WSAID_ACCEPTEX): + OSError(OSLastError()) + if not initPointer(dummySock, getAcceptExSockAddrsPtr, WSAID_GETACCEPTEXSOCKADDRS): + OSError(OSLastError()) + + proc connectEx(s: TSocketHandle, name: ptr TSockAddr, namelen: cint, + lpSendBuffer: pointer, dwSendDataLength: dword, + lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool = + if connectExPtr.isNil: raise newException(EInvalidValue, "Need to initialise ConnectEx().") + let func = + cast[proc (s: TSocketHandle, name: ptr TSockAddr, namelen: cint, + lpSendBuffer: pointer, dwSendDataLength: dword, + lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool {.stdcall.}](connectExPtr) + + result = func(s, name, namelen, lpSendBuffer, dwSendDataLength, lpdwBytesSent, + lpOverlapped) + + proc acceptEx(listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer, + dwReceiveDataLength, dwLocalAddressLength, + dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD, + lpOverlapped: POverlapped): bool = + if acceptExPtr.isNil: raise newException(EInvalidValue, "Need to initialise AcceptEx().") + let func = + cast[proc (listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer, + dwReceiveDataLength, dwLocalAddressLength, + dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD, + lpOverlapped: POverlapped): bool {.stdcall.}](acceptExPtr) + result = func(listenSock, acceptSock, lpOutputBuffer, dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived, + lpOverlapped) + + proc getAcceptExSockaddrs(lpOutputBuffer: pointer, + dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: DWORD, + LocalSockaddr: ptr ptr TSockAddr, LocalSockaddrLength: lpint, + RemoteSockaddr: ptr ptr TSockAddr, RemoteSockaddrLength: lpint) = + if getAcceptExSockAddrsPtr.isNil: + raise newException(EInvalidValue, "Need to initialise getAcceptExSockAddrs().") + + let func = + cast[proc (lpOutputBuffer: pointer, + dwReceiveDataLength, dwLocalAddressLength, + dwRemoteAddressLength: DWORD, LocalSockaddr: ptr ptr TSockAddr, + LocalSockaddrLength: lpint, RemoteSockaddr: ptr ptr TSockAddr, + RemoteSockaddrLength: lpint) {.stdcall.}](getAcceptExSockAddrsPtr) + + func(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength, + dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength, + RemoteSockaddr, RemoteSockaddrLength) + + proc connect*(p: PDispatcher, socket: TSocketHandle, address: string, port: TPort, + af = AF_INET): PFuture[int] = + ## Connects ``socket`` to server at ``address:port``. + ## + ## Returns a ``PFuture`` which will complete when the connection succeeds + ## or an error occurs. + + var retFuture = newFuture[int]()# TODO: Change to void when that regression is fixed. + # Apparently ``ConnectEx`` expects the socket to be initially bound: + var saddr: Tsockaddr_in + saddr.sin_family = int16(toInt(af)) + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(socket, cast[ptr TSockAddr](addr(saddr)), + sizeof(saddr).TSockLen) < 0'i32: + OSError(OSLastError()) + + var aiList = getAddrInfo(address, port, af) + var success = false + var lastError: TOSErrorCode + var it = aiList + while it != nil: + # "the OVERLAPPED structure must remain valid until the I/O completes" + # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx + var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) + ol.data = TCompletionData(sock: socket, cb: + proc (sock: TSocketHandle, errcode: TOSErrorCode) = + if errcode == TOSErrorCode(-1): + retFuture.complete(0) + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) + ) + + var ret = connectEx(socket, it.ai_addr, sizeof(TSockAddrIn).cint, + nil, 0, nil, cast[POverlapped](ol)) + if ret: + # Request to connect completed immediately. + success = true + retFuture.complete(0) + dealloc(ol) + break + else: + lastError = OSLastError() + if lastError.int32 == ERROR_IO_PENDING: + # In this case ``ol`` will be deallocated in ``poll``. + success = true + break + else: + dealloc(ol) + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(EOS, osErrorMsg(lastError))) + return retFuture + + proc recv*(p: PDispatcher, socket: TSocketHandle, size: int): PFuture[string] = + ## Reads ``size`` bytes from ``socket``. Returned future will complete once + ## all of the requested data is read. + + var retFuture = newFuture[string]() + + var dataBuf: TWSABuf + dataBuf.buf = newString(size) + dataBuf.len = size + + var bytesReceived, flags: DWord + var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) + ol.data = TCompletionData(sock: socket, cb: + proc (sock: TSocketHandle, errcode: TOSErrorCode) = + if errcode == TOSErrorCode(-1): + var data = newString(size) + copyMem(addr data[0], addr dataBuf.buf[0], size) + retFuture.complete($data) + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) + ) + + let ret = WSARecv(socket, addr dataBuf, 1, addr bytesReceived, + addr flags, cast[POverlapped](ol), nil) + if ret == -1: + let err = OSLastError() + if err.int32 != ERROR_IO_PENDING: + retFuture.fail(newException(EOS, osErrorMsg(err))) + dealloc(ol) + else: + # Request to read completed immediately. + var data = newString(size) + copyMem(addr data[0], addr dataBuf.buf[0], size) + retFuture.complete($data) + dealloc(ol) + return retFuture + + proc send*(p: PDispatcher, socket: TSocketHandle, data: string): PFuture[int] = + ## Sends ``data`` to ``socket``. The returned future will complete once all + ## data has been sent. + var retFuture = newFuture[int]() + + var dataBuf: TWSABuf + dataBuf.buf = data + dataBuf.len = data.len + + var bytesReceived, flags: DWord + var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) + ol.data = TCompletionData(sock: socket, cb: + proc (sock: TSocketHandle, errcode: TOSErrorCode) = + if errcode == TOSErrorCode(-1): + retFuture.complete(0) + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) + ) + + let ret = WSASend(socket, addr dataBuf, 1, addr bytesReceived, + flags, cast[POverlapped](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + retFuture.fail(newException(EOS, osErrorMsg(err))) + dealloc(ol) + else: + retFuture.complete(0) + dealloc(ol) + return retFuture + + proc acceptAddr*(p: PDispatcher, socket: TSocketHandle): + PFuture[tuple[address: string, client: TSocketHandle]] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection and the remote address of the client. + ## The future will complete when the connection is successfully accepted. + + var retFuture = newFuture[tuple[address: string, client: TSocketHandle]]() + + var clientSock = socket() + if clientSock == OSInvalidSocket: osError(osLastError()) + + const lpOutputLen = 1024 + var lpOutputBuf = newString(lpOutputLen) + var dwBytesReceived: DWORD + let dwReceiveDataLength = 0.DWORD # We don't want any data to be read. + let dwLocalAddressLength = DWORD(sizeof (TSockaddr_in) + 16) + let dwRemoteAddressLength = DWORD(sizeof(TSockaddr_in) + 16) + + template completeAccept(): stmt {.immediate, dirty.} = + var listenSock = socket + let setoptRet = setsockopt(clientSock, SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, + sizeof(listenSock).TSockLen) + if setoptRet != 0: osError(osLastError()) + + var LocalSockaddr, RemoteSockaddr: ptr TSockAddr + var localLen, remoteLen: int32 + getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, + addr LocalSockaddr, addr localLen, + addr RemoteSockaddr, addr remoteLen) + # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 + retFuture.complete( + (address: $inet_ntoa(cast[ptr Tsockaddr_in](remoteSockAddr).sin_addr), + client: clientSock) + ) + + var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) + ol.data = TCompletionData(sock: socket, cb: + proc (sock: TSocketHandle, errcode: TOSErrorCode) = + if errcode == TOSErrorCode(-1): + completeAccept() + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) + ) + + # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx + let ret = acceptEx(socket, clientSock, addr lpOutputBuf[0], + dwReceiveDataLength, + dwLocalAddressLength, + dwRemoteAddressLength, + addr dwBytesReceived, cast[POverlapped](ol)) + + if not ret: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + retFuture.fail(newException(EOS, osErrorMsg(err))) + dealloc(ol) + else: + completeAccept() + dealloc(ol) + + return retFuture + + proc accept*(p: PDispatcher, socket: TSocketHandle): PFuture[TSocketHandle] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection. + ## The future will complete when the connection is successfully accepted. + var retFut = newFuture[TSocketHandle]() + var fut = p.acceptAddr(socket) + fut.callback = + proc (future: PFuture[tuple[address: string, client: TSocketHandle]]) = + assert future.finished + if future.failed: + retFut.fail(future.error) + else: + retFut.complete(future.read.client) + return retFut + + initAll() +else: + # TODO: Selectors. + + +when isMainModule: + + var p = newDispatcher() + var sock = socket() + #sock.setBlocking false + p.register(sock) + + when true: + + var f = p.connect(sock, "irc.freenode.org", TPort(6667)) + f.callback = + proc (future: PFuture[int]) = + echo("Connected in future!") + echo(future.read) + for i in 0 .. 50: + var recvF = p.recv(sock, 10) + recvF.callback = + proc (future: PFuture[string]) = + echo("Read: ", future.read) + + else: + + sock.bindAddr(TPort(6667)) + sock.listen() + proc onAccept(future: PFuture[TSocketHandle]) = + echo "Accepted" + var t = p.send(future.read, "test\c\L") + t.callback = + proc (future: PFuture[int]) = + echo(future.read) + + var f = p.accept(sock) + f.callback = onAccept + + var f = p.accept(sock) + f.callback = onAccept + + while true: + p.poll() + echo "polled" + + + + + + + + \ No newline at end of file diff --git a/lib/pure/net.nim b/lib/pure/net.nim new file mode 100644 index 0000000000..bdcae677e8 --- /dev/null +++ b/lib/pure/net.nim @@ -0,0 +1,40 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements a high-level cross-platform sockets interface. + +import sockets2, os + +type + TSocket* = TSocketHandle + +proc bindAddr*(socket: TSocket, port = TPort(0), address = "") {. + tags: [FReadIO].} = + + ## binds an address/port number to a socket. + ## Use address string in dotted decimal form like "a.b.c.d" + ## or leave "" for any address. + + if address == "": + var name: TSockaddr_in + when defined(windows): + name.sin_family = toInt(AF_INET).int16 + else: + name.sin_family = toInt(AF_INET) + name.sin_port = htons(int16(port)) + name.sin_addr.s_addr = htonl(INADDR_ANY) + if bindAddr(socket, cast[ptr TSockAddr](addr(name)), + sizeof(name).TSocklen) < 0'i32: + osError(osLastError()) + else: + var aiList = getAddrInfo(address, port, AF_INET) + if bindAddr(socket, aiList.ai_addr, aiList.ai_addrlen.TSocklen) < 0'i32: + dealloc(aiList) + osError(osLastError()) + dealloc(aiList) \ No newline at end of file diff --git a/lib/pure/sockets2.nim b/lib/pure/sockets2.nim new file mode 100644 index 0000000000..22624bbad5 --- /dev/null +++ b/lib/pure/sockets2.nim @@ -0,0 +1,202 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements a low-level cross-platform sockets interface. Look +## at the ``net`` module for the higher-level version. + +import unsigned, os + +when hostos == "solaris": + {.passl: "-lsocket -lnsl".} + +when defined(Windows): + import winlean +else: + import posix + +export TSocketHandle, TSockaddr_in, TAddrinfo, INADDR_ANY, TSockAddr, TSockLen, + inet_ntoa + +type + + TPort* = distinct uint16 ## port type + + TDomain* = enum ## domain, which specifies the protocol family of the + ## created socket. Other domains than those that are listed + ## here are unsupported. + AF_UNIX, ## for local socket (using a file). Unsupported on Windows. + AF_INET = 2, ## for network protocol IPv4 or + AF_INET6 = 23 ## for network protocol IPv6. + + TType* = enum ## second argument to `socket` proc + SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets + SOCK_DGRAM = 2, ## datagram service or Datagram Sockets + SOCK_RAW = 3, ## raw protocols atop the network layer. + SOCK_SEQPACKET = 5 ## reliable sequenced packet service + + TProtocol* = enum ## third argument to `socket` proc + IPPROTO_TCP = 6, ## Transmission control protocol. + IPPROTO_UDP = 17, ## User datagram protocol. + IPPROTO_IP, ## Internet protocol. Unsupported on Windows. + IPPROTO_IPV6, ## Internet Protocol Version 6. Unsupported on Windows. + IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. + IPPROTO_ICMP ## Control message protocol. Unsupported on Windows. + + TServent* {.pure, final.} = object ## information about a service + name*: string + aliases*: seq[string] + port*: TPort + proto*: string + + Thostent* {.pure, final.} = object ## information about a given host + name*: string + aliases*: seq[string] + addrtype*: TDomain + length*: int + addrList*: seq[string] + +when defined(windows): + let + OSInvalidSocket* = winlean.INVALID_SOCKET +else: + let + OSInvalidSocket* = posix.INVALID_SOCKET + +proc `==`*(a, b: TPort): bool {.borrow.} + ## ``==`` for ports. + +proc `$`*(p: TPort): string {.borrow.} + ## returns the port number as a string + +proc toInt*(domain: TDomain): cint + ## Converts the TDomain enum to a platform-dependent ``cint``. + +proc toInt*(typ: TType): cint + ## Converts the TType enum to a platform-dependent ``cint``. + +proc toInt*(p: TProtocol): cint + ## Converts the TProtocol enum to a platform-dependent ``cint``. + +when defined(posix): + proc toInt(domain: TDomain): cint = + case domain + of AF_UNIX: result = posix.AF_UNIX + of AF_INET: result = posix.AF_INET + of AF_INET6: result = posix.AF_INET6 + else: nil + + proc toInt(typ: TType): cint = + case typ + of SOCK_STREAM: result = posix.SOCK_STREAM + of SOCK_DGRAM: result = posix.SOCK_DGRAM + of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET + of SOCK_RAW: result = posix.SOCK_RAW + else: nil + + proc toInt(p: TProtocol): cint = + case p + of IPPROTO_TCP: result = posix.IPPROTO_TCP + of IPPROTO_UDP: result = posix.IPPROTO_UDP + of IPPROTO_IP: result = posix.IPPROTO_IP + of IPPROTO_IPV6: result = posix.IPPROTO_IPV6 + of IPPROTO_RAW: result = posix.IPPROTO_RAW + of IPPROTO_ICMP: result = posix.IPPROTO_ICMP + else: nil + +else: + proc toInt(domain: TDomain): cint = + result = toU16(ord(domain)) + + proc toInt(typ: TType): cint = + result = cint(ord(typ)) + + proc toInt(p: TProtocol): cint = + result = cint(ord(p)) + + +proc socket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, + protocol: TProtocol = IPPROTO_TCP): TSocketHandle = + ## Creates a new socket; returns `InvalidSocket` if an error occurs. + + # TODO: The function which will use this will raise EOS. + socket(toInt(domain), toInt(typ), toInt(protocol)) + +proc close*(socket: TSocketHandle) = + ## closes a socket. + when defined(windows): + discard winlean.closeSocket(socket) + else: + discard posix.close(socket) + # TODO: These values should not be discarded. An EOS should be raised. + # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times + +proc bindAddr*(socket: TSocketHandle, name: ptr TSockAddr, namelen: TSockLen): cint = + result = bindSocket(socket, name, namelen) + +proc listen*(socket: TSocketHandle, backlog = SOMAXCONN) {.tags: [FReadIO].} = + ## Marks ``socket`` as accepting connections. + ## ``Backlog`` specifies the maximum length of the + ## queue of pending connections. + when defined(windows): + if winlean.listen(socket, cint(backlog)) < 0'i32: osError(osLastError()) + else: + if posix.listen(socket, cint(backlog)) < 0'i32: osError(osLastError()) + +proc getAddrInfo*(address: string, port: TPort, af: TDomain = AF_INET, typ: TType = SOCK_STREAM, + prot: TProtocol = IPPROTO_TCP): ptr TAddrInfo = + ## + ## + ## **Warning**: The resulting ``ptr TAddrInfo`` must be freed using ``dealloc``! + var hints: TAddrInfo + result = nil + hints.ai_family = toInt(af) + hints.ai_socktype = toInt(typ) + hints.ai_protocol = toInt(prot) + var gaiResult = getAddrInfo(address, $port, addr(hints), result) + if gaiResult != 0'i32: + when defined(windows): + OSError(OSLastError()) + else: + raise newException(EOS, $gai_strerror(gaiResult)) + +proc dealloc*(ai: ptr TAddrInfo) = + freeaddrinfo(ai) + +proc ntohl*(x: int32): int32 = + ## Converts 32-bit integers from network to host byte order. + ## On machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 4-byte swap operation. + when cpuEndian == bigEndian: result = x + else: result = (x shr 24'i32) or + (x shr 8'i32 and 0xff00'i32) or + (x shl 8'i32 and 0xff0000'i32) or + (x shl 24'i32) + +proc ntohs*(x: int16): int16 = + ## Converts 16-bit integers from network to host byte order. On machines + ## where the host byte order is the same as network byte order, this is + ## a no-op; otherwise, it performs a 2-byte swap operation. + when cpuEndian == bigEndian: result = x + else: result = (x shr 8'i16) or (x shl 8'i16) + +proc htonl*(x: int32): int32 = + ## Converts 32-bit integers from host to network byte order. On machines + ## where the host byte order is the same as network byte order, this is + ## a no-op; otherwise, it performs a 4-byte swap operation. + result = sockets2.ntohl(x) + +proc htons*(x: int16): int16 = + ## Converts 16-bit positive integers from host to network byte order. + ## On machines where the host byte order is the same as network byte + ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. + result = sockets2.ntohs(x) + +when defined(Windows): + var wsa: TWSADATA + if WSAStartup(0x0101'i16, addr wsa) != 0: OSError(OSLastError()) \ No newline at end of file diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 5b641185eb..7713d503aa 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -16,8 +16,12 @@ const type THandle* = int LONG* = int32 + ULONG* = int + PULONG* = ptr int WINBOOL* = int32 DWORD* = int32 + PDWORD* = ptr DWORD + LPINT* = ptr int32 HDC* = THandle HGLRC* = THandle @@ -632,3 +636,76 @@ when not useWinUnicode: proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall, dynlib: "kernel32", importc: "UnmapViewOfFile".} +type + TOVERLAPPED* {.final, pure.} = object + Internal*: DWORD + InternalHigh*: DWORD + Offset*: DWORD + OffsetHigh*: DWORD + hEvent*: THANDLE + + POVERLAPPED* = ptr TOVERLAPPED + + POVERLAPPED_COMPLETION_ROUTINE* = proc (para1: DWORD, para2: DWORD, + para3: POVERLAPPED){.stdcall.} + + TGUID* {.final, pure.} = object + D1*: int32 + D2*: int16 + D3*: int16 + D4*: array [0..7, int8] + +const + ERROR_IO_PENDING* = 997 + +proc CreateIoCompletionPort*(FileHandle: THANDLE, ExistingCompletionPort: THANDLE, + CompletionKey: DWORD, + NumberOfConcurrentThreads: DWORD): THANDLE{.stdcall, + dynlib: "kernel32", importc: "CreateIoCompletionPort".} + +proc GetQueuedCompletionStatus*(CompletionPort: THandle, + lpNumberOfBytesTransferred: PDWORD, lpCompletionKey: PULONG, + lpOverlapped: ptr POverlapped, + dwMilliseconds: DWORD): WINBOOL{.stdcall, + dynlib: "kernel32", importc: "GetQueuedCompletionStatus".} + +const + IOC_OUT* = 0x40000000 + IOC_IN* = 0x80000000 + IOC_WS2* = 0x08000000 + IOC_INOUT* = IOC_IN or IOC_OUT + +template WSAIORW*(x,y): expr = (IOC_INOUT or x or y) + +const + SIO_GET_EXTENSION_FUNCTION_POINTER* = WSAIORW(IOC_WS2,6).DWORD + SO_UPDATE_ACCEPT_CONTEXT* = 0x700B + +var + WSAID_CONNECTEX*: TGUID = TGUID(D1: 0x25a207b9, D2: 0xddf3'i16, D3: 0x4660, D4: [ + 0x8e'i8, 0xe9'i8, 0x76'i8, 0xe5'i8, 0x8c'i8, 0x74'i8, 0x06'i8, 0x3e'i8]) + WSAID_ACCEPTEX*: TGUID = TGUID(D1: 0xb5367df1'i32, D2: 0xcbac'i16, D3: 0x11cf, D4: [ + 0x95'i8, 0xca'i8, 0x00'i8, 0x80'i8, 0x5f'i8, 0x48'i8, 0xa1'i8, 0x92'i8]) + WSAID_GETACCEPTEXSOCKADDRS*: TGUID = TGUID(D1: 0xb5367df2'i32, D2: 0xcbac'i16, D3: 0x11cf, D4: [ + 0x95'i8, 0xca'i8, 0x00'i8, 0x80'i8, 0x5f'i8, 0x48'i8, 0xa1'i8, 0x92'i8]) + +proc WSAIoctl*(s: TSocketHandle, dwIoControlCode: dword, lpvInBuffer: pointer, + cbInBuffer: dword, lpvOutBuffer: pointer, cbOutBuffer: dword, + lpcbBytesReturned: PDword, lpOverlapped: POVERLAPPED, + lpCompletionRoutine: POVERLAPPED_COMPLETION_ROUTINE): cint + {.stdcall, importc: "WSAIoctl", dynlib: "Ws2_32.dll".} + +type + TWSABuf* {.importc: "WSABUF", header: "winsock2.h".} = object + len*: ULONG + buf*: cstring + +proc WSARecv*(s: TSocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesReceived, flags: PDWORD, lpOverlapped: POverlapped, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSARecv", dynlib: "Ws2_32.dll".} + +proc WSASend*(s: TSocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesSent: PDWord, flags: DWORD, lpOverlapped: POverlapped, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSASend", dynlib: "Ws2_32.dll".} From 76e8efe36eede6db4c79586919c479a62fdb52dc Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 8 Feb 2014 22:59:40 +0000 Subject: [PATCH 67/72] Added a generic-lacking version of PFuture. --- lib/pure/asyncio2.nim | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/pure/asyncio2.nim b/lib/pure/asyncio2.nim index 38fa934526..cdb4a6f492 100644 --- a/lib/pure/asyncio2.nim +++ b/lib/pure/asyncio2.nim @@ -23,9 +23,12 @@ import sockets2, net # -- Futures type - PFuture*[T] = ref object - value: T + PFutureVoid* = ref object of PObject + cbVoid: proc () {.closure.} finished: bool + + PFuture*[T] = ref object of PFutureVoid + value: T error: ref EBase cb: proc (future: PFuture[T]) {.closure.} @@ -42,6 +45,8 @@ proc complete*[T](future: PFuture[T], val: T) = future.finished = true if future.cb != nil: future.cb(future) + if future.cbVoid != nil: + future.cbVoid() proc fail*[T](future: PFuture[T], error: ref EBase) = ## Completes ``future`` with ``error``. @@ -60,6 +65,17 @@ proc `callback=`*[T](future: PFuture[T], if future.finished: future.cb(future) +proc `callbackVoid=`*(future: PFutureVoid, cb: proc () {.closure.}) = + ## Sets the **void** callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + ## + ## **Note**: This is used for the ``await`` functionality, you most likely + ## want to use ``callback``. + future.cbVoid = cb + if future.finished: + future.cbVoid() + proc read*[T](future: PFuture[T]): T = ## Retrieves the value of ``future``. Future must be finished otherwise ## this function will fail with a ``EInvalidValue`` exception. From dbc8aa60e30da3f85306455387bb563437215725 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 9 Feb 2014 00:41:53 +0100 Subject: [PATCH 68/72] fixes 'newSeq[T]' instantiation bug --- compiler/semexprs.nim | 12 +++++++++--- compiler/semgnrc.nim | 11 ++++------- compiler/sigmatch.nim | 1 + doc/manual.txt | 2 +- tests/generics/tgenericrefs.nim | 12 ++++++++++++ 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 6c76795780..a8a16672d6 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1819,6 +1819,10 @@ proc semExport(c: PContext, n: PNode): PNode = c.module.ast.add x result = n +proc setGenericParams(c: PContext, n: PNode) = + for i in 1 .. Date: Sun, 9 Feb 2014 00:58:18 +0100 Subject: [PATCH 69/72] fixes #882; fixes #853 --- compiler/semthreads.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semthreads.nim b/compiler/semthreads.nim index f7322db80d..d3426ca3e4 100644 --- a/compiler/semthreads.nim +++ b/compiler/semthreads.nim @@ -358,7 +358,7 @@ proc analyse(c: PProcCtx, n: PNode): TThreadOwner = of nkConstSection: result = analyseConstSection(c, n) of nkTypeSection, nkCommentStmt: result = toVoid of nkIfStmt, nkWhileStmt, nkTryStmt, nkCaseStmt, nkStmtList, nkBlockStmt, - nkElifBranch, nkElse, nkExceptBranch, nkOfBranch: + nkElifBranch, nkElse, nkExceptBranch, nkOfBranch, nkFinally: for i in 0 .. Date: Sun, 9 Feb 2014 01:34:17 +0100 Subject: [PATCH 70/72] compiles on sparc again --- lib/system/gc.nim | 2 +- todo.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 820093b3e0..b08a6d214f 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -802,7 +802,7 @@ when defined(sparc): # For SPARC architecture. # Addresses decrease as the stack grows. while sp <= max: gcMark(gch, sp[]) - sp = cast[ppointer](cast[TAddress](sp) +% sizeof(pointer)) + sp = cast[PPointer](cast[TAddress](sp) +% sizeof(pointer)) elif defined(ELATE): {.error: "stack marking code is to be written for this architecture".} diff --git a/todo.txt b/todo.txt index 738e9b3fad..0962d172e5 100644 --- a/todo.txt +++ b/todo.txt @@ -4,7 +4,6 @@ version 0.9.4 - fix GC issues - fix macros\tstringinterp.nim - test and fix showoff -- test C source code generation - fix closures - test and fix exception handling - implement 'union' and 'bits' pragmas From e6a50307bb505304b37147417ba2a93586d59a51 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 9 Feb 2014 02:37:49 +0100 Subject: [PATCH 71/72] fixes #885 --- compiler/lambdalifting.nim | 53 +++++++++++++++++++++----------------- tests/iter/tanoniter1.nim | 4 +-- todo.txt | 7 ++--- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 00fa045562..ebb4aeac4f 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -637,6 +637,22 @@ proc outerProcSons(o: POuterContext, n: PNode) = let x = transformOuterProc(o, n.sons[i]) if x != nil: n.sons[i] = x +proc liftIterSym*(n: PNode): PNode = + # transforms (iter) to (let env = newClosure[iter](); (iter, env)) + let iter = n.sym + assert iter.kind == skIterator + + result = newNodeIT(nkStmtListExpr, n.info, n.typ) + + var env = copySym(getHiddenParam(iter)) + env.kind = skLet + var v = newNodeI(nkVarSection, n.info) + addVar(v, newSymNode(env)) + result.add(v) + # add 'new' statement: + result.add(newCall(getSysSym"internalNew", env)) + result.add makeClosure(iter, env, n.info) + proc transformOuterProc(o: POuterContext, n: PNode): PNode = if n == nil: return nil case n.kind @@ -649,17 +665,22 @@ proc transformOuterProc(o: POuterContext, n: PNode): PNode = return indirectAccess(newSymNode(o.closureParam), local, n.info) var closure = PEnv(idTableGet(o.lambdasToEnv, local)) - if closure != nil: - # we need to replace the lambda with '(lambda, env)': - if local.kind == skIterator and local.typ.callConv == ccClosure: - # consider: [i1, i2, i1] Since we merged the iterator's closure - # with the captured owning variables, we need to generate the - # closure generation code again: - #if local == o.fn: message(n.info, errRecursiveDependencyX, local.name.s) - # XXX why doesn't this work? + + if local.kind == skIterator and local.typ.callConv == ccClosure: + # consider: [i1, i2, i1] Since we merged the iterator's closure + # with the captured owning variables, we need to generate the + # closure generation code again: + if local == o.fn: message(n.info, errRecursiveDependencyX, local.name.s) + # XXX why doesn't this work? + if closure.isNil: + return liftIterSym(n) + else: let createdVar = generateIterClosureCreation(o, closure, closure.attachedNode) return makeClosure(local, createdVar, n.info) + + if closure != nil: + # we need to replace the lambda with '(lambda, env)': let a = closure.createdVar if a != nil: @@ -773,22 +794,6 @@ proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode = # ------------------- iterator transformation -------------------------------- -proc liftIterSym*(n: PNode): PNode = - # transforms (iter) to (let env = newClosure[iter](); (iter, env)) - let iter = n.sym - assert iter.kind == skIterator - - result = newNodeIT(nkStmtListExpr, n.info, n.typ) - - var env = copySym(getHiddenParam(iter)) - env.kind = skLet - var v = newNodeI(nkVarSection, n.info) - addVar(v, newSymNode(env)) - result.add(v) - # add 'new' statement: - result.add(newCall(getSysSym"internalNew", env)) - result.add makeClosure(iter, env, n.info) - proc liftForLoop*(body: PNode): PNode = # problem ahead: the iterator could be invoked indirectly, but then # we don't know what environment to create here: diff --git a/tests/iter/tanoniter1.nim b/tests/iter/tanoniter1.nim index 9db5ab8eca..578749caf6 100644 --- a/tests/iter/tanoniter1.nim +++ b/tests/iter/tanoniter1.nim @@ -22,11 +22,11 @@ proc factory2(a, b: int): iterator (): int = yield x inc x -let foo = factory 1, 4 +let foo = factory(1, 4) for f in foo(): echo f -let foo2 = factory2 1,2 +let foo2 = factory2(1,2) for f in foo2(): echo f diff --git a/todo.txt b/todo.txt index 0962d172e5..44aa397910 100644 --- a/todo.txt +++ b/todo.txt @@ -4,9 +4,7 @@ version 0.9.4 - fix GC issues - fix macros\tstringinterp.nim - test and fix showoff -- fix closures -- test and fix exception handling -- implement 'union' and 'bits' pragmas +- fix closure iterators Bugs @@ -27,6 +25,9 @@ Bugs version 0.9.x ============= +- implement 'union' and 'bits' pragmas +- fix closures +- test and fix exception handling - ensure (ref T)(a, b) works as a type conversion and type constructor - optimize 'genericReset'; 'newException' leads to code bloat - stack-less GC From e478374e0d16cf42051e753ccdd364aec13cf7d7 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 9 Feb 2014 21:47:46 +0100 Subject: [PATCH 72/72] capturing of an iterator works better --- compiler/lambdalifting.nim | 31 ++++++++++++++++++++++--------- lib/windows/winlean.nim | 4 ++-- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index ebb4aeac4f..3738f89b2d 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -116,7 +116,8 @@ type TDep = tuple[e: PEnv, field: PSym] TEnv {.final.} = object of TObject attachedNode: PNode - createdVar: PSym # if != nil it is a used environment + createdVar: PSym # if != nil it is a used environment + createdVarComesFromIter: bool capturedVars: seq[PSym] # captured variables in this environment deps: seq[TDep] # dependencies up: PEnv @@ -571,7 +572,14 @@ proc rawClosureCreation(o: POuterContext, scope: PEnv; env: PSym): PNode = # maybe later: (sfByCopy in local.flags) # add ``env.param = param`` result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info)) - idNodeTablePut(o.localsToAccess, local, fieldAccess) + # it can happen that we already captured 'local' in some other environment + # then we capture by copy for now. This is not entirely correct but better + # than nothing: + let existing = idNodeTableGet(o.localsToAccess, local) + if existing.isNil: + idNodeTablePut(o.localsToAccess, local, fieldAccess) + else: + result.add(newAsgnStmt(fieldAccess, existing, env.info)) # add support for 'up' references: for e, field in items(scope.deps): # add ``env.up = env2`` @@ -584,14 +592,19 @@ proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode = proc generateIterClosureCreation(o: POuterContext; env: PEnv; scope: PNode): PSym = - result = newClosureCreationVar(o, env) - let cc = rawClosureCreation(o, env, result) - var insertPoint = scope.sons[0] - if insertPoint.kind == nkEmpty: scope.sons[0] = cc + if env.createdVarComesFromIter or env.createdVar.isNil: + # we have to create a new closure: + result = newClosureCreationVar(o, env) + let cc = rawClosureCreation(o, env, result) + var insertPoint = scope.sons[0] + if insertPoint.kind == nkEmpty: scope.sons[0] = cc + else: + assert cc.kind == nkStmtList and insertPoint.kind == nkStmtList + for x in cc: insertPoint.add(x) + if env.createdVar == nil: env.createdVar = result else: - assert cc.kind == nkStmtList and insertPoint.kind == nkStmtList - for x in cc: insertPoint.add(x) - if env.createdVar == nil: env.createdVar = result + result = env.createdVar + env.createdVarComesFromIter = true proc interestingIterVar(s: PSym): bool {.inline.} = result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 7713d503aa..6c8fa48823 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -689,8 +689,8 @@ var WSAID_GETACCEPTEXSOCKADDRS*: TGUID = TGUID(D1: 0xb5367df2'i32, D2: 0xcbac'i16, D3: 0x11cf, D4: [ 0x95'i8, 0xca'i8, 0x00'i8, 0x80'i8, 0x5f'i8, 0x48'i8, 0xa1'i8, 0x92'i8]) -proc WSAIoctl*(s: TSocketHandle, dwIoControlCode: dword, lpvInBuffer: pointer, - cbInBuffer: dword, lpvOutBuffer: pointer, cbOutBuffer: dword, +proc WSAIoctl*(s: TSocketHandle, dwIoControlCode: DWORD, lpvInBuffer: pointer, + cbInBuffer: DWORD, lpvOutBuffer: pointer, cbOutBuffer: DWORD, lpcbBytesReturned: PDword, lpOverlapped: POVERLAPPED, lpCompletionRoutine: POVERLAPPED_COMPLETION_ROUTINE): cint {.stdcall, importc: "WSAIoctl", dynlib: "Ws2_32.dll".}