From def3e015c7a8c82430fff20994b75b233c0c423b Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Mon, 27 Jun 2016 17:44:05 +0300 Subject: [PATCH 01/48] Added closureScope template --- lib/system.nim | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index 5a84f4a522..6a4265e5ab 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3633,6 +3633,27 @@ proc `==` *(x, y: cstring): bool {.magic: "EqCString", noSideEffect, elif x.isNil or y.isNil: result = false else: result = strcmp(x, y) == 0 +template closureScope*(body: untyped): stmt = + ## Useful when creating a closure in a loop to capture local loop variables by + ## their current iteration values. Example: + ## + ## .. code-block:: nim + ## var myClosure : proc() + ## # without closureScope: + ## for i in 0 .. 5: + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 5. `j` is changed after closure creation + ## # with closureScope: + ## for i in 0 .. 5: + ## closureScope: # Everything in this scope is locked after closure creation + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 3 + (proc() = body)() + {.pop.} #{.push warning[GcMem]: off, warning[Uninit]: off.} when defined(nimconfig): From ecfcf49a9d57c40f32eb019368b32828b86a041b Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 28 Jun 2016 13:07:19 +0300 Subject: [PATCH 02/48] Added a note on closureScope. Added Kyiv :) --- doc/manual/procs.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/manual/procs.txt b/doc/manual/procs.txt index dbd5932869..ea6866845b 100644 --- a/doc/manual/procs.txt +++ b/doc/manual/procs.txt @@ -215,6 +215,12 @@ the closure and its enclosing scope (i.e. any modifications made to them are visible in both places). The closure environment may be allocated on the heap or on the stack if the compiler determines that this would be safe. +Creating closures in loops +~~~~~~~~~~~~~~~~ + +Since closures capture local variables by reference it is often not wanted +behavior inside loop bodies. See `closureScope `_ +for details on how to change this behavior. Anonymous Procs --------------- @@ -223,7 +229,7 @@ Procs can also be treated as expressions, in which case it's allowed to omit the proc's name. .. code-block:: nim - var cities = @["Frankfurt", "Tokyo", "New York"] + var cities = @["Frankfurt", "Tokyo", "New York", "Kyiv"] cities.sort(proc (x,y: string): int = cmp(x.len, y.len)) From e61cfea78b9ebb8df734c7b9d15d81b50ce76613 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 28 Jun 2016 13:13:37 +0300 Subject: [PATCH 03/48] Fixed broken test. Added closureScope test. --- tests/assert/tunittests.nim | 3 +++ tests/closure/uclosures.nim | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/assert/tunittests.nim b/tests/assert/tunittests.nim index cbbfe63c60..de917511c7 100644 --- a/tests/assert/tunittests.nim +++ b/tests/assert/tunittests.nim @@ -1 +1,4 @@ +discard """ +output: "" +""" import "../template/utemplates", "../closure/uclosures" diff --git a/tests/closure/uclosures.nim b/tests/closure/uclosures.nim index 817bfec6b8..f259cfeb9c 100644 --- a/tests/closure/uclosures.nim +++ b/tests/closure/uclosures.nim @@ -1,12 +1,23 @@ +# This test is included from within tunittests import unittest -test "loop variables are captured by copy": +test "loop variables are captured by ref": var funcs: seq[proc (): int {.closure.}] = @[] for i in 0..10: let ii = i funcs.add do -> int: return ii * ii + check funcs[0]() == 100 + check funcs[3]() == 100 + +test "loop variables in closureScope are captured by copy": + var funcs: seq[proc (): int {.closure.}] = @[] + + for i in 0..10: + closureScope: + let ii = i + funcs.add do -> int: return ii * ii + check funcs[0]() == 0 check funcs[3]() == 9 - From 76f81d4aa4fa79de19019b77d0ca972b68e7be6e Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Thu, 16 Jun 2016 13:40:56 -0600 Subject: [PATCH 04/48] Fix #4305: Make split proc for set[char] consistent --- lib/pure/strutils.nim | 133 ++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 70 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 7d1b1a3d9e..623ab3199f 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -341,17 +341,49 @@ proc isNilOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl, extern: " if not c.isSpace(): return false +proc substrEq(s: string, pos: int, substr: string): bool = + var i = 0 + var length = substr.len + while i < length and s[pos+i] == substr[i]: + inc i + + return i == length + +# --------- Private templates for different split separators ----------- + +template stringHasSep(s: string, index: int, seps: set[char]): bool = + s[index] in seps + +template stringHasSep(s: string, index: int, sep: char): bool = + s[index] == sep + +template stringHasSep(s: string, index: int, sep: string): bool = + s.substrEq(index, sep) + +template splitCommon(s, sep, maxsplit, sepLen) = + ## Common code for split procedures + var last = 0 + var splits = maxsplit + + if len(s) > 0: + while last <= len(s): + var first = last + while last < len(s) and not stringHasSep(s, last, sep): + inc(last) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) + iterator split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a group of separators. ## - ## Substrings are separated by a substring containing only `seps`. Note - ## that whole sequences of characters found in ``seps`` will be counted as - ## a single split point and leading/trailing separators will be ignored. - ## The following example: + ## Substrings are separated by a substring containing only `seps`. ## ## .. code-block:: nim - ## for word in split(" this is an example "): + ## for word in split("this\lis an\texample"): ## writeLine(stdout, word) ## ## ...generates this output: @@ -365,7 +397,7 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## And the following code: ## ## .. code-block:: nim - ## for word in split(";;this;is;an;;example;;;", {';'}): + ## for word in split("this:is;an$example", {';', ':', '$'}): ## writeLine(stdout, word) ## ## ...produces the same output as the first example. The code: @@ -386,26 +418,13 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## "08" ## "08.398990" ## - var last = 0 - var splits = maxsplit - assert(not ('\0' in seps)) - while last < len(s): - while s[last] in seps: inc(last) - var first = last - while last < len(s) and s[last] notin seps: inc(last) # BUGFIX! - if first <= last-1: - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) + splitCommon(s, seps, maxsplit, 1) iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a single separator. ## ## Substrings are separated by the character `sep`. - ## Unlike the version of the iterator which accepts a set of separator - ## characters, this proc will not coalesce groups of the - ## separator, returning a string for each found character. The code: + ## The code: ## ## .. code-block:: nim ## for word in split(";;this;is;an;;example;;;", ';'): @@ -425,56 +444,27 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## "" ## "" ## - var last = 0 - var splits = maxsplit - assert('\0' != sep) - if len(s) > 0: - # `<=` is correct here for the edge cases! - while last <= len(s): - var first = last - while last < len(s) and s[last] != sep: inc(last) - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) - inc(last) - -proc substrEq(s: string, pos: int, substr: string): bool = - var i = 0 - var length = substr.len - while i < length and s[pos+i] == substr[i]: - inc i - - return i == length + splitCommon(s, sep, maxsplit, 1) iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. - var last = 0 - var splits = maxsplit + ## The code: + ## + ## .. code-block:: nim + ## for word in split("thisDATAisDATAcorrupted", "DATA"): + ## writeLine(stdout, word) + ## + ## Results in: + ## + ## .. code-block:: + ## "this" + ## "is" + ## "corrupted" + ## - if len(s) > 0: - while last <= len(s): - var first = last - while last < len(s) and not s.substrEq(last, sep): - inc(last) - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) - inc(last, sep.len) - -# --------- Private templates for different rsplit separators ----------- - -template stringHasSep(s: string, index: int, seps: set[char]): bool = - s[index] in seps - -template stringHasSep(s: string, index: int, sep: char): bool = - s[index] == sep - -template stringHasSep(s: string, index: int, sep: string): bool = - s.substrEq(index, sep) + splitCommon(s, sep, maxsplit, sep.len) template rsplitCommon(s, sep, maxsplit, sepLen) = ## Common code for rsplit functions @@ -2244,11 +2234,14 @@ bar bar """.unindent() == "foo\nfoo\nbar\n" - let s = " this is an example " - doAssert s.split() == @["this", "is", "an", "example"] - doAssert s.split(maxsplit=4) == @["this", "is", "an", "example"] - doAssert s.split(' ', maxsplit=4) == @["", "this", "", "", "is an example "] - doAssert s.split(" ", maxsplit=4) == @["", "this", "", "", "is an example "] + let s = " this is an example " + let s2 = ":this;is;an:example;;" + + doAssert s.split() == @["", "this", "is", "an", "example", "", ""] + doAssert s2.split(seps={':', ';'}) == @["", "this", "is", "an", "example", "", ""] + doAssert s.split(maxsplit=4) == @["", "this", "is", "an", "example "] + doAssert s.split(' ', maxsplit=1) == @["", "this is an example "] + doAssert s.split(" ", maxsplit=4) == @["", "this", "is", "an", "example "] block: # formatEng tests doAssert formatEng(0, 2, trim=false) == "0.00" From b80f12533300b1e5382ea6cdf300fa52543a213d Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Fri, 17 Jun 2016 11:33:38 -0600 Subject: [PATCH 05/48] Add new split to breaking changes doc --- web/news/version_0_15_released.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/web/news/version_0_15_released.rst b/web/news/version_0_15_released.rst index ecda59fcda..e3678d8dca 100644 --- a/web/news/version_0_15_released.rst +++ b/web/news/version_0_15_released.rst @@ -13,6 +13,12 @@ Changes affecting backwards compatibility - De-deprecated ``re.nim`` because we have too much code using it and it got the basic API right. +- ``split`` with ``set[char]`` as a delimiter in ``strutils.nim`` + no longer strips and splits characters out of the target string + by the entire set of characters. Instead, it now behaves in a + similar fashion to ``split`` with ``string`` and ``char`` + delimiters. + Library Additions ----------------- From 890d7fac14914c9749d707be37e082af1afa9e80 Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Sat, 18 Jun 2016 06:21:56 -0600 Subject: [PATCH 06/48] Fix split stdlib test --- tests/stdlib/tsplit.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/stdlib/tsplit.nim b/tests/stdlib/tsplit.nim index 5a1cd2f5fa..44da58acaa 100644 --- a/tests/stdlib/tsplit.nim +++ b/tests/stdlib/tsplit.nim @@ -9,7 +9,7 @@ for w in split("|abc|xy|z", {'|'}): s.add("#") s.add(w) -if s == "#abc#xy#z": +if s == "##abc#xy#z": echo "true" else: echo "false" From 79a8a5ee72b7aca09e1dfdb03744020ebd4dae2b Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Sat, 18 Jun 2016 13:58:14 -0600 Subject: [PATCH 07/48] Add transition define for old split behavior --- lib/pure/strutils.nim | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 623ab3199f..15bef15786 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -26,6 +26,12 @@ include "system/inclrtl" {.pop.} +# Support old split with set[char] +when defined(nimOldSplit): + {.pragma: deprecatedSplit, deprecated.} +else: + {.pragma: deprecatedSplit.} + type CharSet* {.deprecated.} = set[char] # for compatibility with Nim {.deprecated: [TCharSet: CharSet].} @@ -376,6 +382,22 @@ template splitCommon(s, sep, maxsplit, sepLen) = dec(splits) inc(last, sepLen) +when defined(nimOldSplit): + template oldSplit(s, seps, maxsplit) = + ## Deprecated split[char] for transition period + var last = 0 + var splits = maxsplit + assert(not ('\0' in seps)) + while last < len(s): + while s[last] in seps: inc(last) + var first = last + while last < len(s) and s[last] notin seps: inc(last) # BUGFIX! + if first <= last-1: + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + iterator split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a group of separators. @@ -418,7 +440,10 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## "08" ## "08.398990" ## - splitCommon(s, seps, maxsplit, 1) + when defined(nimOldSplit): + oldSplit(s, seps, maxsplit) + else: + splitCommon(s, seps, maxsplit, 1) iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a single separator. From e0203a44630d834224879187a44739eaeb122a81 Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Tue, 21 Jun 2016 15:06:01 -0600 Subject: [PATCH 08/48] Add useful unicode procs for string manipulation Added: isUpper, isLower, isAlpha, isWhiteSpace, toUpper, toLower, and capitalize Renamed strutils procs that are similar to avoid conflicts --- lib/pure/strutils.nim | 257 +++++++++++++++++++++++++++++++----------- lib/pure/unicode.nim | 134 ++++++++++++++++++++++ 2 files changed, 328 insertions(+), 63 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 7d1b1a3d9e..38807b29a6 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -64,8 +64,8 @@ const ## doAssert "01234".find(invalid) == -1 ## doAssert "01A34".find(invalid) == 2 -proc isAlpha*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaChar".}= +proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaAsciiChar".}= ## Checks whether or not `c` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -85,27 +85,27 @@ proc isDigit*(c: char): bool {.noSideEffect, procvar, ## This checks 0-9 ASCII characters only. return c in Digits -proc isSpace*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceChar".}= +proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsSpaceAsciiChar".}= ## Checks whether or not `c` is a whitespace character. return c in Whitespace -proc isLower*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerChar".}= +proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsLowerAsciiChar".}= ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. return c in {'a'..'z'} -proc isUpper*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperChar".}= +proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsUpperAsciiChar".}= ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. return c in {'A'..'Z'} -proc isAlpha*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaStr".}= +proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaAsciiStr".}= ## Checks whether or not `s` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -117,7 +117,7 @@ proc isAlpha*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isAlpha() and result + result = c.isAlphaAscii() and result proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsAlphaNumericStr".}= @@ -149,8 +149,8 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar, for c in s: result = c.isDigit() and result -proc isSpace*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceStr".}= +proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsSpaceAsciiStr".}= ## Checks whether or not `s` is completely whitespace. ## ## Returns true if all characters in `s` are whitespace @@ -160,11 +160,11 @@ proc isSpace*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - if not c.isSpace(): + if not c.isSpaceAscii(): return false -proc isLower*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerStr".}= +proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsLowerAsciiStr".}= ## Checks whether or not `s` contains all lower case characters. ## ## This checks ASCII characters only. @@ -175,10 +175,10 @@ proc isLower*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isLower() and result + result = c.isLowerAscii() and result -proc isUpper*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperStr".}= +proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsUpperAsciiStr".}= ## Checks whether or not `s` contains all upper case characters. ## ## This checks ASCII characters only. @@ -189,10 +189,10 @@ proc isUpper*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isUpper() and result + result = c.isUpperAscii() and result -proc toLower*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToLowerChar".} = +proc toLowerAscii*(c: char): char {.noSideEffect, procvar, + rtl, extern: "nsuToLowerAsciiChar".} = ## Converts `c` into lower case. ## ## This works only for the letters ``A-Z``. See `unicode.toLower @@ -203,8 +203,8 @@ proc toLower*(c: char): char {.noSideEffect, procvar, else: result = c -proc toLower*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToLowerStr".} = +proc toLowerAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuToLowerAsciiStr".} = ## Converts `s` into lower case. ## ## This works only for the letters ``A-Z``. See `unicode.toLower @@ -212,10 +212,10 @@ proc toLower*(s: string): string {.noSideEffect, procvar, ## character. result = newString(len(s)) for i in 0..len(s) - 1: - result[i] = toLower(s[i]) + result[i] = toLowerAscii(s[i]) -proc toUpper*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToUpperChar".} = +proc toUpperAscii*(c: char): char {.noSideEffect, procvar, + rtl, extern: "nsuToUpperAsciiChar".} = ## Converts `c` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper @@ -226,8 +226,8 @@ proc toUpper*(c: char): char {.noSideEffect, procvar, else: result = c -proc toUpper*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToUpperStr".} = +proc toUpperAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuToUpperAsciiStr".} = ## Converts `s` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper @@ -235,14 +235,145 @@ proc toUpper*(s: string): string {.noSideEffect, procvar, ## character. result = newString(len(s)) for i in 0..len(s) - 1: - result[i] = toUpper(s[i]) + result[i] = toUpperAscii(s[i]) -proc capitalize*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuCapitalize".} = +proc capitalizeAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuCapitalizeAscii".} = ## Converts the first character of `s` into upper case. ## ## This works only for the letters ``A-Z``. - result = toUpper(s[0]) & substr(s, 1) + result = toUpperAscii(s[0]) & substr(s, 1) + +proc isSpace*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsSpaceChar".}= + ## Checks whether or not `c` is a whitespace character. + ## + ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. + isSpaceAscii(c) + +proc isLower*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsLowerChar".}= + ## Checks whether or not `c` is a lower case character. + ## + ## This checks ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. + isLowerAscii(c) + +proc isUpper*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsUpperChar".}= + ## Checks whether or not `c` is an upper case character. + ## + ## This checks ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. + isUpperAscii(c) + +proc isAlpha*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsAlphaChar".}= + ## Checks whether or not `c` is alphabetical. + ## + ## This checks a-z, A-Z ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. + isAlphaAscii(c) + +proc isAlpha*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsAlphaStr".}= + ## Checks whether or not `s` is alphabetical. + ## + ## This checks a-z, A-Z ASCII characters only. + ## Returns true if all characters in `s` are + ## alphabetic and there is at least one character + ## in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. + isAlphaAscii(s) + +proc isSpace*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsSpaceStr".}= + ## Checks whether or not `s` is completely whitespace. + ## + ## Returns true if all characters in `s` are whitespace + ## characters and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. + isSpaceAscii(s) + +proc isLower*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsLowerStr".}= + ## Checks whether or not `s` contains all lower case characters. + ## + ## This checks ASCII characters only. + ## Returns true if all characters in `s` are lower case + ## and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. + isLowerAscii(s) + +proc isUpper*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsUpperStr".}= + ## Checks whether or not `s` contains all upper case characters. + ## + ## This checks ASCII characters only. + ## Returns true if all characters in `s` are upper case + ## and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. + isUpperAscii(s) + +proc toLower*(c: char): char {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToLowerChar".} = + ## Converts `c` into lower case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toLower + ## `_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. + toLowerAscii(c) + +proc toLower*(s: string): string {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToLowerStr".} = + ## Converts `s` into lower case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toLower + ## `_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. + toLowerAscii(s) + +proc toUpper*(c: char): char {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToUpperChar".} = + ## Converts `c` into upper case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## `_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. + toUpperAscii(c) + +proc toUpper*(s: string): string {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToUpperStr".} = + ## Converts `s` into upper case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## `_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. + toUpperAscii(s) + +proc capitalize*(s: string): string {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuCapitalize".} = + ## Converts the first character of `s` into upper case. + ## + ## This works only for the letters ``A-Z``. + ## + ## **Deprecated since version 0.15.0**: use ``capitalizeAscii`` instead. + capitalizeAscii(s) proc normalize*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuNormalize".} = @@ -271,7 +402,7 @@ proc cmpIgnoreCase*(a, b: string): int {.noSideEffect, var i = 0 var m = min(a.len, b.len) while i < m: - result = ord(toLower(a[i])) - ord(toLower(b[i])) + result = ord(toLowerAscii(a[i])) - ord(toLowerAscii(b[i])) if result != 0: return inc(i) result = a.len - b.len @@ -292,8 +423,8 @@ proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, while true: while a[i] == '_': inc(i) while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[j]) result = ord(aa) - ord(bb) if result != 0 or aa == '\0': break inc(i) @@ -2138,13 +2269,13 @@ when isMainModule: doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" - doAssert isAlpha('r') - doAssert isAlpha('A') - doAssert(not isAlpha('$')) + doAssert isAlphaAscii('r') + doAssert isAlphaAscii('A') + doAssert(not isAlphaAscii('$')) - doAssert isAlpha("Rasp") - doAssert isAlpha("Args") - doAssert(not isAlpha("$Tomato")) + doAssert isAlphaAscii("Rasp") + doAssert isAlphaAscii("Args") + doAssert(not isAlphaAscii("$Tomato")) doAssert isAlphaNumeric('3') doAssert isAlphaNumeric('R') @@ -2163,13 +2294,13 @@ when isMainModule: doAssert(not isDigit("12.33")) doAssert(not isDigit("A45b")) - doAssert isSpace('\t') - doAssert isSpace('\l') - doAssert(not isSpace('A')) + doAssert isSpaceAscii('\t') + doAssert isSpaceAscii('\l') + doAssert(not isSpaceAscii('A')) - doAssert isSpace("\t\l \v\r\f") - doAssert isSpace(" ") - doAssert(not isSpace("ABc \td")) + doAssert isSpaceAscii("\t\l \v\r\f") + doAssert isSpaceAscii(" ") + doAssert(not isSpaceAscii("ABc \td")) doAssert(isNilOrEmpty("")) doAssert(isNilOrEmpty(nil)) @@ -2182,24 +2313,24 @@ when isMainModule: doAssert(isNilOrWhitespace("\t\l \v\r\f")) doAssert(not isNilOrWhitespace("ABc \td")) - doAssert isLower('a') - doAssert isLower('z') - doAssert(not isLower('A')) - doAssert(not isLower('5')) - doAssert(not isLower('&')) + doAssert isLowerAscii('a') + doAssert isLowerAscii('z') + doAssert(not isLowerAscii('A')) + doAssert(not isLowerAscii('5')) + doAssert(not isLowerAscii('&')) - doAssert isLower("abcd") - doAssert(not isLower("abCD")) - doAssert(not isLower("33aa")) + doAssert isLowerAscii("abcd") + doAssert(not isLowerAscii("abCD")) + doAssert(not isLowerAscii("33aa")) - doAssert isUpper('A') - doAssert(not isUpper('b')) - doAssert(not isUpper('5')) - doAssert(not isUpper('%')) + doAssert isUpperAscii('A') + doAssert(not isUpperAscii('b')) + doAssert(not isUpperAscii('5')) + doAssert(not isUpperAscii('%')) - doAssert isUpper("ABC") - doAssert(not isUpper("AAcc")) - doAssert(not isUpper("A#$")) + doAssert isUpperAscii("ABC") + doAssert(not isUpperAscii("AAcc")) + doAssert(not isUpperAscii("A#$")) doAssert rsplit("foo bar", seps=Whitespace) == @["foo", "bar"] doAssert rsplit(" foo bar", seps=Whitespace, maxsplit=1) == @[" foo", "bar"] diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index ac25dccef5..b5383c5d3a 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -1369,6 +1369,64 @@ proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = (c >= 0x20d0 and c <= 0x20ff) or (c >= 0xfe20 and c <= 0xfe2f)) +template runeCheck(s, runeProc) = + ## Common code for rune.isLower, rune.isUpper, etc + result = if len(s) == 0: false else: true + + var + i = 0 + rune: Rune + + while i < len(s) and result: + fastRuneAt(s, i, rune, doInc=true) + result = runeProc(rune) and result + +proc isUpper*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all upper case unicode characters. + runeCheck(s, isUpper) + +proc isLower*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all lower case unicode characters. + runeCheck(s, isLower) + +proc isAlpha*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all alphabetic unicode characters. + runeCheck(s, isAlpha) + +proc isSpace*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all whitespace unicode characters. + runeCheck(s, isWhiteSpace) + +template convertRune(s, runeProc) = + ## Convert runes in `s` using `runeProc` as the converter. + result = newString(len(s)) + + var + i = 0 + lastIndex = 0 + rune: Rune + + while i < len(s): + lastIndex = i + fastRuneAt(s, i, rune, doInc=true) + rune = runeProc(rune) + + rune.fastToUTF8Copy(result, lastIndex) + +proc toUpper*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Converts `s` into upper-case unicode characters. + convertRune(s, toUpper) + +proc toLower*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Converts `s` into lower-case unicode characters. + convertRune(s, toLower) + proc swapCase*(s: string): string {.noSideEffect, procvar, rtl, extern: "nuc$1".} = ## Swaps the case of unicode characters in `s` @@ -1395,6 +1453,20 @@ proc swapCase*(s: string): string {.noSideEffect, procvar, rune.fastToUTF8Copy(result, lastIndex) +proc capitalize*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1".} = + ## Converts the first character of `s` into an upper-case unicode character. + if len(s) == 0: + return s + + var + rune: Rune + i = 0 + + fastRuneAt(s, i, rune, doInc=true) + + result = $toUpper(rune) & substr(s, i) + proc translate*(s: string, replacements: proc(key: string): string): string {. rtl, extern: "nuc$1".} = ## Translates words in a string using the `replacements` proc to substitute @@ -1618,6 +1690,10 @@ when isMainModule: doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" doAssert title("") == "" + doAssert capitalize("βeta") == "Βeta" + doAssert capitalize("foo") == "Foo" + doAssert capitalize("") == "" + doAssert isTitle("Foo") doAssert(not isTitle("Foo bar")) doAssert(not isTitle("αlpha Βeta")) @@ -1630,6 +1706,64 @@ when isMainModule: doAssert swapCase("a✓B") == "A✓b" doAssert swapCase("") == "" + doAssert isAlpha("r") + doAssert isAlpha("α") + doAssert(not isAlpha("$")) + doAssert(not isAlpha("")) + + doAssert isAlpha("Βeta") + doAssert isAlpha("Args") + doAssert(not isAlpha("$Foo✓")) + + doAssert isSpace("\t") + doAssert isSpace("\l") + doAssert(not isSpace("Β")) + doAssert(not isSpace("Βeta")) + + doAssert isSpace("\t\l \v\r\f") + doAssert isSpace(" ") + doAssert(not isSpace("")) + doAssert(not isSpace("ΑΓc \td")) + + doAssert isLower("a") + doAssert isLower("γ") + doAssert(not isLower("Γ")) + doAssert(not isLower("4")) + doAssert(not isLower("")) + + doAssert isLower("abcdγ") + doAssert(not isLower("abCDΓ")) + doAssert(not isLower("33aaΓ")) + + doAssert isUpper("Γ") + doAssert(not isUpper("b")) + doAssert(not isUpper("α")) + doAssert(not isUpper("✓")) + doAssert(not isUpper("")) + + doAssert isUpper("ΑΒΓ") + doAssert(not isUpper("AAccβ")) + doAssert(not isUpper("A#$β")) + + doAssert toUpper("Γ") == "Γ" + doAssert toUpper("b") == "B" + doAssert toUpper("α") == "Α" + doAssert toUpper("✓") == "✓" + doAssert toUpper("") == "" + + doAssert toUpper("ΑΒΓ") == "ΑΒΓ" + doAssert toUpper("AAccβ") == "AACCΒ" + doAssert toUpper("A✓$β") == "A✓$Β" + + doAssert toLower("a") == "a" + doAssert toLower("γ") == "γ" + doAssert toLower("Γ") == "γ" + doAssert toLower("4") == "4" + doAssert toLower("") == "" + + doAssert toLower("abcdγ") == "abcdγ" + doAssert toLower("abCDΓ") == "abcdγ" + doAssert toLower("33aaΓ") == "33aaγ" doAssert reversed("Reverse this!") == "!siht esreveR" doAssert reversed("先秦兩漢") == "漢兩秦先" From b1ab82715e1921a0159fce73cc047ee48b9f75bf Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Tue, 21 Jun 2016 15:17:13 -0600 Subject: [PATCH 09/48] Fix modules that import both strutils and unicode This is only an issue when a proc in both modules that is named the same is used, such as toLower or toUpper for strings. --- lib/pure/htmlparser.nim | 6 +++--- lib/pure/pegs.nim | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index d620e816ee..fd58bed25d 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -433,7 +433,7 @@ proc htmlTag*(n: XmlNode): HtmlTag = proc htmlTag*(s: string): HtmlTag = ## converts `s` to a ``HtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is ## returned. - let s = if allLower(s): s else: s.toLower + let s = if allLower(s): s else: toLowerAscii(s) result = toHtmlTag(s) proc entityToUtf8*(entity: string): string = @@ -513,13 +513,13 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = errors.add(errorMsg(x)) next(x) of xmlElementStart: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) untilElementEnd(x, result, errors) of xmlElementEnd: errors.add(errorMsg(x, "unexpected ending tag: " & x.elemName)) of xmlElementOpen: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) result.attrs = newStringTable() while true: diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 7e1f50266d..5c978a2f8c 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1841,8 +1841,8 @@ when isMainModule: result.add ", " result.add case n: - of 2: c[0].toLower & ": '" & c[1] & "'" - of 1: c[0].toLower & ": ''" + of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" + of 1: toLowerAscii(c[0]) & ": ''" else: "" assert("Var1=key1;var2=Key2; VAR3". From ff85ef456ad3d538a6cef1aad477b69a8e98eb0e Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Tue, 28 Jun 2016 17:36:00 -0600 Subject: [PATCH 10/48] Add new unicode procs to the news --- web/news/version_0_15_released.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/news/version_0_15_released.rst b/web/news/version_0_15_released.rst index ecda59fcda..96d9abc997 100644 --- a/web/news/version_0_15_released.rst +++ b/web/news/version_0_15_released.rst @@ -23,8 +23,10 @@ Library Additions - Added ``center`` and ``rsplit`` to ``strutils.nim`` to provide similar Python functionality for Nim's strings. -- Added ``isTitle``, ``title``, and ``swapCase`` to ``unicode.nim`` to - provide unicode aware string case manipulation. +- Added ``isTitle``, ``title``, ``swapCase``, ``isUpper``, ``toUpper``, + ``isLower``, ``toLower``, ``isAlpha``, ``isSpace``, and ``capitalize`` + to ``unicode.nim`` to provide unicode aware case manipulation and case + testing. - Added a new module ``lib/pure/strmisc.nim`` to hold uncommon string operations. Currently contains ``partition``, ``rpartition`` From e4b16ac6086d0311d3bd74a20e9cfbfc0c19ca4f Mon Sep 17 00:00:00 2001 From: Vladislav Vorobiev Date: Fri, 1 Jul 2016 21:50:26 +0300 Subject: [PATCH 11/48] Remove line breaks in OSError messages (Windows) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before: ``` Error: unhandled exception: Не удается найти указанный файл. [OSError] ``` After: ``` Error: unhandled exception: Не удается найти указанный файл. [OSError] ``` --- lib/pure/os.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 52ba110a97..d543ec9a0a 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -66,13 +66,13 @@ proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = if err != 0'i32: when useWinUnicode: var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: result = $msgbuf if msgbuf != nil: localFree(cast[pointer](msgbuf)) else: var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, + if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: result = $msgbuf if msgbuf != nil: localFree(msgbuf) From f6fda5e704758decedf469c87c4f2a2a6ec12786 Mon Sep 17 00:00:00 2001 From: Chris Heller Date: Sat, 2 Jul 2016 17:19:03 -0600 Subject: [PATCH 12/48] Add doc to parsexml.nim for template assertions Added additional documentation for parsexml.nim to describe the assertions that are used to check the parser's current status when accessing parser data. --- lib/pure/parsexml.nim | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index 06daa37823..d16a553023 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -142,6 +142,9 @@ proc kind*(my: XmlParser): XmlEventKind {.inline.} = template charData*(my: XmlParser): string = ## returns the character data for the events: ``xmlCharData``, ## ``xmlWhitespace``, ``xmlComment``, ``xmlCData``, ``xmlSpecial`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlCharData, xmlWhitespace, xmlComment, xmlCData, xmlSpecial}) my.a @@ -149,31 +152,49 @@ template charData*(my: XmlParser): string = template elementName*(my: XmlParser): string = ## returns the element name for the events: ``xmlElementStart``, ## ``xmlElementEnd``, ``xmlElementOpen`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlElementStart, xmlElementEnd, xmlElementOpen}) my.a template entityName*(my: XmlParser): string = ## returns the entity name for the event: ``xmlEntity`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlEntity``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlEntity) my.a template attrKey*(my: XmlParser): string = ## returns the attribute key for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.a template attrValue*(my: XmlParser): string = ## returns the attribute value for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.b template piName*(my: XmlParser): string = ## returns the processing instruction name for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.a template piRest*(my: XmlParser): string = ## returns the rest of the processing instruction for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.b From b8ab419067207127bd05bbefb5a5332d792bca61 Mon Sep 17 00:00:00 2001 From: Denis Kolodin Date: Sat, 25 Jun 2016 08:35:37 +0300 Subject: [PATCH 13/48] Use relative paths in install script template --- tools/niminst/install.tmpl | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tools/niminst/install.tmpl b/tools/niminst/install.tmpl index f9266b8670..3f17840a89 100644 --- a/tools/niminst/install.tmpl +++ b/tools/niminst/install.tmpl @@ -3,7 +3,35 @@ # result = "#! /bin/sh\n# Generated by niminst\n" # var proj = c.name.toLower -set -e +## Current directory you start script from +BASE_DIR=$(pwd) + +## The following one-liner takes directory path which contains install script. +## `command -v -- "$0"` takes path if script sourced from interactive shell +## `dirname` returns relative directory path to install script +## `cd -P` dive into directory to use `pwd` +## `pwd -P` prints full path to install script directory path +## -P option allows to use symlinks in path +## Good explanation can be found here: +## http://stackoverflow.com/questions/29832037/how-to-get-script-directory-in-posix-sh +NIM_DIR=$(cd -P -- "$(dirname -- "$(command -v -- "$0")")" && pwd -P) + +go_back() { + cd $BASE_DIR +} + +## Go to base dir on exit +trap go_back EXIT + +install_error() { + echo "Nim installation failed!" + exit 1 +} + +## Exit if any command failed +trap install_error ERR ## `set -e` alternative + +cd $NIM_DIR if [ $# -eq 1 ] ; then # if c.cat[fcUnixBin].len > 0: From 9a7b6af5fab3dc418ca4e63e071f3764ec1b73de Mon Sep 17 00:00:00 2001 From: Matthew Baulch Date: Tue, 5 Jul 2016 06:14:36 +1000 Subject: [PATCH 14/48] Report wrong arg count when too many params and no varargs. --- compiler/ccgcalls.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index bd17f85e46..879b574235 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -260,6 +260,8 @@ proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope = else: result = genArgNoParam(p, ri.sons[i]) #, typ.n.sons[i].sym) else: + if tfVarargs notin typ.flags: + localError(ri.info, "wrong argument count") result = genArgNoParam(p, ri.sons[i]) discard """ From 94d1aa5109ebe022e6e51c7cd84033e2457b184f Mon Sep 17 00:00:00 2001 From: Jeff Ciesielski Date: Mon, 4 Jul 2016 18:11:25 -0400 Subject: [PATCH 15/48] Add the ability to pass a value with the -d flag This allows the end user to use the {.magic: "IntDefine"/"StrDefine"} pragmas to pass values into code at compile time. This has a nice side effect of also allowing/requiring a default value to be assigned in the code (see osalloc.nim/StandaloneHeapSize for an example) --- compiler/ast.nim | 2 +- compiler/commands.nim | 13 ++++++++++++- compiler/condsyms.nim | 9 +++++++-- compiler/semfold.nim | 6 ++++++ compiler/wordrecg.nim | 3 ++- doc/basicopt.txt | 3 ++- lib/system/osalloc.nim | 3 ++- 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index c3bb7cd622..34ffd6f583 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -610,7 +610,7 @@ type mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mNGenSym, - mNimvm + mNimvm, mIntDefine, mStrDefine # things that we can evaluate safely at compile time, even if not asked for it: const diff --git a/compiler/commands.nim b/compiler/commands.nim index b3edb5e149..640e07b1b7 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -122,6 +122,13 @@ proc splitSwitch(switch: string, cmd, arg: var string, pass: TCmdLinePass, elif switch[i] in {':', '=', '['}: arg = substr(switch, i + 1) else: invalidCmdLineOption(pass, switch, info) +proc hasKeyValuePair(arg: string): bool = + for i in 0..arg.high: + if arg[i] in {':', '='}: + return true + + return false + proc processOnOffSwitch(op: TOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = case whichKeyword(arg) @@ -342,7 +349,11 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = discard "allow for backwards compatibility, but don't do anything" of "define", "d": expectArg(switch, arg, pass, info) - defineSymbol(arg) + if hasKeyValuePair(arg): + splitSwitch(arg, key, val, pass, info) + defineSymbol(key, val) + else: + defineSymbol(arg) of "undef", "u": expectArg(switch, arg, pass, info) undefSymbol(arg) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 60e8f28262..02f7e764de 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -19,8 +19,8 @@ var gSymbols: StringTableRef const catNone = "false" -proc defineSymbol*(symbol: string) = - gSymbols[symbol] = "true" +proc defineSymbol*(symbol: string, value: string = "true") = + gSymbols[symbol] = value proc undefSymbol*(symbol: string) = gSymbols[symbol] = catNone @@ -62,6 +62,11 @@ proc isDefined*(symbol: string): bool = proc isDefined*(symbol: PIdent): bool = isDefined(symbol.s) +proc lookupSymbol*(symbol: string): string = + result = if isDefined(symbol): gSymbols[symbol] else: nil + +proc lookupSymbol*(symbol: PIdent): string = lookupSymbol(symbol.s) + iterator definedSymbolNames*: string = for key, val in pairs(gSymbols): if val != catNone: yield key diff --git a/compiler/semfold.nim b/compiler/semfold.nim index feea6ce347..26d309cfdb 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -640,6 +640,12 @@ proc getConstExpr(m: PSym, n: PNode): PNode = of mNaN: result = newFloatNodeT(NaN, n) of mInf: result = newFloatNodeT(Inf, n) of mNegInf: result = newFloatNodeT(NegInf, n) + of mIntDefine: + if isDefined(s.name): + result = newIntNodeT(lookupSymbol(s.name).parseInt, n) + of mStrDefine: + if isDefined(s.name): + result = newStrNodeT(lookupSymbol(s.name), n) else: if sfFakeConst notin s.flags: result = copyTree(s.ast) of {skProc, skMethod}: diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 3e0e05a948..b5ffd51c28 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -36,6 +36,7 @@ type wColon, wColonColon, wEquals, wDot, wDotDot, wStar, wMinus, wMagic, wThread, wFinal, wProfiler, wObjChecks, + wIntDefine, wStrDefine, wDestroy, @@ -121,7 +122,7 @@ const ":", "::", "=", ".", "..", "*", "-", - "magic", "thread", "final", "profiler", "objchecks", + "magic", "thread", "final", "profiler", "objchecks", "intdefine", "strdefine", "destroy", diff --git a/doc/basicopt.txt b/doc/basicopt.txt index 6a905bd538..c68d731e6c 100644 --- a/doc/basicopt.txt +++ b/doc/basicopt.txt @@ -11,7 +11,8 @@ Arguments: arguments are passed to the program being run (if --run option is selected) Options: -p, --path:PATH add path to search paths - -d, --define:SYMBOL define a conditional symbol + -d, --define:SYMBOL(:VAL) define a conditional symbol + (Optionally: Define the value for that symbol) -u, --undef:SYMBOL undefine a conditional symbol -f, --forceBuild force rebuilding of all modules --stackTrace:on|off turn stack tracing on|off diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index de26f52d96..cfa9ae18fb 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -150,8 +150,9 @@ elif defined(windows): #VirtualFree(p, size, MEM_DECOMMIT) elif hostOS == "standalone": + const StandaloneHeapSize {.magic: "IntDefine"}: int = 1024 * PageSize var - theHeap: array[1024*PageSize, float64] # 'float64' for alignment + theHeap: array[StandaloneHeapSize, float64] # 'float64' for alignment bumpPointer = cast[int](addr theHeap) proc osAllocPages(size: int): pointer {.inline.} = From 4f4aafda6c1cc13e1d8073e341344a49507e7174 Mon Sep 17 00:00:00 2001 From: Jeff Ciesielski Date: Mon, 4 Jul 2016 22:52:24 -0400 Subject: [PATCH 16/48] Plumb {.intdefine.} and {.strdefine.} pragmas. Shorthand so that users won't need to use the .magic pragma --- compiler/pragmas.nim | 7 ++++++- lib/system/osalloc.nim | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index dc618d9aa5..fffb69ca8d 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -63,7 +63,8 @@ const wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal, wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims} constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl, - wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims} + wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims, + wIntDefine, wStrDefine} letPragmas* = varPragmas procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNosideeffect, wThread, wRaises, wLocks, wTags, wGcSafe} @@ -898,6 +899,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wBase: noVal(it) sym.flags.incl sfBase + of wIntDefine: + sym.magic = mIntDefine + of wStrDefine: + sym.magic = mIntDefine else: invalidPragma(it) else: invalidPragma(it) diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index cfa9ae18fb..e864253b66 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -150,7 +150,7 @@ elif defined(windows): #VirtualFree(p, size, MEM_DECOMMIT) elif hostOS == "standalone": - const StandaloneHeapSize {.magic: "IntDefine"}: int = 1024 * PageSize + const StandaloneHeapSize {.intdefine.}: int = 1024 * PageSize var theHeap: array[StandaloneHeapSize, float64] # 'float64' for alignment bumpPointer = cast[int](addr theHeap) From 835ff4a2f8622cd3db7eece7d07edac51c4d10f3 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 5 Jul 2016 13:18:26 +0300 Subject: [PATCH 17/48] ioselectors separated and refactored version. adopted asyncdispatch version --- lib/pure/ioselectors.nim | 1764 --------------- lib/pure/ioselectors/ioselectors.nim | 261 +++ lib/pure/ioselectors/ioselectors_epoll.nim | 461 ++++ lib/pure/ioselectors/ioselectors_kqueue.nim | 439 ++++ lib/pure/ioselectors/ioselectors_poll.nim | 295 +++ lib/pure/ioselectors/ioselectors_select.nim | 416 ++++ lib/upcoming/asyncdispatch.nim | 2154 +++++++++++++++++++ tests/async/tioselectors.nim | 8 +- 8 files changed, 4030 insertions(+), 1768 deletions(-) delete mode 100644 lib/pure/ioselectors.nim create mode 100644 lib/pure/ioselectors/ioselectors.nim create mode 100644 lib/pure/ioselectors/ioselectors_epoll.nim create mode 100644 lib/pure/ioselectors/ioselectors_kqueue.nim create mode 100644 lib/pure/ioselectors/ioselectors_poll.nim create mode 100644 lib/pure/ioselectors/ioselectors_select.nim create mode 100644 lib/upcoming/asyncdispatch.nim diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim deleted file mode 100644 index 034b182ab6..0000000000 --- a/lib/pure/ioselectors.nim +++ /dev/null @@ -1,1764 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Eugene Kabanov -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module allows high-level and efficient I/O multiplexing. -## -## Supported OS primitives: ``epoll``, ``kqueue``, ``poll`` and -## Windows ``select``. -## -## To use threadsafe version of this module, it needs to be compiled -## with both ``-d:threadsafe`` and ``--threads:on`` options. -## -## Supported features: files, sockets, pipes, timers, processes, signals -## and user events. -## -## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux. -## -## Partially supported OS: Windows (only sockets and user events), -## Solaris (files, sockets, handles and user events). -## -## TODO: ``/dev/poll``, ``event ports`` and filesystem events. - -import os - -const hasThreadSupport = compileOption("threads") and defined(threadsafe) - -const supportedPlatform = defined(macosx) or defined(freebsd) or - defined(netbsd) or defined(openbsd) or - defined(linux) - -const bsdPlatform = defined(macosx) or defined(freebsd) or - defined(netbsd) or defined(openbsd) - -when defined(linux): - import posix, times -elif bsdPlatform: - import posix, kqueue, times -elif defined(windows): - import winlean -else: - import posix - -when defined(nimdoc): - type - Selector*[T] = ref object - ## An object which holds descriptors to be checked for read/write status - - Event* {.pure.} = enum - ## An enum which hold event types - Read, ## Descriptor is available for read - Write, ## Descriptor is available for write - Timer, ## Timer descriptor is completed - Signal, ## Signal is raised - Process, ## Process is finished - Vnode, ## Currently not supported - User, ## User event is raised - Error ## Error happens while waiting, for descriptor - - ReadyKey*[T] = object - ## An object which holds result for descriptor - fd* : int ## file/socket descriptor - events*: set[Event] ## set of events - data*: T ## application-defined data - - SelectEvent* = object - ## An object which holds user defined event - - proc newSelector*[T](): Selector[T] = - ## Creates a new selector - - proc close*[T](s: Selector[T]) = - ## Closes selector - - proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], - data: T) = - ## Registers file/socket descriptor ``fd`` to selector ``s`` - ## with events set in ``events``. The ``data`` is application-defined - ## data, which to be passed when event happens. - - proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = - ## Update file/socket descriptor ``fd``, registered in selector - ## ``s`` with new events set ``event``. - - proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, - data: T): int {.discardable.} = - ## Registers timer notification with ``timeout`` in milliseconds - ## to selector ``s``. - ## If ``oneshot`` is ``true`` timer will be notified only once. - ## Set ``oneshot`` to ``false`` if your want periodic notifications. - ## The ``data`` is application-defined data, which to be passed, when - ## time limit expired. - - proc registerSignal*[T](s: Selector[T], signal: int, - data: T): int {.discardable.} = - ## Registers Unix signal notification with ``signal`` to selector - ## ``s``. The ``data`` is application-defined data, which to be - ## passed, when signal raises. - ## - ## This function is not supported for ``Windows``. - - proc registerProcess*[T](s: Selector[T], pid: int, - data: T): int {.discardable.} = - ## Registers process id (pid) notification when process has - ## exited to selector ``s``. - ## The ``data`` is application-defined data, which to be passed, when - ## process with ``pid`` has exited. - - proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = - ## Registers selector event ``ev`` to selector ``s``. - ## ``data`` application-defined data, which to be passed, when - ## ``ev`` happens. - - proc newEvent*(): SelectEvent = - ## Creates new event ``SelectEvent``. - - proc setEvent*(ev: SelectEvent) = - ## Trigger event ``ev``. - - proc close*(ev: SelectEvent) = - ## Closes selector event ``ev``. - - proc unregister*[T](s: Selector[T], ev: SelectEvent) = - ## Unregisters event ``ev`` from selector ``s``. - - proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = - ## Unregisters file/socket descriptor ``fd`` from selector ``s``. - - proc flush*[T](s: Selector[T]) = - ## Flushes all changes was made to kernel pool/queue. - ## This function is usefull only for BSD and MacOS, because - ## kqueue supports bulk changes to be made. - ## On Linux/Windows and other Posix compatible operation systems, - ## ``flush`` is alias for `discard`. - - proc selectInto*[T](s: Selector[T], timeout: int, - results: var openarray[ReadyKey[T]]): int = - ## Process call waiting for events registered in selector ``s``. - ## The ``timeout`` argument specifies the minimum number of milliseconds - ## the function will be blocked, if no events are not ready. Specifying a - ## timeout of ``-1`` causes function to block indefinitely. - ## All available events will be stored in ``results`` array. - ## - ## Function returns number of triggered events. - - proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = - ## Process call waiting for events registered in selector ``s``. - ## The ``timeout`` argument specifies the minimum number of milliseconds - ## the function will be blocked, if no events are not ready. Specifying a - ## timeout of -1 causes function to block indefinitely. - ## - ## Function returns sequence of triggered events. - - template isEmpty*[T](s: Selector[T]): bool = - ## Returns ``true``, if there no registered events or descriptors - ## in selector. - - template withData*[T](s: Selector[T], fd: SocketHandle, value, - body: untyped) = - ## retrieves the application-data assigned with descriptor ``fd`` - ## to ``value``. This ``value`` can be modified in the scope of - ## the ``withData`` call. - ## - ## .. code-block:: nim - ## - ## s.withData(fd, value) do: - ## # block is executed only if ``fd`` registered in selector ``s`` - ## value.uid = 1000 - ## - - template withData*[T](s: Selector[T], fd: SocketHandle, value, - body1, body2: untyped) = - ## retrieves the application-data assigned with descriptor ``fd`` - ## to ``value``. This ``value`` can be modified in the scope of - ## the ``withData`` call. - ## - ## .. code-block:: nim - ## - ## s.withData(fd, value) do: - ## # block is executed only if ``fd`` registered in selector ``s``. - ## value.uid = 1000 - ## do: - ## # block is executed if ``fd`` not registered in selector ``s``. - ## raise - ## - -else: - when defined(macosx) or defined(freebsd): - when defined(macosx): - const maxDescriptors = 29 # KERN_MAXFILESPERPROC (MacOS) - else: - const maxDescriptors = 27 # KERN_MAXFILESPERPROC (FreeBSD) - proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, - newp: pointer, newplen: int): cint - {.importc: "sysctl",header: """#include - #include """} - elif defined(netbsd) or defined(openbsd): - # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using - # KERN_MAXFILES, because KERN_MAXFILES is always bigger, - # than KERN_MAXFILESPERPROC - const maxDescriptors = 7 # KERN_MAXFILES - proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, - newp: pointer, newplen: int): cint - {.importc: "sysctl",header: """#include - #include """} - elif defined(linux) or defined(solaris): - proc ulimit(cmd: cint): clong - {.importc: "ulimit", header: "", varargs.} - elif defined(windows): - discard - else: - var - RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", - header: "".}: cint - type - rlimit {.importc: "struct rlimit", - header: "", pure, final.} = object - rlim_cur: int - rlim_max: int - proc getrlimit(resource: cint, rlp: var rlimit): cint - {.importc: "getrlimit",header: "".} - - proc getMaxFds*(): int = - when defined(macosx) or defined(freebsd) or defined(netbsd) or - defined(openbsd): - var count = cint(0) - var size = sizeof(count) - var namearr = [cint(1), cint(maxDescriptors)] - - if sysctl(addr namearr[0], 2, cast[pointer](addr count), addr size, - nil, 0) != 0: - raiseOsError(osLastError()) - result = count - elif defined(linux) or defined(solaris): - result = int(ulimit(4, 0)) - elif defined(windows): - result = FD_SETSIZE - else: - var a = rlimit() - if getrlimit(RLIMIT_NOFILE, a) != 0: - raiseOsError(osLastError()) - result = a.rlim_max - - when hasThreadSupport: - import locks - - type - Event* {.pure.} = enum - Read, Write, Timer, Signal, Process, Vnode, User, Error, - flagHandle, flagTimer, flagSignal, flagProcess, flagVnode, flagUser, - flagOneshot - - ReadyKey*[T] = object - fd* : int - events*: set[Event] - data*: T - - SelectorKey[T] = object - ident : int - flags : set[Event] - param : int - key : ReadyKey[T] - - when not defined(windows): - type - SharedArrayHolder[T] = object - part: array[16, T] - SharedArray {.unchecked.}[T] = array[0..100_000_000, T] - - proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = - let holder = cast[ptr SharedArrayHolder[T]]( - allocShared0(sizeof(T) * nsize) - ) - result = cast[ptr SharedArray[T]](addr(holder.part[0])) - - proc deallocSharedArray[T](sa: ptr SharedArray[T]) = - deallocShared(cast[pointer](sa)) - - template setNonBlocking(fd) = - var x: int = fcntl(fd, F_GETFL, 0) - if x == -1: raiseOSError(osLastError()) - else: - var mode = x or O_NONBLOCK - if fcntl(fd, F_SETFL, mode) == -1: - raiseOSError(osLastError()) - - template setKey(s, f1, f2, e, p, d) = - s.fds[f1].ident = f1 - s.fds[f1].flags = e - s.fds[f1].param = p - s.fds[f1].key.fd = f2 - s.fds[f1].key.data = d - - template clearKey(s, f) = - s.fds[f].ident = 0 - s.fds[f].flags = {} - - template checkMaxFd(s, fd) = - if fd.uint >= s.maxFD: - raise newException(ValueError, "Maximum file descriptors exceeded") - - when supportedPlatform: - template blockSignals(newmask: var Sigset, oldmask: var Sigset) = - when hasThreadSupport: - if posix.pthread_sigmask(SIG_BLOCK, newmask, oldmask) == -1: - raiseOSError(osLastError()) - else: - if posix.sigprocmask(SIG_BLOCK, newmask, oldmask) == -1: - raiseOSError(osLastError()) - - template unblockSignals(newmask: var Sigset, oldmask: var Sigset) = - when hasThreadSupport: - if posix.pthread_sigmask(SIG_UNBLOCK, newmask, oldmask) == -1: - raiseOSError(osLastError()) - else: - if posix.sigprocmask(SIG_UNBLOCK, newmask, oldmask) == -1: - raiseOSError(osLastError()) - # - # BSD kqueue - # - # I have tried to adopt kqueue's EVFILT_USER filter for user-events, but it - # looks not very usable, because of 2 cases: - # 1) EVFILT_USER does not supported by OpenBSD and NetBSD - # 2) You can't have one event, which you can use with many kqueue handles. - # So decision was made in favor of the pipes - # - when bsdPlatform: - const - # Maximum number of cached changes - MAX_KQUEUE_CHANGE_EVENTS = 64 - # Maximum number of events that can be returned - MAX_KQUEUE_RESULT_EVENTS = 64 - - type - SelectorImpl[T] = object - kqFD : cint - maxFD : uint - changesTable: array[MAX_KQUEUE_CHANGE_EVENTS, KEvent] - changesCount: int - fds: ptr SharedArray[SelectorKey[T]] - count: int - when hasThreadSupport: - changesLock: Lock - Selector*[T] = ptr SelectorImpl[T] - - type - SelectEventImpl = object - rfd: cint - wfd: cint - # SelectEvent is declared as `ptr` to be placed in `shared memory`, - # so you can share one SelectEvent handle between threads. - type SelectEvent* = ptr SelectEventImpl - - proc newSelector*[T](): Selector[T] = - var maxFD = getMaxFds() - var kqFD = kqueue() - if kqFD < 0: - raiseOsError(osLastError()) - - result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) - result.kqFD = kqFD - result.maxFD = maxFD.uint - result.fds = allocSharedArray[SelectorKey[T]](maxFD) - when hasThreadSupport: - initLock(result.changesLock) - - proc close*[T](s: Selector[T]) = - if posix.close(s.kqFD) != 0: - raiseOSError(osLastError()) - when hasThreadSupport: - deinitLock(s.changesLock) - deallocSharedArray(s.fds) - deallocShared(cast[pointer](s)) - - when hasThreadSupport: - template withChangeLock[T](s: Selector[T], body: untyped) = - acquire(s.changesLock) - {.locks: [s.changesLock].}: - try: - body - finally: - release(s.changesLock) - else: - template withChangeLock(s, body: untyped) = - body - - template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, - nflags: cushort, nfflags: cuint, ndata: int, - nudata: pointer) = - mixin withChangeLock - s.withChangeLock(): - s.changesTable[s.changesCount] = KEvent(ident: nident, - filter: nfilter, flags: nflags, - fflags: nfflags, data: ndata, - udata: nudata) - inc(s.changesCount) - if s.changesCount == MAX_KQUEUE_CHANGE_EVENTS: - if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), - nil, 0, nil) == -1: - raiseOSError(osLastError()) - s.changesCount = 0 - - proc registerHandle*[T](s: Selector[T], fd: SocketHandle, - events: set[Event], data: T) = - var fdi = int(fd) - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident == 0) - setKey(s, fdi, fdi, {Event.flagHandle} + events, 0, data) - if events != {}: - if Event.Read in events: - modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) - inc(s.count) - if Event.Write in events: - modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) - inc(s.count) - - proc updateHandle*[T](s: Selector[T], fd: SocketHandle, - events: set[Event]) = - var fdi = int(fd) - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident != 0) - doAssert(Event.flagHandle in s.fds[fdi].flags) - var ne = events + {Event.flagHandle} - var oe = s.fds[fdi].flags - if oe != ne: - if (Event.Read in oe) and (Event.Read notin ne): - modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) - dec(s.count) - if (Event.Write in oe) and (Event.Write notin ne): - modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) - dec(s.count) - if (Event.Read notin oe) and (Event.Read in ne): - modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) - inc(s.count) - if (Event.Write notin oe) and (Event.Write in ne): - modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) - inc(s.count) - s.fds[fdi].flags = ne - - proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, - data: T): int {.discardable.} = - var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, - posix.IPPROTO_TCP).int - if fdi == -1: - raiseOsError(osLastError()) - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident == 0) - var mflags = if oneshot: {Event.flagTimer, Event.flagOneshot} - else: {Event.flagTimer} - var kflags: cushort = if oneshot: EV_ONESHOT or EV_ADD - else: EV_ADD - setKey(s, fdi, fdi, mflags, 0, data) - # EVFILT_TIMER on Open/Net(BSD) has granularity of only milliseconds, - # but MacOS and FreeBSD allow use `0` as `fflags` to use milliseconds - # too - modifyKQueue(s, fdi.uint, EVFILT_TIMER, kflags, 0, cint(timeout), nil) - inc(s.count) - result = fdi - - proc registerSignal*[T](s: Selector[T], signal: int, - data: T): int {.discardable.} = - var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, - posix.IPPROTO_TCP).int - if fdi == -1: - raiseOsError(osLastError()) - - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident == 0) - setKey(s, fdi, signal, {Event.flagSignal}, signal, data) - # block signal `signal` - var nmask: Sigset - var omask: Sigset - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, cint(signal)) - blockSignals(nmask, omask) - # to be compatible with linux semantic we need to "eat" signals - posix.signal(cint(signal), SIG_IGN) - modifyKQueue(s, signal.uint, EVFILT_SIGNAL, EV_ADD, 0, 0, - cast[pointer](fdi)) - inc(s.count) - result = fdi - - proc registerProcess*[T](s: Selector[T], pid: int, - data: T): int {.discardable.} = - var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, - posix.IPPROTO_TCP).int - if fdi == -1: - raiseOsError(osLastError()) - - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident == 0) - var kflags: cushort = EV_ONESHOT or EV_ADD - setKey(s, fdi, pid, {Event.flagProcess, Event.flagOneshot}, pid, data) - modifyKQueue(s, pid.uint, EVFILT_PROC, kflags, NOTE_EXIT, 0, - cast[pointer](fdi)) - inc(s.count) - result = fdi - - proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = - var fdi = int(fd) - if fdi.uint < s.maxFD: - var flags = s.fds[fdi].flags - var filter: cshort = 0 - if s.fds[fdi].ident != 0 and flags != {}: - if Event.flagHandle in flags: - # if events == 0, than descriptor was modified with - # updateHandle(fd, 0), so it was already deleted from kqueue. - if flags != {Event.flagHandle}: - if Event.Read in flags: - modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) - dec(s.count) - if Event.Write in flags: - modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) - dec(s.count) - elif Event.flagTimer in flags: - filter = EVFILT_TIMER - discard posix.close(cint(s.fds[fdi].key.fd)) - modifyKQueue(s, fdi.uint, filter, EV_DELETE, 0, 0, nil) - dec(s.count) - elif Event.flagSignal in flags: - filter = EVFILT_SIGNAL - # unblocking signal - var nmask = Sigset() - var omask = Sigset() - var signal = cint(s.fds[fdi].param) - discard sigaddset(nmask, signal) - unblockSignals(nmask, omask) - posix.signal(signal, SIG_DFL) - discard posix.close(cint(s.fds[fdi].key.fd)) - modifyKQueue(s, fdi.uint, filter, EV_DELETE, 0, 0, nil) - dec(s.count) - elif Event.flagProcess in flags: - filter = EVFILT_PROC - discard posix.close(cint(s.fds[fdi].key.fd)) - modifyKQueue(s, fdi.uint, filter, EV_DELETE, 0, 0, nil) - dec(s.count) - elif Event.flagUser in flags: - filter = EVFILT_READ - modifyKQueue(s, fdi.uint, filter, EV_DELETE, 0, 0, nil) - dec(s.count) - clearKey(s, fdi) - - proc flush*[T](s: Selector[T]) = - s.withChangeLock(): - var tv = Timespec() - if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), - nil, 0, addr tv) == -1: - raiseOSError(osLastError()) - s.changesCount = 0 - - template isEmpty*[T](s: Selector[T]): bool = - (s.count == 0) - - proc newEvent*(): SelectEvent = - var fds: array[2, cint] - - if posix.pipe(fds) == -1: - raiseOSError(osLastError()) - - setNonBlocking(fds[0]) - setNonBlocking(fds[1]) - - result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) - result.rfd = fds[0] - result.wfd = fds[1] - - proc setEvent*(ev: SelectEvent) = - var data: int = 1 - if posix.write(ev.wfd, addr data, sizeof(int)) != sizeof(int): - raiseOSError(osLastError()) - - proc close*(ev: SelectEvent) = - discard posix.close(cint(ev.rfd)) - discard posix.close(cint(ev.wfd)) - deallocShared(cast[pointer](ev)) - - proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = - let fdi = ev.rfd.int - doAssert(s.fds[fdi].ident == 0) - setKey(s, fdi, fdi, {Event.flagUser}, 0, data) - modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) - inc(s.count) - - proc unregister*[T](s: Selector[T], ev: SelectEvent) = - let fdi = ev.rfd.int - var flags = s.fds[fdi].flags - if s.fds[fdi].ident != 0 and flags != {}: - modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) - dec(s.count) - clearKey(s, fdi) - - proc selectInto*[T](s: Selector[T], timeout: int, - results: var openarray[ReadyKey[T]]): int = - var - tv: Timespec - resultsTable: array[MAX_KQUEUE_RESULT_EVENTS, KEvent] - ptv: ptr Timespec = addr tv - - if timeout != -1: - if timeout >= 1000: - tv.tv_sec = (timeout div 1_000).Time - tv.tv_nsec = (timeout %% 1_000) * 1_000_000 - else: - tv.tv_sec = 0.Time - tv.tv_nsec = timeout * 1_000_000 - else: - ptv = nil - - var maxResults = MAX_KQUEUE_RESULT_EVENTS - if maxResults > len(results): - maxResults = len(results) - - var count = 0 - s.withChangeLock(): - count = kevent(s.kqFD, - addr(s.changesTable[0]), cint(s.changesCount), - addr(resultsTable[0]), cint(maxResults), ptv) - s.changesCount = 0 - if count >= 0: - var skey: ptr SelectorKey[T] - var i = 0 - var k = 0 - while i < count: - var kevent = addr(resultsTable[i]) - if (kevent.flags and EV_ERROR) == 0: - var events: set[Event] = {} - case kevent.filter - of EVFILT_READ: - skey = addr(s.fds[kevent.ident.int]) - if Event.flagHandle in skey.flags: - events = {Event.Read} - elif Event.flagUser in skey.flags: - var data: int = 0 - if posix.read(kevent.ident.cint, addr data, - sizeof(int)) != sizeof(int): - let err = osLastError() - if err == OSErrorCode(EAGAIN): - # someone already consumed event data - inc(i) - continue - else: - raiseOSError(osLastError()) - events = {Event.User} - else: - events = {Event.Read} - of EVFILT_WRITE: - skey = addr(s.fds[kevent.ident.int]) - events = {Event.Write} - of EVFILT_TIMER: - skey = addr(s.fds[kevent.ident.int]) - if Event.flagOneshot in skey.flags: - if posix.close(skey.ident.cint) == -1: - raiseOSError(osLastError()) - clearKey(s, skey.ident) - # no need to modify kqueue, because EV_ONESHOT is already made - # this for us - dec(s.count) - events = {Event.Timer} - of EVFILT_VNODE: - skey = addr(s.fds[kevent.ident.int]) - events = {Event.Vnode} - of EVFILT_SIGNAL: - skey = addr(s.fds[cast[int](kevent.udata)]) - events = {Event.Signal} - of EVFILT_PROC: - skey = addr(s.fds[cast[int](kevent.udata)]) - if posix.close(skey.ident.cint) == -1: - raiseOSError(osLastError()) - clearKey(s, skey.ident) - # no need to modify kqueue, because EV_ONESHOT is already made - # this for us - dec(s.count) - events = {Event.Process} - else: - raise newException(ValueError, - "Unsupported kqueue filter in queue") - - if (kevent.flags and EV_EOF) != 0: - events = events + {Event.Error} - results[k].fd = skey.key.fd - results[k].events = events - results[k].data = skey.key.data - inc(k) - inc(i) - result = k - else: - result = 0 - let err = osLastError() - if cint(err) != EINTR: - raiseOSError(err) - - proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = - result = newSeq[ReadyKey[T]](MAX_KQUEUE_RESULT_EVENTS) - var count = selectInto(s, timeout, result) - result.setLen(count) - - # - # Linux epoll - # - - elif defined(linux): - const - # Maximum number of events that can be returned - MAX_EPOLL_RESULT_EVENTS = 64 - type - SignalFdInfo* {.importc: "struct signalfd_siginfo", - header: "", pure, final.} = object - ssi_signo*: uint32 - ssi_errno*: int32 - ssi_code*: int32 - ssi_pid*: uint32 - ssi_uid*: uint32 - ssi_fd*: int32 - ssi_tid*: uint32 - ssi_band*: uint32 - ssi_overrun*: uint32 - ssi_trapno*: uint32 - ssi_status*: int32 - ssi_int*: int32 - ssi_ptr*: uint64 - ssi_utime*: uint64 - ssi_stime*: uint64 - ssi_addr*: uint64 - pad* {.importc: "__pad".}: array[0..47, uint8] - type - eventFdData {.importc: "eventfd_t", - header: "", pure, final.} = uint64 - epoll_data {.importc: "union epoll_data", - header: "", - pure, final.} = object - u64 {.importc: "u64".}: uint64 - - epoll_event {.importc: "struct epoll_event", - header: "", pure, final.} = object - events: uint32 # Epoll events - data: epoll_data # User data variable - const - EPOLL_CTL_ADD = 1 # Add a file descriptor to the interface. - EPOLL_CTL_DEL = 2 # Remove a file descriptor from the interface. - EPOLL_CTL_MOD = 3 # Change file descriptor epoll_event structure. - const - EPOLLIN = 0x00000001 - EPOLLOUT = 0x00000004 - EPOLLERR = 0x00000008 - EPOLLHUP = 0x00000010 - EPOLLRDHUP = 0x00002000 - EPOLLONESHOT = 1 shl 30 - - proc epoll_create(size: cint): cint - {.importc: "epoll_create", header: "".} - proc epoll_ctl(epfd: cint; op: cint; fd: cint; event: ptr epoll_event): cint - {.importc: "epoll_ctl", header: "".} - proc epoll_wait(epfd: cint; events: ptr epoll_event; maxevents: cint; - timeout: cint): cint - {.importc: "epoll_wait", header: "".} - proc timerfd_create(clock_id: ClockId, flags: cint): cint - {.cdecl, importc: "timerfd_create", header: "".} - proc timerfd_settime(ufd: cint, flags: cint, - utmr: var Itimerspec, otmr: var Itimerspec): cint - {.cdecl, importc: "timerfd_settime", header: "".} - proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint - {.cdecl, importc: "signalfd", header: "".} - proc eventfd(count: cuint, flags: cint): cint - {.cdecl, importc: "eventfd", header: "".} - - type - SelectorImpl[T] = object - epollFD : cint - maxFD : uint - fds: ptr SharedArray[SelectorKey[T]] - count: int - - Selector*[T] = ptr SelectorImpl[T] - - SelectEventImpl = object - efd: cint - - SelectEvent* = ptr SelectEventImpl - - proc newSelector*[T](): Selector[T] = - var maxFD = getMaxFds() - var epollFD = epoll_create(MAX_EPOLL_RESULT_EVENTS) - if epollFD < 0: - raiseOsError(osLastError()) - - result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) - result.epollFD = epollFD - result.maxFD = maxFD.uint - result.fds = allocSharedArray[SelectorKey[T]](maxFD) - - proc close*[T](s: Selector[T]) = - if posix.close(s.epollFD) != 0: - raiseOSError(osLastError()) - deallocSharedArray(s.fds) - deallocShared(cast[pointer](s)) - - proc registerHandle*[T](s: Selector[T], fd: SocketHandle, - events: set[Event], data: T) = - var fdi = int(fd) - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident == 0) - setKey(s, fdi, fdi, events + {Event.flagHandle}, 0, data) - if events != {}: - var epv: epoll_event - epv.events = EPOLLRDHUP - epv.data.u64 = fdi.uint - if Event.Read in events: - epv.events = epv.events or EPOLLIN - if Event.Write in events: - epv.events = epv.events or EPOLLOUT - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - inc(s.count) - - proc updateHandle*[T](s: Selector[T], fd: SocketHandle, - events: set[Event]) = - var fdi = int(fd) - s.checkMaxFd(fdi) - var oe = s.fds[fdi].flags - doAssert(s.fds[fdi].ident != 0) - doAssert(Event.flagHandle in oe) - var ne = events + {Event.flagHandle} - if oe != ne: - var epv: epoll_event - epv.data.u64 = fdi.uint - epv.events = EPOLLRDHUP - - if Event.Read in events: - epv.events = epv.events or EPOLLIN - if Event.Write in events: - epv.events = epv.events or EPOLLOUT - - if oe == {Event.flagHandle}: - if ne != {Event.flagHandle}: - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, - addr epv) == -1: - raiseOSError(osLastError()) - inc(s.count) - else: - if ne != {Event.flagHandle}: - if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fdi.cint, - addr epv) == -1: - raiseOSError(osLastError()) - else: - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, - addr epv) == -1: - raiseOSError(osLastError()) - dec(s.count) - s.fds[fdi].flags = ne - - proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = - var epv: epoll_event - var fdi = int(fd) - if fdi.uint < s.maxFD: - var flags = s.fds[fdi].flags - if s.fds[fdi].ident != 0 and flags != {}: - if Event.flagHandle in flags: - # if events == {flagHandle}, then descriptor was already - # unregistered from epoll with updateHandle() call. - # This check is done to omit EBADF error. - if flags != {Event.flagHandle}: - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, - addr epv) == -1: - raiseOSError(osLastError()) - dec(s.count) - elif Event.flagTimer in flags: - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - discard posix.close(fdi.cint) - dec(s.count) - elif Event.flagSignal in flags: - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - var nmask: Sigset - var omask: Sigset - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, cint(s.fds[fdi].param)) - unblockSignals(nmask, omask) - discard posix.close(fdi.cint) - dec(s.count) - elif Event.flagProcess in flags: - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - var nmask: Sigset - var omask: Sigset - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, SIGCHLD) - unblockSignals(nmask, omask) - discard posix.close(fdi.cint) - dec(s.count) - clearKey(s, fdi) - - proc unregister*[T](s: Selector[T], ev: SelectEvent) = - let fdi = int(ev.efd) - if fdi.uint < s.maxFD: - if s.fds[fdi].ident != 0 and (Event.flagUser in s.fds[fdi].flags): - clearKey(s, fdi) - var epv: epoll_event - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - dec(s.count) - - proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, - data: T): int {.discardable.} = - var - new_ts: Itimerspec - old_ts: Itimerspec - var fdi = timerfd_create(CLOCK_MONOTONIC, 0) - if fdi == -1: - raiseOSError(osLastError()) - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident == 0) - var flags = {Event.flagTimer} - var epv: epoll_event - epv.data.u64 = fdi.uint - epv.events = EPOLLIN or EPOLLRDHUP - setNonBlocking(fdi.cint) - if oneshot: - new_ts.it_interval.tv_sec = 0.Time - new_ts.it_interval.tv_nsec = 0 - new_ts.it_value.tv_sec = (timeout div 1_000).Time - new_ts.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000 - flags = flags + {Event.flagOneshot} - epv.events = epv.events or EPOLLONESHOT - else: - new_ts.it_interval.tv_sec = (timeout div 1000).Time - new_ts.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000 - new_ts.it_value.tv_sec = new_ts.it_interval.tv_sec - new_ts.it_value.tv_nsec = new_ts.it_interval.tv_nsec - if timerfd_settime(fdi.cint, cint(0), new_ts, old_ts) == -1: - raiseOSError(osLastError()) - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - setKey(s, fdi, fdi, flags, 0, data) - inc(s.count) - result = fdi - - proc registerSignal*[T](s: Selector[T], signal: int, - data: T): int {.discardable.} = - var - nmask: Sigset - omask: Sigset - - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, cint(signal)) - blockSignals(nmask, omask) - - var fdi = signalfd(-1, nmask, 0).int - if fdi == -1: - raiseOSError(osLastError()) - - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident == 0) - setNonBlocking(fdi.cint) - - var epv: epoll_event - epv.data.u64 = fdi.uint - epv.events = EPOLLIN or EPOLLRDHUP - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - setKey(s, fdi, signal, {Event.flagSignal}, signal, data) - inc(s.count) - result = fdi - - proc registerProcess*[T](s: Selector, pid: int, - data: T): int {.discardable.} = - var - nmask: Sigset - omask: Sigset - - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, posix.SIGCHLD) - blockSignals(nmask, omask) - - var fdi = signalfd(-1, nmask, 0).int - if fdi == -1: - raiseOSError(osLastError()) - - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident == 0) - setNonBlocking(fdi.cint) - - var epv: epoll_event - epv.data.u64 = fdi.uint - epv.events = EPOLLIN or EPOLLRDHUP - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - setKey(s, fdi, pid, {Event.flagProcess}, pid, data) - inc(s.count) - result = fdi - - proc flush*[T](s: Selector[T]) = - discard - - template isEmpty*[T](s: Selector[T]): bool = - (s.count == 0) - - proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = - let fdi = int(ev.efd) - doAssert(s.fds[fdi].ident == 0) - setKey(s, fdi, fdi, {Event.flagUser}, 0, data) - var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) - epv.data.u64 = ev.efd.uint - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, ev.efd, addr epv) == -1: - raiseOSError(osLastError()) - inc(s.count) - - proc setEvent*(ev: SelectEvent) = - var data : uint64 = 1 - if posix.write(ev.efd, addr data, sizeof(uint64)) == -1: - raiseOSError(osLastError()) - - proc close*(ev: SelectEvent) = - discard posix.close(ev.efd) - deallocShared(cast[pointer](ev)) - - proc newEvent*(): SelectEvent = - var fdi = eventfd(0, 0) - if fdi == -1: - raiseOSError(osLastError()) - setNonBlocking(fdi) - result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) - result.efd = cint(fdi) - - proc selectInto*[T](s: Selector[T], timeout: int, - results: var openarray[ReadyKey[T]]): int = - var - resultsTable: array[MAX_EPOLL_RESULT_EVENTS, epoll_event] - - var maxResults = MAX_EPOLL_RESULT_EVENTS - if maxResults > len(results): - maxResults = len(results) - - var count = epoll_wait(s.epollFD, addr(resultsTable[0]), maxResults.cint, - timeout.cint) - if count > 0: - var i = 0 - var k = 0 - while i < count: - var events: set[Event] = {} - let fdi = int(resultsTable[i].data.u64) - var skey = addr(s.fds[fdi]) - let pevents = resultsTable[i].events - var flags = s.fds[fdi].flags - - if skey.ident != 0 and flags != {}: - block processItem: - if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: - events = events + {Event.Error} - if (pevents and EPOLLOUT) != 0: - events = events + {Event.Write} - if (pevents and EPOLLIN) != 0: - if Event.flagHandle in flags: - events = events + {Event.Read} - elif Event.flagTimer in flags: - var data: uint64 = 0 - if posix.read(fdi.cint, addr data, - sizeof(uint64)) != sizeof(uint64): - raiseOSError(osLastError()) - events = events + {Event.Timer} - elif Event.flagSignal in flags: - var data: SignalFdInfo - if posix.read(fdi.cint, addr data, - sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): - raiseOsError(osLastError()) - events = events + {Event.Signal} - elif Event.flagProcess in flags: - var data: SignalFdInfo - if posix.read(fdi.cint, addr data, - sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): - raiseOsError(osLastError()) - if cast[int](data.ssi_pid) == skey.param: - events = events + {Event.Process} - # we want to free resources for this event - flags = flags + {Event.flagOneshot} - else: - break processItem - elif Event.flagUser in flags: - var data: uint = 0 - if posix.read(fdi.cint, addr data, - sizeof(uint)) != sizeof(uint): - let err = osLastError() - if err == OSErrorCode(EAGAIN): - # someone already consumed event data - inc(i) - continue - else: - raiseOSError(err) - events = events + {Event.User} - else: - raise newException(ValueError, - "Unsupported epoll event in queue") - results[k].fd = skey.key.fd - results[k].events = events - results[k].data = skey.key.data - - if Event.flagOneshot in flags: - var epv: epoll_event - try: - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, - addr epv) == -1: - raiseOSError(osLastError()) - finally: - discard posix.close(fdi.cint) - s.fds[fdi].ident = 0 - s.fds[fdi].flags = {} - dec(s.count) - inc(k) - inc(i) - result = k - elif count == 0: - discard - else: - result = 0 - let err = osLastError() - if cint(err) != EINTR: - raiseOSError(err) - - proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = - result = newSeq[ReadyKey[T]](MAX_EPOLL_RESULT_EVENTS) - var count = selectInto(s, timeout, result) - result.setLen(count) - - # - # Windows select - # - - elif defined(windows): - const FD_SETSIZE = 64 - - import hashes, nativesockets - - when hasThreadSupport: - import sharedtables - else: - import tables - - proc hash*(x: SocketHandle): Hash {.borrow.} - proc `$`*(x: SocketHandle): string {.borrow.} - - proc WSAFDIsSet(s: SocketHandle, fdSet: var TFdSet): bool {. - stdcall, importc: "__WSAFDIsSet", dynlib: "ws2_32.dll", noSideEffect.} - - template iFD_ISSET(s: SocketHandle, fdSet: var TFdSet): bool = - if WSAFDIsSet(s, fdSet): true else: false - - template iFD_SET(s: SocketHandle, fdSet: var TFdSet) = - block: - var i = 0 - while i < fdSet.fd_count: - if fdSet.fd_array[i] == s: - break - inc(i) - if i == fdSet.fd_count: - if fdSet.fd_count < ioselectors.FD_SETSIZE: - fdSet.fd_array[i] = s - inc(fdSet.fd_count) - - template iFD_CLR(s: SocketHandle, fdSet: var TFdSet) = - block: - var i = 0 - while i < fdSet.fd_count: - if fdSet.fd_array[i] == s: - if i == fdSet.fd_count - 1: - fdSet.fd_array[i] = 0.SocketHandle - else: - while i < (fdSet.fd_count - 1): - fdSet.fd_array[i] = fdSet.fd_array[i + 1] - inc(i) - dec(fdSet.fd_count) - break - inc(i) - - template iFD_ZERO(fdSet: var TFdSet) = - fdSet.fd_count = 0 - - when hasThreadSupport: - type - SelectorImpl[T] = object - rSet: TFdSet - wSet: TFdSet - eSet: TFdSet - maxFD: uint - fds: SharedTable[SocketHandle, SelectorKey[T]] - count: int - lock: Lock - else: - type - SelectorImpl[T] = object - rSet: TFdSet - wSet: TFdSet - eSet: TFdSet - maxFD: uint - fds: Table[SocketHandle, SelectorKey[T]] - count: int - - when hasThreadSupport: - type Selector*[T] = ptr SelectorImpl[T] - else: - type Selector*[T] = ref SelectorImpl[T] - - type - SelectEventImpl = object - rsock: SocketHandle - wsock: SocketHandle - - type SelectEvent* = ptr SelectEventImpl - - when hasThreadSupport: - template withSelectLock[T](s: Selector[T], body: untyped) = - acquire(s.lock) - {.locks: [s.lock].}: - try: - body - finally: - release(s.lock) - else: - template withSelectLock[T](s: Selector[T], body: untyped) = - body - - proc newSelector*[T](): Selector[T] = - var maxFD = FD_SETSIZE - when hasThreadSupport: - result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) - result.maxFD = maxFD.uint - result.fds = initSharedTable[SocketHandle, SelectorKey[T]]() - initLock result.lock - else: - result = Selector[T](maxFD: FD_SETSIZE) - result.maxFD = maxFD.uint - result.fds = initTable[SocketHandle, SelectorKey[T]]() - - iFD_ZERO(result.rSet) - iFD_ZERO(result.wSet) - iFD_ZERO(result.eSet) - - proc close*(s: Selector) = - when hasThreadSupport: - deinitSharedTable(s.fds) - deallocShared(cast[pointer](s)) - - template isEmpty*[T](s: Selector[T]): bool = - (s.count == 0) - - template selectAdd[T](s: Selector[T], fd: SocketHandle, - events: set[Event]) = - mixin withSelectLock - s.withSelectLock(): - if Event.Read in events: - if s.rSet.fd_count == FD_SETSIZE: - raise newException(ValueError, "Maximum numbers of fds exceeded") - iFD_SET(fd, s.rSet) - inc(s.count) - if Event.Write in events: - if s.wSet.fd_count == FD_SETSIZE: - raise newException(ValueError, "Maximum numbers of fds exceeded") - iFD_SET(fd, s.wSet) - iFD_SET(fd, s.eSet) - inc(s.count) - - proc registerHandle*[T](s: Selector[T], fd: SocketHandle, - events: set[Event], data: T) = - var fdi = int(fd) - var flags = {Event.flagHandle} + events - var nkey = SelectorKey[T](ident: fdi, flags: flags) - nkey.key.fd = fdi - nkey.key.data = data - - if s.fds.hasKeyOrPut(fd, nkey): - raise newException(ValueError, "Re-use of non closed descriptor") - selectAdd(s, fd, flags) - - proc updateHandle*[T](s: Selector[T], fd: SocketHandle, - events: set[Event]) = - s.withSelectLock(): - withValue(s.fds, fd, skey) do: - if Event.flagHandle in skey.flags: - var oe = skey.flags - var ne = events + {Event.flagHandle} - if oe != ne: - if (Event.Read in oe) and (Event.Read notin ne): - iFD_CLR(fd, s.rSet) - dec(s.count) - if (Event.Write in oe) and (Event.Write notin ne): - iFD_CLR(fd, s.wSet) - iFD_CLR(fd, s.eSet) - dec(s.count) - if (Event.Read notin oe) and (Event.Read in ne): - iFD_SET(fd, s.rSet) - inc(s.count) - if (Event.Write notin oe) and (Event.Write in ne): - iFD_SET(fd, s.wSet) - iFD_SET(fd, s.eSet) - inc(s.count) - skey.flags = ne - else: - raise newException(ValueError, - "Could not update non-handle descriptor") - do: - raise newException(ValueError, - "Descriptor is not registered in queue") - - proc registerTimer*[T](s: Selector, timeout: int, oneshot: bool, - data: T): int {.discardable.} = - raise newException(ValueError, "Not implemented") - - proc registerSignal*[T](s: Selector, signal: int, - data: T): int {.discardable.} = - raise newException(ValueError, "Not implemented") - - proc registerProcess*[T](s: Selector, pid: int, - data: T): int {.discardable.} = - raise newException(ValueError, "Not implemented") - - proc flush*[T](s: Selector[T]) = discard - - proc unregister*[T](s: Selector[T], ev: SelectEvent) = - let fd = ev.rsock - s.withSelectLock(): - iFD_CLR(fd, s.rSet) - dec(s.count) - s.fds.del(fd) - - - proc unregister*[T](s: Selector[T], fd: SocketHandle) = - s.withSelectLock(): - s.fds.withValue(fd, skey) do: - if Event.Read in skey.flags: - iFD_CLR(fd, s.rSet) - dec(s.count) - if Event.Write in skey.flags: - iFD_CLR(fd, s.wSet) - iFD_CLR(fd, s.eSet) - dec(s.count) - s.fds.del(fd) - - proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = - var flags = {Event.flagUser, Event.Read} - var nkey = SelectorKey[T](ident: ev.rsock.int, flags: flags) - nkey.key.fd = ev.rsock.int - nkey.key.data = data - if s.fds.hasKeyOrPut(ev.rsock, nkey): - raise newException(ValueError, "Re-use of non closed descriptor") - selectAdd(s, ev.rsock, flags) - - proc newEvent*(): SelectEvent = - var ssock = newNativeSocket() - var wsock = newNativeSocket() - var rsock: SocketHandle = INVALID_SOCKET - var saddr = Sockaddr_in() - try: - saddr.sin_family = winlean.AF_INET - saddr.sin_port = 0 - saddr.sin_addr.s_addr = INADDR_ANY - if bindAddr(ssock, cast[ptr SockAddr](addr(saddr)), - sizeof(saddr).SockLen) < 0'i32: - raiseOSError(osLastError()) - - if winlean.listen(ssock, 1) == -1: - raiseOSError(osLastError()) - - var namelen = sizeof(saddr).SockLen - if getsockname(ssock, cast[ptr SockAddr](addr(saddr)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - - saddr.sin_addr.s_addr = 0x0100007F - if winlean.connect(wsock, cast[ptr SockAddr](addr(saddr)), - sizeof(saddr).SockLen) == -1: - raiseOSError(osLastError()) - namelen = sizeof(saddr).SockLen - rsock = winlean.accept(ssock, cast[ptr SockAddr](addr(saddr)), - cast[ptr SockLen](addr(namelen))) - if rsock == SocketHandle(-1): - raiseOSError(osLastError()) - - if winlean.closesocket(ssock) == -1: - raiseOSError(osLastError()) - - var mode = clong(1) - if ioctlsocket(rsock, FIONBIO, addr(mode)) == -1: - raiseOSError(osLastError()) - mode = clong(1) - if ioctlsocket(wsock, FIONBIO, addr(mode)) == -1: - raiseOSError(osLastError()) - - result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) - result.rsock = rsock - result.wsock = wsock - except: - discard winlean.closesocket(ssock) - discard winlean.closesocket(wsock) - if rsock != INVALID_SOCKET: - discard winlean.closesocket(rsock) - - proc setEvent*(ev: SelectEvent) = - var data: int = 1 - if winlean.send(ev.wsock, cast[pointer](addr data), - cint(sizeof(int)), 0) != sizeof(int): - raiseOSError(osLastError()) - - proc close*(ev: SelectEvent) = - discard winlean.closesocket(ev.rsock) - discard winlean.closesocket(ev.wsock) - deallocShared(cast[pointer](ev)) - - proc selectInto*[T](s: Selector[T], timeout: int, - results: var openarray[ReadyKey[T]]): int = - var tv = Timeval() - var ptv = addr tv - var rset, wset, eset: TFdSet - - if timeout != -1: - tv.tv_sec = timeout.int32 div 1_000 - tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 - else: - ptv = nil - - s.withSelectLock(): - rset = s.rSet - wset = s.wSet - eset = s.eSet - - var count = select(cint(0), addr(rset), addr(wset), - addr(eset), ptv).int - if count > 0: - var rindex = 0 - var i = 0 - while i < rset.fd_count: - let fd = rset.fd_array[i] - if iFD_ISSET(fd, rset): - var events = {Event.Read} - if iFD_ISSET(fd, eset): events = events + {Event.Error} - if iFD_ISSET(fd, wset): events = events + {Event.Write} - s.fds.withValue(fd, skey) do: - if Event.flagHandle in skey.flags: - skey.key.events = events - elif Event.flagUser in skey.flags: - var data: int = 0 - if winlean.recv(fd, cast[pointer](addr(data)), - sizeof(int).cint, 0) != sizeof(int): - let err = osLastError() - if err != OSErrorCode(WSAEWOULDBLOCK): - raiseOSError(err) - else: - # someone already consumed event data - inc(i) - continue - skey.key.events = {Event.User} - results[rindex].fd = skey.key.fd - results[rindex].data = skey.key.data - results[rindex].events = skey.key.events - inc(rindex) - inc(i) - - i = 0 - while i < wset.fd_count: - let fd = wset.fd_array[i] - if iFD_ISSET(fd, wset): - var events = {Event.Write} - if not iFD_ISSET(fd, rset): - if iFD_ISSET(fd, eset): events = events + {Event.Error} - s.fds.withValue(fd, skey) do: - skey.key.events = events - results[rindex].fd = skey.key.fd - results[rindex].data = skey.key.data - results[rindex].events = skey.key.events - inc(rindex) - inc(i) - count = rindex - elif count == 0: - discard - else: - raiseOSError(osLastError()) - result = count - - proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = - result = newSeq[ReadyKey[T]](FD_SETSIZE) - var count = selectInto(s, timeout, result) - result.setLen(count) - - # - # Posix poll - # - - else: - # Maximum number of events that can be returned - const MAX_POLL_RESULT_EVENTS = 64 - - type - SelectorImpl[T] = object - maxFD : uint - pollcnt: int - fds: ptr SharedArray[SelectorKey[T]] - pollfds: ptr SharedArray[TPollFd] - count: int - when hasThreadSupport: - lock: Lock - - Selector*[T] = ptr SelectorImpl[T] - - SelectEventImpl = object - rfd: cint - wfd: cint - - SelectEvent* = ptr SelectEventImpl - - when hasThreadSupport: - template withPollLock[T](s: Selector[T], body: untyped) = - acquire(s.lock) - {.locks: [s.lock].}: - try: - body - finally: - release(s.lock) - else: - template withPollLock(s, body: untyped) = - body - - proc newSelector*[T](): Selector[T] = - var maxFD = getMaxFds() - - result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) - result.maxFD = maxFD.uint - result.fds = allocSharedArray[SelectorKey[T]](maxFD) - result.pollfds = allocSharedArray[TPollFd](maxFD) - when hasThreadSupport: - initLock(result.lock) - - proc close*[T](s: Selector[T]) = - when hasThreadSupport: - deinitLock(s.lock) - deallocSharedArray(s.fds) - deallocSharedArray(s.pollfds) - deallocShared(cast[pointer](s)) - - template pollAdd[T](s: Selector[T], sock: cint, events: set[Event]) = - withPollLock(s): - var pollev: cshort = 0 - if Event.Read in events: pollev = pollev or POLLIN - if Event.Write in events: pollev = pollev or POLLOUT - s.pollfds[s.pollcnt].fd = cint(sock) - s.pollfds[s.pollcnt].events = pollev - inc(s.count) - inc(s.pollcnt) - - template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) = - withPollLock(s): - var i = 0 - var pollev: cshort = 0 - if Event.Read in events: pollev = pollev or POLLIN - if Event.Write in events: pollev = pollev or POLLOUT - - while i < s.pollcnt: - if s.pollfds[i].fd == sock: - s.pollfds[i].events = pollev - break - inc(i) - - if i == s.pollcnt: - raise newException(ValueError, - "Descriptor is not registered in queue") - - template pollRemove[T](s: Selector[T], sock: cint) = - withPollLock(s): - var i = 0 - while i < s.pollcnt: - if s.pollfds[i].fd == sock: - if i == s.pollcnt - 1: - s.pollfds[i].fd = 0 - s.pollfds[i].events = 0 - s.pollfds[i].revents = 0 - else: - while i < (s.pollcnt - 1): - s.pollfds[i].fd = s.pollfds[i + 1].fd - s.pollfds[i].events = s.pollfds[i + 1].events - inc(i) - break - inc(i) - dec(s.pollcnt) - dec(s.count) - - proc registerHandle*[T](s: Selector[T], fd: SocketHandle, - events: set[Event], data: T) = - var fdi = int(fd) - s.checkMaxFd(fdi) - doAssert(s.fds[fdi].ident == 0) - setKey(s, fdi, fdi, {Event.flagHandle} + events, 0, data) - s.pollAdd(fdi.cint, events) - - proc updateHandle*[T](s: Selector[T], fd: SocketHandle, - events: set[Event]) = - var fdi = int(fd) - s.checkMaxFd(fdi) - var oe = s.fds[fdi].flags - doAssert(s.fds[fdi].ident != 0) - doAssert(Event.flagHandle in oe) - var ne = events + {Event.flagHandle} - if ne != oe: - if events != {}: - s.pollUpdate(fd.cint, events) - else: - s.pollRemove(fd.cint) - s.fds[fdi].flags = ne - - proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, - data: T): int {.discardable.} = - raise newException(ValueError, "Not implemented") - - proc registerSignal*[T](s: Selector[T], signal: int, - data: T): int {.discardable.} = - raise newException(ValueError, "Not implemented") - - proc registerProcess*[T](s: Selector[T], pid: int, - data: T): int {.discardable.} = - raise newException(ValueError, "Not implemented") - - proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = - var fdi = int(ev.rfd) - doAssert(s.fds[fdi].ident == 0) - var events = {Event.flagUser, Event.Read} - setKey(s, fdi, fdi, events, 0, data) - s.pollAdd(fdi.cint, events) - - proc flush*[T](s: Selector[T]) = discard - - template isEmpty*[T](s: Selector[T]): bool = - (s.count == 0) - - proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = - var fdi = int(fd) - if fdi.uint < s.maxFD: - if s.fds[fdi].ident != 0 and s.fds[fdi].flags != {}: - clearKey(s, fdi) - s.pollRemove(fdi.cint) - - proc unregister*[T](s: Selector[T], ev: SelectEvent) = - var fdi = int(ev.rfd) - if fdi.uint < s.maxFD: - if s.fds[fdi].ident != 0 and (Event.flagUser in s.fds[fdi].flags): - clearKey(s, fdi) - s.pollRemove(fdi.cint) - - proc newEvent*(): SelectEvent = - var fds: array[2, cint] - if posix.pipe(fds) == -1: - raiseOSError(osLastError()) - setNonBlocking(fds[0]) - setNonBlocking(fds[1]) - result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) - result.rfd = fds[0] - result.wfd = fds[1] - - proc setEvent*(ev: SelectEvent) = - var data: int = 1 - if posix.write(ev.wfd, addr data, sizeof(int)) != sizeof(int): - raiseOSError(osLastError()) - - proc close*(ev: SelectEvent) = - discard posix.close(cint(ev.rfd)) - discard posix.close(cint(ev.wfd)) - deallocShared(cast[pointer](ev)) - - proc selectInto*[T](s: Selector[T], timeout: int, - results: var openarray[ReadyKey[T]]): int = - var maxResults = MAX_POLL_RESULT_EVENTS - if maxResults > len(results): - maxResults = len(results) - - s.withPollLock(): - var count = posix.poll(addr(s.pollfds[0]), Tnfds(s.pollcnt), timeout) - if count > 0: - var i = 0 - var k = 0 - var rindex = 0 - while (i < s.pollcnt) and (k < count) and (rindex < maxResults): - let revents = s.pollfds[i].revents - let fd = s.pollfds[i].fd - if revents != 0: - var events: set[Event] = {} - if (revents and POLLIN) != 0: - events = events + {Event.Read} - if (revents and POLLOUT) != 0: - events = events + {Event.Write} - if (revents and POLLERR) != 0 or (revents and POLLHUP) != 0 or - (revents and POLLNVAL) != 0: - events = events + {Event.Error} - - var skey = addr(s.fds[fd]) - if Event.flagUser in skey.flags: - if Event.Read in events: - var data: int = 0 - if posix.read(fd, addr data, sizeof(int)) != sizeof(int): - let err = osLastError() - if err != OSErrorCode(EAGAIN): - raiseOSError(osLastError()) - else: - # someone already consumed event data - inc(i) - continue - events = {Event.User} - - results[rindex].fd = fd - results[rindex].events = events - results[rindex].data = skey.key.data - s.pollfds[i].revents = 0 - inc(rindex) - inc(k) - inc(i) - result = k - elif count == 0: - discard - else: - let err = osLastError() - if err.cint == EINTR: - discard - else: - raiseOSError(osLastError()) - - proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = - result = newSeq[ReadyKey[T]](MAX_POLL_RESULT_EVENTS) - var count = selectInto(s, timeout, result) - result.setLen(count) - - when not defined(windows): - template withData*[T](s: Selector[T], fd: SocketHandle, value, - body: untyped) = - var fdi = int(fd) - s.checkMaxFd(fdi) - if s.fds[fdi].ident != 0: - var value = addr(s.fds[fdi].key.data) - body - - template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, - body2: untyped) = - var fdi = int(fd) - s.checkMaxFd(fdi) - if s.fds[fdi].ident != 0: - var value = addr(s.fds[fdi].key.data) - body1 - else: - body2 - else: - template withData*(s: Selector, fd: SocketHandle, value, body: untyped) = - s.fds.withValue(fd, skey) do: - var value {.inject.} = addr(skey.key.data) - body - - template withData*(s: Selector, fd: SocketHandle, value, - body1, body2: untyped) = - s.fds.withValue(fd, skey) do: - var value {.inject.} = addr(skey.key.data) - body1 - do: - body2 diff --git a/lib/pure/ioselectors/ioselectors.nim b/lib/pure/ioselectors/ioselectors.nim new file mode 100644 index 0000000000..1662c1d785 --- /dev/null +++ b/lib/pure/ioselectors/ioselectors.nim @@ -0,0 +1,261 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module allows high-level and efficient I/O multiplexing. +## +## Supported OS primitives: ``epoll``, ``kqueue``, ``poll`` and +## Windows ``select``. +## +## To use threadsafe version of this module, it needs to be compiled +## with both ``-d:threadsafe`` and ``--threads:on`` options. +## +## Supported features: files, sockets, pipes, timers, processes, signals +## and user events. +## +## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux. +## +## Partially supported OS: Windows (only sockets and user events), +## Solaris (files, sockets, handles and user events). +## +## TODO: ``/dev/poll``, ``event ports`` and filesystem events. + +import os + +const hasThreadSupport = compileOption("threads") and defined(threadsafe) + +const supportedPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(linux) + +const bsdPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) + + +when defined(nimdoc): + type + Selector*[T] = ref object + ## An object which holds descriptors to be checked for read/write status + + Event* {.pure.} = enum + ## An enum which hold event types + Read, ## Descriptor is available for read + Write, ## Descriptor is available for write + Timer, ## Timer descriptor is completed + Signal, ## Signal is raised + Process, ## Process is finished + Vnode, ## Currently not supported + User, ## User event is raised + Error ## Error happens while waiting, for descriptor + + ReadyKey*[T] = object + ## An object which holds result for descriptor + fd* : int ## file/socket descriptor + events*: set[Event] ## set of events + data*: T ## application-defined data + + SelectEvent* = object + ## An object which holds user defined event + + proc newSelector*[T](): Selector[T] = + ## Creates a new selector + + proc close*[T](s: Selector[T]) = + ## Closes selector + + proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], + data: T) = + ## Registers file/socket descriptor ``fd`` to selector ``s`` + ## with events set in ``events``. The ``data`` is application-defined + ## data, which to be passed when event happens. + + proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + ## Update file/socket descriptor ``fd``, registered in selector + ## ``s`` with new events set ``event``. + + proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + ## Registers timer notification with ``timeout`` in milliseconds + ## to selector ``s``. + ## If ``oneshot`` is ``true`` timer will be notified only once. + ## Set ``oneshot`` to ``false`` if your want periodic notifications. + ## The ``data`` is application-defined data, which to be passed, when + ## time limit expired. + + proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + ## Registers Unix signal notification with ``signal`` to selector + ## ``s``. The ``data`` is application-defined data, which to be + ## passed, when signal raises. + ## + ## This function is not supported for ``Windows``. + + proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + ## Registers process id (pid) notification when process has + ## exited to selector ``s``. + ## The ``data`` is application-defined data, which to be passed, when + ## process with ``pid`` has exited. + + proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + ## Registers selector event ``ev`` to selector ``s``. + ## ``data`` application-defined data, which to be passed, when + ## ``ev`` happens. + + proc newSelectEvent*(): SelectEvent = + ## Creates new event ``SelectEvent``. + + proc setEvent*(ev: SelectEvent) = + ## Trigger event ``ev``. + + proc close*(ev: SelectEvent) = + ## Closes selector event ``ev``. + + proc unregister*[T](s: Selector[T], ev: SelectEvent) = + ## Unregisters event ``ev`` from selector ``s``. + + proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = + ## Unregisters file/socket descriptor ``fd`` from selector ``s``. + + proc flush*[T](s: Selector[T]) = + ## Flushes all changes was made to kernel pool/queue. + ## This function is usefull only for BSD and MacOS, because + ## kqueue supports bulk changes to be made. + ## On Linux/Windows and other Posix compatible operation systems, + ## ``flush`` is alias for `discard`. + + proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + ## Process call waiting for events registered in selector ``s``. + ## The ``timeout`` argument specifies the minimum number of milliseconds + ## the function will be blocked, if no events are not ready. Specifying a + ## timeout of ``-1`` causes function to block indefinitely. + ## All available events will be stored in ``results`` array. + ## + ## Function returns number of triggered events. + + proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + ## Process call waiting for events registered in selector ``s``. + ## The ``timeout`` argument specifies the minimum number of milliseconds + ## the function will be blocked, if no events are not ready. Specifying a + ## timeout of -1 causes function to block indefinitely. + ## + ## Function returns sequence of triggered events. + + template isEmpty*[T](s: Selector[T]): bool = + ## Returns ``true``, if there no registered events or descriptors + ## in selector. + + template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + ## retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s`` + ## value.uid = 1000 + ## + + template withData*[T](s: Selector[T], fd: SocketHandle, value, + body1, body2: untyped) = + ## retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s``. + ## value.uid = 1000 + ## do: + ## # block is executed if ``fd`` not registered in selector ``s``. + ## raise + ## + +else: + when hasThreadSupport: + import locks + + type + SharedArrayHolder[T] = object + part: array[1, T] + SharedArray {.unchecked.}[T] = array[0..100_000_000, T] + + proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = + let holder = cast[ptr SharedArrayHolder[T]]( + allocShared0(sizeof(T) * nsize) + ) + result = cast[ptr SharedArray[T]](addr(holder.part[0])) + + proc deallocSharedArray[T](sa: ptr SharedArray[T]) = + deallocShared(cast[pointer](sa)) + + type + Event* {.pure.} = enum + Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot + + ReadyKey*[T] = object + fd* : int + events*: set[Event] + data*: T + + SelectorKey[T] = object + ident: int + events: set[Event] + param: int + key: ReadyKey[T] + + when not defined(windows): + import posix + proc setNonBlocking(fd: cint) {.inline.} = + var x = fcntl(fd, F_GETFL, 0) + if x == -1: + raiseOSError(osLastError()) + else: + var mode = x or O_NONBLOCK + if fcntl(fd, F_SETFL, mode) == -1: + raiseOSError(osLastError()) + + template setKey(s, pident, pkeyfd, pevents, pparam, pdata) = + var skey = addr(s.fds[pident]) + skey.ident = pident + skey.events = pevents + skey.param = pparam + skey.key.fd = pkeyfd + skey.key.data = pdata + + when supportedPlatform: + template blockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + else: + if posix.sigprocmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + + template unblockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + else: + if posix.sigprocmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + + when defined(linux): + include ioselectors_epoll + elif bsdPlatform: + include ioselectors_kqueue + elif defined(windows): + include ioselectors_select + elif defined(solaris): + include ioselectors_poll # need to replace it with event ports + else: + include ioselectors_poll diff --git a/lib/pure/ioselectors/ioselectors_epoll.nim b/lib/pure/ioselectors/ioselectors_epoll.nim new file mode 100644 index 0000000000..92b2cdc076 --- /dev/null +++ b/lib/pure/ioselectors/ioselectors_epoll.nim @@ -0,0 +1,461 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Linux epoll(). + +import posix, times + +# Maximum number of events that can be returned +const MAX_EPOLL_RESULT_EVENTS = 64 + +type + SignalFdInfo* {.importc: "struct signalfd_siginfo", + header: "", pure, final.} = object + ssi_signo*: uint32 + ssi_errno*: int32 + ssi_code*: int32 + ssi_pid*: uint32 + ssi_uid*: uint32 + ssi_fd*: int32 + ssi_tid*: uint32 + ssi_band*: uint32 + ssi_overrun*: uint32 + ssi_trapno*: uint32 + ssi_status*: int32 + ssi_int*: int32 + ssi_ptr*: uint64 + ssi_utime*: uint64 + ssi_stime*: uint64 + ssi_addr*: uint64 + pad* {.importc: "__pad".}: array[0..47, uint8] + + eventFdData {.importc: "eventfd_t", + header: "", pure, final.} = uint64 + epoll_data {.importc: "union epoll_data", header: "", + pure, final.} = object + u64 {.importc: "u64".}: uint64 + epoll_event {.importc: "struct epoll_event", + header: "", pure, final.} = object + events: uint32 # Epoll events + data: epoll_data # User data variable + +const + EPOLL_CTL_ADD = 1 # Add a file descriptor to the interface. + EPOLL_CTL_DEL = 2 # Remove a file descriptor from the interface. + EPOLL_CTL_MOD = 3 # Change file descriptor epoll_event structure. + EPOLLIN = 0x00000001 + EPOLLOUT = 0x00000004 + EPOLLERR = 0x00000008 + EPOLLHUP = 0x00000010 + EPOLLRDHUP = 0x00002000 + EPOLLONESHOT = 1 shl 30 + +proc epoll_create(size: cint): cint + {.importc: "epoll_create", header: "".} +proc epoll_ctl(epfd: cint; op: cint; fd: cint; event: ptr epoll_event): cint + {.importc: "epoll_ctl", header: "".} +proc epoll_wait(epfd: cint; events: ptr epoll_event; maxevents: cint; + timeout: cint): cint + {.importc: "epoll_wait", header: "".} +proc timerfd_create(clock_id: ClockId, flags: cint): cint + {.cdecl, importc: "timerfd_create", header: "".} +proc timerfd_settime(ufd: cint, flags: cint, + utmr: var Itimerspec, otmr: var Itimerspec): cint + {.cdecl, importc: "timerfd_settime", header: "".} +proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint + {.cdecl, importc: "signalfd", header: "".} +proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "".} +proc ulimit(cmd: cint): clong + {.importc: "ulimit", header: "", varargs.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + epollFD : cint + maxFD : int + fds: ptr SharedArray[SelectorKey[T]] + count: int + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + epollFD : cint + maxFD : int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] +type + SelectEventImpl = object + efd: cint + SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] = + var maxFD = int(ulimit(4, 0)) + doAssert(maxFD > 0) + + var epollFD = epoll_create(MAX_EPOLL_RESULT_EVENTS) + if epollFD < 0: + raiseOsError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.epollFD = epollFD + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + else: + result = Selector[T]() + result.epollFD = epollFD + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + +proc close*[T](s: Selector[T]) = + if posix.close(s.epollFD) != 0: + raiseOSError(osLastError()) + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +proc newSelectEvent*(): SelectEvent = + let fdci = eventfd(0, 0) + if fdci == -1: + raiseOSError(osLastError()) + setNonBlocking(fdci) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.efd = fdci + +proc setEvent*(ev: SelectEvent) = + var data : uint64 = 1 + if posix.write(ev.efd, addr data, sizeof(uint64)) == -1: + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(ev.efd) + deallocShared(cast[pointer](ev)) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: + var epv = epoll_event(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + var epv = epoll_event(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + + if pkey.events == {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + else: + if events != {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + else: + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Signal in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(s.fds[fdi].param)) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Process in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, SIGCHLD) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + pkey.ident = 0 + pkey.events = {} + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.efd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + var + new_ts: Itimerspec + old_ts: Itimerspec + let fdi = timerfd_create(CLOCK_MONOTONIC, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var events = {Event.Timer} + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if oneshot: + new_ts.it_interval.tv_sec = 0.Time + new_ts.it_interval.tv_nsec = 0 + new_ts.it_value.tv_sec = (timeout div 1_000).Time + new_ts.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000 + incl(events, Event.Oneshot) + epv.events = epv.events or EPOLLONESHOT + else: + new_ts.it_interval.tv_sec = (timeout div 1000).Time + new_ts.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000 + new_ts.it_value.tv_sec = new_ts.it_interval.tv_sec + new_ts.it_value.tv_nsec = new_ts.it_interval.tv_nsec + + if timerfd_settime(fdi.cint, cint(0), new_ts, old_ts) == -1: + raiseOSError(osLastError()) + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, fdi, events, 0, data) + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, signal, {Event.Signal}, signal, data) + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector, pid: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, posix.SIGCHLD) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + epv.events = EPOLLIN or EPOLLRDHUP + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, pid, {Event.Process, Event.Oneshot}, pid, data) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = int(ev.efd) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, {Event.User}, 0, data) + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = ev.efd.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, ev.efd, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + +proc flush*[T](s: Selector[T]) = + discard + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var + resTable: array[MAX_EPOLL_RESULT_EVENTS, epoll_event] + maxres = MAX_EPOLL_RESULT_EVENTS + events: set[Event] = {} + i, k: int + + if maxres > len(results): + maxres = len(results) + + let count = epoll_wait(s.epollFD, addr(resTable[0]), maxres.cint, + timeout.cint) + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + i = 0 + k = 0 + while i < count: + let fdi = int(resTable[i].data.u64) + let pevents = resTable[i].events + var skey = addr(s.fds[fdi]) + doAssert(skey.ident != 0) + events = {} + + if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: + events.incl(Event.Error) + if (pevents and EPOLLOUT) != 0: + events.incl(Event.Write) + if (pevents and EPOLLIN) != 0: + if Event.Read in skey.events: + events.incl(Event.Read) + elif Event.Timer in skey.events: + var data: uint64 = 0 + if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + events = {Event.Timer} + elif Event.Signal in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + events = {Event.Signal} + elif Event.Process in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + if cast[int](data.ssi_pid) == skey.param: + events = {Event.Process} + else: + inc(i) + continue + elif Event.User in skey.events: + var data: uint = 0 + if posix.read(fdi.cint, addr data, sizeof(uint)) != sizeof(uint): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseOSError(err) + events = {Event.User} + + skey.key.events = events + results[k] = skey.key + inc(k) + + if Event.Oneshot in skey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + skey.ident = 0 + skey.events = {} + dec(s.count) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_EPOLL_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselectors/ioselectors_kqueue.nim b/lib/pure/ioselectors/ioselectors_kqueue.nim new file mode 100644 index 0000000000..78823dea4e --- /dev/null +++ b/lib/pure/ioselectors/ioselectors_kqueue.nim @@ -0,0 +1,439 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements BSD kqueue(). + +import posix, times, kqueue + +const + # Maximum number of cached changes. + MAX_KQUEUE_CHANGE_EVENTS = 64 + # Maximum number of events that can be returned. + MAX_KQUEUE_RESULT_EVENTS = 64 + +when defined(macosx) or defined(freebsd): + when defined(macosx): + const MAX_DESCRIPTORS_ID = 29 # KERN_MAXFILESPERPROC (MacOS) + else: + const MAX_DESCRIPTORS_ID = 27 # KERN_MAXFILESPERPROC (FreeBSD) + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, + newp: pointer, newplen: int): cint + {.importc: "sysctl",header: """#include + #include """} +elif defined(netbsd) or defined(openbsd): + # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using + # KERN_MAXFILES, because KERN_MAXFILES is always bigger, + # than KERN_MAXFILESPERPROC. + const MAX_DESCRIPTORS_ID = 7 # KERN_MAXFILES + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, + newp: pointer, newplen: int): cint + {.importc: "sysctl",header: """#include + #include """} + +when hasThreadSupport: + type + SelectorImpl[T] = object + kqFD : cint + maxFD : int + changesTable: array[MAX_KQUEUE_CHANGE_EVENTS, KEvent] + changesCount: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + changesLock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + kqFD : cint + maxFD : int + changesTable: array[MAX_KQUEUE_CHANGE_EVENTS, KEvent] + changesCount: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint +# SelectEvent is declared as `ptr` to be placed in `shared memory`, +# so you can share one SelectEvent handle between threads. +type SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] = + var maxFD = 0.cint + var size = sizeof(cint) + var namearr = [1.cint, MAX_DESCRIPTORS_ID.cint] + # Obtain maximum number of file descriptors for process + if sysctl(addr(namearr[0]), 2, cast[pointer](addr maxFD), addr size, + nil, 0) != 0: + raiseOsError(osLastError()) + + var kqFD = kqueue() + if kqFD < 0: + raiseOsError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.kqFD = kqFD + result.maxFD = maxFD.int + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + initLock(result.changesLock) + else: + result = Selector[T]() + result.kqFD = kqFD + result.maxFD = maxFD.int + result.fds = newSeq[SelectorKey[T]](maxFD) + +proc close*[T](s: Selector[T]) = + if posix.close(s.kqFD) != 0: + raiseOSError(osLastError()) + when hasThreadSupport: + deinitLock(s.changesLock) + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rfd)) + discard posix.close(cint(ev.wfd)) + deallocShared(cast[pointer](ev)) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +when hasThreadSupport: + template withChangeLock[T](s: Selector[T], body: untyped) = + acquire(s.changesLock) + {.locks: [s.changesLock].}: + try: + body + finally: + release(s.changesLock) +else: + template withChangeLock(s, body: untyped) = + body + +template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, + nflags: cushort, nfflags: cuint, ndata: int, + nudata: pointer) = + mixin withChangeLock + s.withChangeLock(): + s.changesTable[s.changesCount] = KEvent(ident: nident, + filter: nfilter, flags: nflags, + fflags: nfflags, data: ndata, + udata: nudata) + inc(s.changesCount) + if s.changesCount == MAX_KQUEUE_CHANGE_EVENTS: + if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + nil, 0, nil) == -1: + raiseOSError(osLastError()) + s.changesCount = 0 + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: + if Event.Read in events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if Event.Write in events: + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + pkey.events = events + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + let events = if oneshot: {Event.Timer, Event.Oneshot} else: {Event.Timer} + let flags: cushort = if oneshot: EV_ONESHOT or EV_ADD else: EV_ADD + + s.setKey(fdi, fdi, events, 0, data) + # EVFILT_TIMER on Open/Net(BSD) has granularity of only milliseconds, + # but MacOS and FreeBSD allow use `0` as `fflags` to use milliseconds + # too + modifyKQueue(s, fdi.uint, EVFILT_TIMER, flags, 0, cint(timeout), nil) + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + s.setKey(fdi, signal, {Event.Signal}, signal, data) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + # to be compatible with linux semantic we need to "eat" signals + posix.signal(cint(signal), SIG_IGN) + modifyKQueue(s, signal.uint, EVFILT_SIGNAL, EV_ADD, 0, 0, + cast[pointer](fdi)) + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var kflags: cushort = EV_ONESHOT or EV_ADD + setKey(s, fdi, pid, {Event.Process, Event.Oneshot}, pid, data) + modifyKQueue(s, pid.uint, EVFILT_PROC, kflags, NOTE_EXIT, 0, + cast[pointer](fdi)) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = ev.rfd.int + doAssert(s.fds[fdi].ident == 0) + setKey(s, fdi, fdi, {Event.User}, 0, data) + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + if Event.Read in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if Event.Write in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Timer in pkey.events: + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_TIMER, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Signal in pkey.events: + var nmask, omask: Sigset + var signal = cint(pkey.param) + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, signal) + unblockSignals(nmask, omask) + posix.signal(signal, SIG_DFL) + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_SIGNAL, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Process in pkey.events: + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_PROC, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.User in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + pkey.ident = 0 + pkey.events = {} + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + +proc flush*[T](s: Selector[T]) = + s.withChangeLock(): + var tv = Timespec() + if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + nil, 0, addr tv) == -1: + raiseOSError(osLastError()) + s.changesCount = 0 + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var + tv: Timespec + resTable: array[MAX_KQUEUE_RESULT_EVENTS, KEvent] + ptv = addr tv + maxres = MAX_KQUEUE_RESULT_EVENTS + + if timeout != -1: + if timeout >= 1000: + tv.tv_sec = (timeout div 1_000).Time + tv.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tv.tv_sec = 0.Time + tv.tv_nsec = timeout * 1_000_000 + else: + ptv = nil + + if maxres > len(results): + maxres = len(results) + + var count = 0 + s.withChangeLock(): + count = kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + addr(resTable[0]), cint(maxres), ptv) + s.changesCount = 0 + + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var pkey: ptr SelectorKey[T] + while i < count: + let kevent = addr(resTable[i]) + if (kevent.flags and EV_ERROR) == 0: + case kevent.filter: + of EVFILT_READ: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Read} + if Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(kevent.ident.cint, addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + # someone already consumed event data + inc(i) + continue + else: + raiseOSError(osLastError()) + pkey.key.events = {Event.User} + of EVFILT_WRITE: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Write} + of EVFILT_TIMER: + pkey = addr(s.fds[kevent.ident.int]) + if Event.Oneshot in pkey.events: + if posix.close(cint(pkey.ident)) == -1: + raiseOSError(osLastError()) + pkey.ident = 0 + pkey.events = {} + dec(s.count) + pkey.key.events = {Event.Timer} + of EVFILT_VNODE: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Vnode} + of EVFILT_SIGNAL: + pkey = addr(s.fds[cast[int](kevent.udata)]) + pkey.key.events = {Event.Signal} + of EVFILT_PROC: + pkey = addr(s.fds[cast[int](kevent.udata)]) + if posix.close(cint(pkey.ident)) == -1: + raiseOSError(osLastError()) + pkey.ident = 0 + pkey.events = {} + dec(s.count) + pkey.key.events = {Event.Process} + else: + raise newException(ValueError, "Unsupported kqueue filter in queue") + + if (kevent.flags and EV_EOF) != 0: + pkey.key.events.incl(Event.Error) + + results[k] = pkey.key + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_KQUEUE_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselectors/ioselectors_poll.nim b/lib/pure/ioselectors/ioselectors_poll.nim new file mode 100644 index 0000000000..d2a0a12739 --- /dev/null +++ b/lib/pure/ioselectors/ioselectors_poll.nim @@ -0,0 +1,295 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix poll(). + +import posix, times + +# Maximum number of events that can be returned +const MAX_POLL_RESULT_EVENTS = 64 + +when hasThreadSupport: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: ptr SharedArray[SelectorKey[T]] + pollfds: ptr SharedArray[TPollFd] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: seq[SelectorKey[T]] + pollfds: seq[TPollFd] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint + SelectEvent* = ptr SelectEventImpl + +var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", + header: "".}: cint +type + rlimit {.importc: "struct rlimit", + header: "", pure, final.} = object + rlim_cur: int + rlim_max: int +proc getrlimit(resource: cint, rlp: var rlimit): cint + {.importc: "getrlimit",header: "".} + +when hasThreadSupport: + template withPollLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withPollLock(s, body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + var a = rlimit() + if getrlimit(RLIMIT_NOFILE, a) != 0: + raiseOsError(osLastError()) + var maxFD = int(a.rlim_max) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.pollfds = allocSharedArray[TPollFd](maxFD) + initLock(result.lock) + else: + result = Selector[T]() + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + result.pollfds = newSeq[TPollFd](maxFD) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deinitLock(s.lock) + deallocSharedArray(s.fds) + deallocSharedArray(s.pollfds) + deallocShared(cast[pointer](s)) + +template pollAdd[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + s.pollfds[s.pollcnt].fd = cint(sock) + s.pollfds[s.pollcnt].events = pollev + inc(s.count) + inc(s.pollcnt) + +template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var i = 0 + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + s.pollfds[i].events = pollev + break + inc(i) + + if i == s.pollcnt: + raise newException(ValueError, "Descriptor is not registered in queue") + +template pollRemove[T](s: Selector[T], sock: cint) = + withPollLock(s): + var i = 0 + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + if i == s.pollcnt - 1: + s.pollfds[i].fd = 0 + s.pollfds[i].events = 0 + s.pollfds[i].revents = 0 + else: + while i < (s.pollcnt - 1): + s.pollfds[i].fd = s.pollfds[i + 1].fd + s.pollfds[i].events = s.pollfds[i + 1].events + inc(i) + break + inc(i) + dec(s.pollcnt) + dec(s.count) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + var fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: s.pollAdd(fdi.cint, events) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if pkey.events == {}: + s.pollAdd(fd.cint, events) + else: + if events != {}: + s.pollUpdate(fd.cint, events) + else: + s.pollRemove(fd.cint) + pkey.events = events + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + var fdi = int(ev.rfd) + doAssert(s.fds[fdi].ident == 0) + var events = {Event.User} + setKey(s, fdi, fdi, events, 0, data) + events.incl(Event.Read) + s.pollAdd(fdi.cint, events) + +proc flush*[T](s: Selector[T]) = discard + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + pkey.ident = 0 + pkey.events = {} + s.pollRemove(fdi.cint) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + s.pollRemove(fdi.cint) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rfd)) + discard posix.close(cint(ev.wfd)) + deallocShared(cast[pointer](ev)) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var maxres = MAX_POLL_RESULT_EVENTS + if maxres > len(results): + maxres = len(results) + + s.withPollLock(): + let count = posix.poll(addr(s.pollfds[0]), Tnfds(s.pollcnt), timeout) + if count < 0: + result = 0 + let err = osLastError() + if err.cint == EINTR: + discard + else: + raiseOSError(osLastError()) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var rindex = 0 + while (i < s.pollcnt) and (k < count) and (rindex < maxres): + let revents = s.pollfds[i].revents + if revents != 0: + let fd = s.pollfds[i].fd + var skey = addr(s.fds[fd]) + skey.key.events = {} + + if (revents and POLLIN) != 0: + skey.key.events.incl(Event.Read) + if Event.User in skey.events: + var data: uint64 = 0 + if posix.read(fd, addr data, sizeof(int)) != sizeof(int): + let err = osLastError() + if err != OSErrorCode(EAGAIN): + raiseOSError(osLastError()) + else: + # someone already consumed event data + inc(i) + continue + skey.key.events = {Event.User} + if (revents and POLLOUT) != 0: + skey.key.events.incl(Event.Write) + if (revents and POLLERR) != 0 or (revents and POLLHUP) != 0 or + (revents and POLLNVAL) != 0: + skey.key.events.incl(Event.Error) + results[rindex] = skey.key + s.pollfds[i].revents = 0 + inc(rindex) + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_POLL_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselectors/ioselectors_select.nim b/lib/pure/ioselectors/ioselectors_select.nim new file mode 100644 index 0000000000..f8099f9a00 --- /dev/null +++ b/lib/pure/ioselectors/ioselectors_select.nim @@ -0,0 +1,416 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix and Windows select(). + +import times, nativesockets + +when defined(windows): + import winlean + when defined(gcc): + {.passL: "-lws2_32".} + elif defined(vcc): + {.passL: "ws2_32.lib".} + const platformHeaders = """#include + #include """ + const EAGAIN = WSAEWOULDBLOCK +else: + const platformHeaders = """#include + #include + #include + #include """ +type + Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object +var + FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint + +proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} +proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} +proc IOFD_ZERO(fdset: ptr Fdset) + {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} + +when defined(windows): + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.stdcall, importc: "select", header: platformHeaders.} +else: + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.cdecl, importc: "select", header: platformHeaders.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rsock: SocketHandle + wsock: SocketHandle + SelectEvent* = ptr SelectEventImpl + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.fds = allocSharedArray[SelectorKey[T]](FD_SETSIZE) + initLock result.lock + else: + result = Selector[T]() + result.fds = newSeq[SelectorKey[T]](FD_SETSIZE) + + IOFD_ZERO(addr result.rSet) + IOFD_ZERO(addr result.wSet) + IOFD_ZERO(addr result.eSet) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +when defined(windows): + proc newSelectEvent*(): SelectEvent = + var ssock = newNativeSocket() + var wsock = newNativeSocket() + var rsock: SocketHandle = INVALID_SOCKET + var saddr = Sockaddr_in() + + saddr.sin_family = winlean.AF_INET + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(ssock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseOSError(osLastError()) + + if winlean.listen(ssock, 1) == -1: + raiseOSError(osLastError()) + + var namelen = sizeof(saddr).SockLen + if getsockname(ssock, cast[ptr SockAddr](addr(saddr)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + saddr.sin_addr.s_addr = 0x0100007F + if winlean.connect(wsock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) == -1: + raiseOSError(osLastError()) + namelen = sizeof(saddr).SockLen + rsock = winlean.accept(ssock, cast[ptr SockAddr](addr(saddr)), + cast[ptr SockLen](addr(namelen))) + if rsock == SocketHandle(-1): + raiseOSError(osLastError()) + + if winlean.closesocket(ssock) == -1: + raiseOSError(osLastError()) + + var mode = clong(1) + if ioctlsocket(rsock, FIONBIO, addr(mode)) == -1: + raiseOSError(osLastError()) + mode = clong(1) + if ioctlsocket(wsock, FIONBIO, addr(mode)) == -1: + raiseOSError(osLastError()) + + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = rsock + result.wsock = wsock + + proc setEvent*(ev: SelectEvent) = + var data: int = 1 + if winlean.send(ev.wsock, cast[pointer](addr data), + cint(sizeof(int)), 0) != sizeof(int): + raiseOSError(osLastError()) + + proc close*(ev: SelectEvent) = + discard winlean.closesocket(ev.rsock) + discard winlean.closesocket(ev.wsock) + deallocShared(cast[pointer](ev)) + +else: + proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = SocketHandle(fds[0]) + result.wsock = SocketHandle(fds[1]) + + proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(cint(ev.wsock), addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + + proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rsock)) + discard posix.close(cint(ev.wsock)) + deallocShared(cast[pointer](ev)) + +proc setKey[T](s: Selector[T], fd: SocketHandle, events: set[Event], data: T) = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == 0: + var pkey = addr(s.fds[i]) + pkey.ident = fdi + pkey.events = events + pkey.key.fd = fd.int + pkey.key.events = {} + pkey.key.data = data + break + inc(i) + if i == FD_SETSIZE: + raise newException(ValueError, "Maximum numbers of fds exceeded") + +proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + result = addr(s.fds[i]) + break + inc(i) + doAssert(i < FD_SETSIZE, "Descriptor not registered in queue") + +proc delKey[T](s: Selector[T], fd: SocketHandle) = + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fd.int: + s.fds[i].ident = 0 + s.fds[i].events = {} + break + inc(i) + doAssert(i < FD_SETSIZE, "Descriptor not registered in queue") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + when not defined(windows): + let fdi = int(fd) + s.withSelectLock(): + s.setKey(fd, events, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + if Event.Read in events: + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if Event.Write in events: + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + when not defined(windows): + let fdi = int(ev.rsock) + s.withSelectLock(): + s.setKey(ev.rsock, {Event.User}, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + IOFD_SET(ev.rsock, addr s.rSet) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + s.withSelectLock(): + var pkey = s.getKey(fd) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: SocketHandle) = + s.withSelectLock(): + var pkey = s.getKey(fd) + if Event.Read in pkey.events: + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if Event.Write in pkey.events: + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + s.delKey(fd) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fd = ev.rsock + s.withSelectLock(): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + s.delKey(fd) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var tv = Timeval() + var ptv = addr tv + var rset, wset, eset: FdSet + + if timeout != -1: + tv.tv_sec = timeout.int32 div 1_000 + tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 + else: + ptv = nil + + s.withSelectLock(): + rset = s.rSet + wset = s.wSet + eset = s.eSet + + var count = ioselect(cint(s.maxFD) + 1, addr(rset), addr(wset), + addr(eset), ptv) + if count < 0: + result = 0 + when defined(windows): + raiseOSError(osLastError()) + else: + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + var rindex = 0 + var i = 0 + var k = 0 + + while (i < FD_SETSIZE) and (k < count): + if s.fds[i].ident != 0: + var flag = false + var pkey = addr(s.fds[i]) + pkey.key.events = {} + let fd = SocketHandle(pkey.ident) + if IOFD_ISSET(fd, addr rset) != 0: + if Event.User in pkey.events: + var data: uint64 = 0 + if recv(fd, cast[pointer](addr(data)), + sizeof(uint64).cint, 0) != sizeof(uint64): + let err = osLastError() + if cint(err) != EAGAIN: + raiseOSError(err) + else: + inc(i) + inc(k) + continue + else: + flag = true + pkey.key.events = {Event.User} + else: + flag = true + pkey.key.events = {Event.Read} + if IOFD_ISSET(fd, addr wset) != 0: + pkey.key.events.incl(Event.Write) + if IOFD_ISSET(fd, addr eset) != 0: + pkey.key.events.incl(Event.Error) + flag = true + if flag: + results[rindex] = pkey.key + inc(rindex) + inc(k) + inc(i) + result = rindex + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](FD_SETSIZE) + var count = selectInto(s, timeout, result) + result.setLen(count) + +proc flush*[T](s: Selector[T]) = discard + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin withSelectLock + s.withSelectLock(): + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].key.data) + break + inc(i) + if i != FD_SETSIZE: + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body1, body2: untyped) = + mixin withSelectLock + s.withSelectLock(): + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].key.data) + break + inc(i) + if i != FD_SETSIZE: + body1 + else: + body2 diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim new file mode 100644 index 0000000000..162ac5e08a --- /dev/null +++ b/lib/upcoming/asyncdispatch.nim @@ -0,0 +1,2154 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +include "system/inclrtl" + +import os, oids, tables, strutils, macros, times, heapqueue + +import nativesockets, net, queues + +export Port, SocketFlag + +#{.injectStmt: newGcInvariant().} + +## AsyncDispatch +## ************* +## +## This module implements asynchronous IO. This includes a dispatcher, +## a ``Future`` type implementation, and an ``async`` macro which allows +## asynchronous code to be written in a synchronous style with the ``await`` +## keyword. +## +## The dispatcher acts as a kind of event loop. You must call ``poll`` on it +## (or a function which does so for you such as ``waitFor`` or ``runForever``) +## in order to poll for any outstanding events. The underlying implementation +## is based on epoll on Linux, IO Completion Ports on Windows and select on +## other operating systems. +## +## The ``poll`` function will not, on its own, return any events. Instead +## an appropriate ``Future`` object will be completed. A ``Future`` is a +## type which holds a value which is not yet available, but which *may* be +## available in the future. You can check whether a future is finished +## by using the ``finished`` function. When a future is finished it means that +## either the value that it holds is now available or it holds an error instead. +## The latter situation occurs when the operation to complete a future fails +## with an exception. You can distinguish between the two situations with the +## ``failed`` function. +## +## Future objects can also store a callback procedure which will be called +## automatically once the future completes. +## +## Futures therefore can be thought of as an implementation of the proactor +## pattern. In this +## pattern you make a request for an action, and once that action is fulfilled +## a future is completed with the result of that action. Requests can be +## made by calling the appropriate functions. For example: calling the ``recv`` +## function will create a request for some data to be read from a socket. The +## future which the ``recv`` function returns will then complete once the +## requested amount of data is read **or** an exception occurs. +## +## Code to read some data from a socket may look something like this: +## +## .. code-block::nim +## var future = socket.recv(100) +## future.callback = +## proc () = +## echo(future.read) +## +## All asynchronous functions returning a ``Future`` will not block. They +## will not however return immediately. An asynchronous function will have +## code which will be executed before an asynchronous request is made, in most +## cases this code sets up the request. +## +## In the above example, the ``recv`` function will return a brand new +## ``Future`` instance once the request for data to be read from the socket +## is made. This ``Future`` instance will complete once the requested amount +## of data is read, in this case it is 100 bytes. The second line sets a +## callback on this future which will be called once the future completes. +## All the callback does is write the data stored in the future to ``stdout``. +## The ``read`` function is used for this and it checks whether the future +## completes with an error for you (if it did it will simply raise the +## error), if there is no error however it returns the value of the future. +## +## Asynchronous procedures +## ----------------------- +## +## Asynchronous procedures remove the pain of working with callbacks. They do +## this by allowing you to write asynchronous code the same way as you would +## write synchronous code. +## +## An asynchronous procedure is marked using the ``{.async.}`` pragma. +## When marking a procedure with the ``{.async.}`` pragma it must have a +## ``Future[T]`` return type or no return type at all. If you do not specify +## a return type then ``Future[void]`` is assumed. +## +## Inside asynchronous procedures ``await`` can be used to call any +## procedures which return a +## ``Future``; this includes asynchronous procedures. When a procedure is +## "awaited", the asynchronous procedure it is awaited in will +## suspend its execution +## until the awaited procedure's Future completes. At which point the +## asynchronous procedure will resume its execution. During the period +## when an asynchronous procedure is suspended other asynchronous procedures +## will be run by the dispatcher. +## +## The ``await`` call may be used in many contexts. It can be used on the right +## hand side of a variable declaration: ``var data = await socket.recv(100)``, +## in which case the variable will be set to the value of the future +## automatically. It can be used to await a ``Future`` object, and it can +## be used to await a procedure returning a ``Future[void]``: +## ``await socket.send("foobar")``. +## +## Discarding futures +## ------------------ +## +## Futures should **never** be discarded. This is because they may contain +## errors. If you do not care for the result of a Future then you should +## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. +## +## Examples +## -------- +## +## For examples take a look at the documentation for the modules implementing +## asynchronous IO. A good place to start is the +## `asyncnet module `_. +## +## Limitations/Bugs +## ---------------- +## +## * The effect system (``raises: []``) does not work with async procedures. +## * Can't await in a ``except`` body +## * Forward declarations for async procs are broken, +## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. +## * FutureVar[T] needs to be completed manually. + +# TODO: Check if yielded future is nil and throw a more meaningful exception + +# -- Futures + +type + FutureBase* = ref object of RootObj ## Untyped future. + cb: proc () {.closure,gcsafe.} + finished: bool + error*: ref Exception ## Stored exception + errorStackTrace*: string + when not defined(release): + stackTrace: string ## For debugging purposes only. + id: int + fromProc: string + + Future*[T] = ref object of FutureBase ## Typed future. + value: T ## Stored value + + FutureVar*[T] = distinct Future[T] + + FutureError* = object of Exception + cause*: FutureBase + +{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} + +when not defined(release): + var currentID = 0 + +proc callSoon*(cbproc: proc ()) {.gcsafe.} + +proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = + ## Creates a new future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + new(result) + result.finished = false + when not defined(release): + result.stackTrace = getStackTrace() + result.id = currentID + result.fromProc = fromProc + currentID.inc() + +proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = + ## Create a new ``FutureVar``. This Future type is ideally suited for + ## situations where you want to avoid unnecessary allocations of Futures. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + result = FutureVar[T](newFuture[T](fromProc)) + +proc clean*[T](future: FutureVar[T]) = + ## Resets the ``finished`` status of ``future``. + Future[T](future).finished = false + Future[T](future).error = nil + +proc checkFinished[T](future: Future[T]) = + ## Checks whether `future` is finished. If it is then raises a + ## ``FutureError``. + when not defined(release): + if future.finished: + var msg = "" + msg.add("An attempt was made to complete a Future more than once. ") + msg.add("Details:") + msg.add("\n Future ID: " & $future.id) + msg.add("\n Created in proc: " & future.fromProc) + msg.add("\n Stack trace to moment of creation:") + msg.add("\n" & indent(future.stackTrace.strip(), 4)) + when T is string: + msg.add("\n Contents (string): ") + msg.add("\n" & indent(future.value.repr, 4)) + msg.add("\n Stack trace to moment of secondary completion:") + msg.add("\n" & indent(getStackTrace().strip(), 4)) + var err = newException(FutureError, msg) + err.cause = future + raise err + +proc complete*[T](future: Future[T], val: T) = + ## Completes ``future`` with value ``val``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.value = val + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*(future: Future[void]) = + ## Completes a void ``future``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*[T](future: FutureVar[T]) = + ## Completes a ``FutureVar``. + template fut: expr = Future[T](future) + checkFinished(fut) + assert(fut.error == nil) + fut.finished = true + if fut.cb != nil: + fut.cb() + +proc fail*[T](future: Future[T], error: ref Exception) = + ## Completes ``future`` with ``error``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + future.finished = true + future.error = error + future.errorStackTrace = + if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) + if future.cb != nil: + future.cb() + else: + # This is to prevent exceptions from being silently ignored when a future + # is discarded. + # TODO: This may turn out to be a bad idea. + # Turns out this is a bad idea. + #raise error + discard + +proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + ## + ## **Note**: You most likely want the other ``callback`` setter which + ## passes ``future`` as a param to the callback. + future.cb = cb + if future.finished: + callSoon(future.cb) + +proc `callback=`*[T](future: Future[T], + cb: proc (future: Future[T]) {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + future.callback = proc () = cb(future) + +proc injectStacktrace[T](future: Future[T]) = + # TODO: Come up with something better. + when not defined(release): + var msg = "" + msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") + + if not future.errorStackTrace.isNil and future.errorStackTrace != "": + msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) + else: + msg.add("\n Empty or nil stack trace.") + future.error.msg.add(msg) + +proc read*[T](future: Future[T]): T = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``ValueError`` exception. + ## + ## If the result of the future is an error then that error will be raised. + if future.finished: + if future.error != nil: + injectStacktrace(future) + raise future.error + when T isnot void: + return future.value + else: + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") + +proc readError*[T](future: Future[T]): ref Exception = + ## Retrieves the exception stored in ``future``. + ## + ## An ``ValueError`` exception will be thrown if no exception exists + ## in the specified Future. + if future.error != nil: return future.error + else: + raise newException(ValueError, "No error in future.") + +proc mget*[T](future: FutureVar[T]): var T = + ## Returns a mutable value stored in ``future``. + ## + ## Unlike ``read``, this function will not raise an exception if the + ## Future has not been finished. + result = Future[T](future).value + +proc finished*[T](future: Future[T]): bool = + ## Determines whether ``future`` has completed. + ## + ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. + future.finished + +proc failed*(future: FutureBase): bool = + ## Determines whether ``future`` completed with an error. + return future.error != nil + +proc asyncCheck*[T](future: Future[T]) = + ## Sets a callback on ``future`` which raises an exception if the future + ## finished with an error. + ## + ## This should be used instead of ``discard`` to discard void futures. + future.callback = + proc () = + if future.failed: + injectStacktrace(future) + raise future.error + +proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once both ``fut1`` and ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`and`") + fut1.callback = + proc () = + if fut2.finished: retFuture.complete() + fut2.callback = + proc () = + if fut1.finished: retFuture.complete() + return retFuture + +proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once either ``fut1`` or ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`or`") + proc cb() = + if not retFuture.finished: retFuture.complete() + fut1.callback = cb + fut2.callback = cb + return retFuture + +proc all*[T](futs: varargs[Future[T]]): auto = + ## Returns a future which will complete once + ## all futures in ``futs`` complete. + ## + ## If the awaited futures are not ``Future[void]``, the returned future + ## will hold the values of all awaited futures in a sequence. + ## + ## If the awaited futures *are* ``Future[void]``, + ## this proc returns ``Future[void]``. + + when T is void: + var + retFuture = newFuture[void]("asyncdispatch.all") + completedFutures = 0 + + let totalFutures = len(futs) + + for fut in futs: + fut.callback = proc(f: Future[T]) = + inc(completedFutures) + + if completedFutures == totalFutures: + retFuture.complete() + + return retFuture + + else: + var + retFuture = newFuture[seq[T]]("asyncdispatch.all") + retValues = newSeq[T](len(futs)) + completedFutures = 0 + + for i, fut in futs: + proc setCallback(i: int) = + fut.callback = proc(f: Future[T]) = + retValues[i] = f.read() + inc(completedFutures) + + if completedFutures == len(retValues): + retFuture.complete(retValues) + + setCallback(i) + + return retFuture + +type + PDispatcherBase = ref object of RootRef + timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] + callbacks: Queue[proc ()] + +proc processTimers(p: PDispatcherBase) {.inline.} = + while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt: + p.timers.pop().fut.complete() + +proc processPendingCallbacks(p: PDispatcherBase) = + while p.callbacks.len > 0: + var cb = p.callbacks.dequeue() + cb() + +proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = + # If dispatcher has active timers this proc returns the timeout + # of the nearest timer. Returns `timeout` otherwise. + result = timeout + if p.timers.len > 0: + let timerTimeout = p.timers[0].finishAt + let curTime = epochTime() + if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: + result = int((timerTimeout - curTime) * 1000) + if result < 0: result = 0 + +when defined(windows) or defined(nimdoc): + import winlean, sets, hashes + type + CompletionKey = ULONG_PTR + + CompletionData* = object + fd*: AsyncFD # TODO: Rename this. + cb*: proc (fd: AsyncFD, bytesTransferred: Dword, + errcode: OSErrorCode) {.closure,gcsafe.} + cell*: ForeignCell # we need this `cell` to protect our `cb` environment, + # when using RegisterWaitForSingleObject, because + # waiting is done in different thread. + + PDispatcher* = ref object of PDispatcherBase + ioPort: Handle + handles: HashSet[AsyncFD] + + CustomOverlapped = object of OVERLAPPED + data*: CompletionData + + PCustomOverlapped* = ref CustomOverlapped + + AsyncFD* = distinct int + + PostCallbackData = object + ioPort: Handle + handleFd: AsyncFD + waitFd: Handle + ovl: PCustomOverlapped + PostCallbackDataPtr = ptr PostCallbackData + + AsyncEventImpl = object + hEvent: Handle + hWaiter: Handle + pcd: PostCallbackDataPtr + AsyncEvent* = ptr AsyncEventImpl + + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, + TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} + + proc hash(x: AsyncFD): Hash {.borrow.} + proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + ## Creates a new Dispatcher instance. + new result + result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) + result.handles = initSet[AsyncFD]() + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + ## Retrieves the global thread-local dispatcher. + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc register*(fd: AsyncFD) = + ## Registers ``fd`` with the dispatcher. + let p = getGlobalDispatcher() + if createIoCompletionPort(fd.Handle, p.ioPort, + cast[CompletionKey](fd), 1) == 0: + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc verifyPresence(fd: AsyncFD) = + ## Ensures that file descriptor has been registered with the dispatcher. + let p = getGlobalDispatcher() + if fd notin p.handles: + raise newException(ValueError, + "Operation performed on a socket which has not been registered with" & + " the dispatcher yet.") + + proc poll*(timeout = 500) = + ## Waits for completion events and processes them. + let p = getGlobalDispatcher() + if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: + raise newException(ValueError, + "No handles or timers registered in dispatcher.") + + let at = p.adjustedTimeout(timeout) + var llTimeout = + if at == -1: winlean.INFINITE + else: at.int32 + + var lpNumberOfBytesTransferred: Dword + var lpCompletionKey: ULONG_PTR + var customOverlapped: PCustomOverlapped + let res = getQueuedCompletionStatus(p.ioPort, + addr lpNumberOfBytesTransferred, addr lpCompletionKey, + cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + if res: + # This is useful for ensuring the reliability of the overlapped struct. + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + + GC_unref(customOverlapped) + else: + let errCode = osLastError() + if customOverlapped != nil: + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) + else: + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + discard + else: raiseOSError(errCode) + + # Timer processing. + processTimers(p) + # Callback queue processing + processPendingCallbacks(p) + + var acceptEx*: WSAPROC_ACCEPTEX + var connectEx*: WSAPROC_CONNECTEX + var getAcceptExSockAddrs*: WSAPROC_GETACCEPTEXSOCKADDRS + + proc initPointer(s: SocketHandle, fun: var pointer, guid: var GUID): bool = + # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c + var bytesRet: Dword + fun = nil + result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, + sizeof(GUID).Dword, addr fun, sizeof(pointer).Dword, + addr bytesRet, nil, nil) == 0 + + proc initAll() = + let dummySock = newNativeSocket() + if dummySock == INVALID_SOCKET: + raiseOSError(osLastError()) + var fun: pointer = nil + if not initPointer(dummySock, fun, WSAID_CONNECTEX): + raiseOSError(osLastError()) + connectEx = cast[WSAPROC_CONNECTEX](fun) + if not initPointer(dummySock, fun, WSAID_ACCEPTEX): + raiseOSError(osLastError()) + acceptEx = cast[WSAPROC_ACCEPTEX](fun) + if not initPointer(dummySock, fun, WSAID_GETACCEPTEXSOCKADDRS): + raiseOSError(osLastError()) + getAcceptExSockAddrs = cast[WSAPROC_GETACCEPTEXSOCKADDRS](fun) + close(dummySock) + + proc connect*(socket: AsyncFD, address: string, port: Port, + domain = nativesockets.AF_INET): Future[void] = + ## Connects ``socket`` to server at ``address:port``. + ## + ## Returns a ``Future`` which will complete when the connection succeeds + ## or an error occurs. + verifyPresence(socket) + var retFuture = newFuture[void]("connect") + # Apparently ``ConnectEx`` expects the socket to be initially bound: + var saddr: Sockaddr_in + saddr.sin_family = int16(toInt(domain)) + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(socket.SocketHandle, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseOSError(osLastError()) + + var aiList = getAddrInfo(address, port, domain) + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + # "the OVERLAPPED structure must remain valid until the I/O completes" + # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + var ret = connectEx(socket.SocketHandle, it.ai_addr, + sizeof(Sockaddr_in).cint, nil, 0, nil, + cast[POVERLAPPED](ol)) + if ret: + # Request to connect completed immediately. + success = true + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + break + else: + lastError = osLastError() + if lastError.int32 == ERROR_IO_PENDING: + # In this case ``ol`` will be deallocated in ``poll``. + success = true + break + else: + GC_unref(ol) + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: AsyncFD, size: int, + flags = {SocketFlag.SafeDisconn}): Future[string] = + ## Reads **up to** ``size`` bytes from ``socket``. Returned future will + ## complete once all the data requested is read, a part of the data has been + ## read, or the socket has disconnected in which case the future will + ## complete with a value of ``""``. + ## + ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + + + # Things to note: + # * When WSARecv completes immediately then ``bytesReceived`` is very + # unreliable. + # * Still need to implement message-oriented socket disconnection, + # '\0' in the message currently signifies a socket disconnect. Who + # knows what will happen when someone sends that to our socket. + verifyPresence(socket) + assert SocketFlag.Peek notin flags, "Peek not supported on Windows." + + var retFuture = newFuture[string]("recv") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](alloc0(size)) + dataBuf.len = size.ULONG + + var bytesReceived: Dword + var flagsio = flags.toOSFlags().Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete("") + else: + var data = newString(bytesCount) + assert bytesCount <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesCount) + retFuture.complete($data) + else: + if flags.isDisconnectionError(errcode): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + if dataBuf.buf != nil: + dealloc dataBuf.buf + dataBuf.buf = nil + ) + + let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + if dataBuf.buf != nil: + dealloc dataBuf.buf + dataBuf.buf = nil + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + var data = newString(bytesReceived) + assert bytesReceived <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) + retFuture.complete($data) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete("") + return retFuture + + proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must + ## at least be of that size. Returned future will complete once all the + ## data requested is read, a part of the data has been read, or the socket + ## has disconnected in which case the future will complete with a value of + ## ``0``. + ## + ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + + + # Things to note: + # * When WSARecv completes immediately then ``bytesReceived`` is very + # unreliable. + # * Still need to implement message-oriented socket disconnection, + # '\0' in the message currently signifies a socket disconnect. Who + # knows what will happen when someone sends that to our socket. + verifyPresence(socket) + assert SocketFlag.Peek notin flags, "Peek not supported on Windows." + + var retFuture = newFuture[int]("recvInto") + + #buf[] = '\0' + var dataBuf: TWSABuf + dataBuf.buf = buf + dataBuf.len = size.ULONG + + var bytesReceived: Dword + var flagsio = flags.toOSFlags().Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete(0) + else: + retFuture.complete(bytesCount) + else: + if flags.isDisconnectionError(errcode): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + if dataBuf.buf != nil: + dataBuf.buf = nil + ) + + let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + if dataBuf.buf != nil: + dataBuf.buf = nil + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + + proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to ``socket``. The returned future will complete once all + ## data has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("send") + + var dataBuf: TWSABuf + dataBuf.buf = data # since this is not used in a callback, this is fine + dataBuf.len = data.len.ULONG + + var bytesReceived, lowFlags: Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + if flags.isDisconnectionError(errcode): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + lowFlags, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: Socklen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to specified destination ``saddr``, using + ## socket ``socket``. The returned future will complete once all data + ## has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("sendTo") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](data) + dataBuf.len = size.ULONG + var bytesSent = 0.Dword + var lowFlags = 0.Dword + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen: cint = cint(saddrLen) + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, + lowFlags, cast[ptr SockAddr](addr(staddr[0])), + stalen, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``buf``, which must + ## be at least of size ``size``, address of datagram's sender will be + ## stored into ``saddr`` and ``saddrLen``. Returned future will complete + ## once one datagram has been received, and will return size of packet + ## received. + verifyPresence(socket) + var retFuture = newFuture[int]("recvFromInto") + + var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) + + var bytesReceived = 0.Dword + var lowFlags = 0.Dword + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount <= size + retFuture.complete(bytesCount) + else: + # datagram sockets don't have disconnection, + # so we can just raise an exception + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, + addr bytesReceived, addr lowFlags, + saddr, cast[ptr cint](saddrLen), + cast[POVERLAPPED](ol), nil) + if res == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + Future[tuple[address: string, client: AsyncFD]] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection and the remote address of the client. + ## The future will complete when the connection is successfully accepted. + ## + ## The resulting client socket is automatically registered to the + ## dispatcher. + ## + ## The ``accept`` call may result in an error if the connecting socket + ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## flag is specified then this error will not be raised and instead + ## accept will be called again. + verifyPresence(socket) + var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") + + var clientSock = newNativeSocket() + if clientSock == osInvalidSocket: raiseOSError(osLastError()) + + const lpOutputLen = 1024 + var lpOutputBuf = newString(lpOutputLen) + var dwBytesReceived: Dword + let dwReceiveDataLength = 0.Dword # We don't want any data to be read. + let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16) + let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) + + template completeAccept(): stmt {.immediate, dirty.} = + var listenSock = socket + let setoptRet = setsockopt(clientSock, SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, + sizeof(listenSock).SockLen) + if setoptRet != 0: raiseOSError(osLastError()) + + var localSockaddr, remoteSockaddr: ptr SockAddr + var localLen, remoteLen: int32 + getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, + addr localSockaddr, addr localLen, + addr remoteSockaddr, addr remoteLen) + register(clientSock.AsyncFD) + # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 + retFuture.complete( + (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr), + client: clientSock.AsyncFD) + ) + + template failAccept(errcode): stmt = + if flags.isDisconnectionError(errcode): + var newAcceptFut = acceptAddr(socket, flags) + newAcceptFut.callback = + proc () = + if newAcceptFut.failed: + retFuture.fail(newAcceptFut.readError) + else: + retFuture.complete(newAcceptFut.read) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + completeAccept() + else: + failAccept(errcode) + ) + + # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx + let ret = acceptEx(socket.SocketHandle, clientSock, addr lpOutputBuf[0], + dwReceiveDataLength, + dwLocalAddressLength, + dwRemoteAddressLength, + addr dwBytesReceived, cast[POVERLAPPED](ol)) + + if not ret: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + failAccept(err) + GC_unref(ol) + else: + completeAccept() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + + return retFuture + + proc newAsyncNativeSocket*(domain, sockType, protocol: cint): AsyncFD = + ## Creates a new socket and registers it with the dispatcher implicitly. + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + register(result) + + proc newAsyncNativeSocket*(domain: Domain = nativesockets.AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): AsyncFD = + ## Creates a new socket and registers it with the dispatcher implicitly. + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + register(result) + + proc closeSocket*(socket: AsyncFD) = + ## Closes a socket and ensures that it is unregistered. + socket.SocketHandle.close() + getGlobalDispatcher().handles.excl(socket) + + proc unregister*(fd: AsyncFD) = + ## Unregisters ``fd``. + getGlobalDispatcher().handles.excl(fd) + + {.push stackTrace:off.} + proc waitableCallback(param: pointer, + timerOrWaitFired: WINBOOL): void {.stdcall.} = + var p = cast[PostCallbackDataPtr](param) + discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.Dword, + ULONG_PTR(p.handleFd), + cast[pointer](p.ovl)) + {.pop.} + + template registerWaitableEvent(mask) = + let p = getGlobalDispatcher() + var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword + var hEvent = wsaCreateEvent() + if hEvent == 0: + raiseOSError(osLastError()) + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + pcd.ioPort = p.ioPort + pcd.handleFd = fd + var ol = PCustomOverlapped() + GC_ref(ol) + + ol.data = CompletionData(fd: fd, cb: + proc(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + # we excluding our `fd` because cb(fd) can register own handler + # for this `fd` + p.handles.excl(fd) + # unregisterWait() is called before callback, because appropriate + # winsockets function can re-enable event. + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + if cb(fd): + # callback returned `true`, so we free all allocated resources + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + # pcd.ovl will be unrefed in poll(). + else: + # callback returned `false` we need to continue + if p.handles.contains(fd): + # new callback was already registered with `fd`, so we free all + # allocated resources. This happens because in callback `cb` + # addRead/addWrite was called with same `fd`. + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + else: + # we need to include `fd` again + p.handles.incl(fd) + # and register WaitForSingleObject again + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + # pcd.ovl will be unrefed in poll() + discard wsaCloseEvent(hEvent) + deallocShared(cast[pointer](pcd)) + raiseOSError(osLastError()) + else: + # we ref pcd.ovl one more time, because it will be unrefed in + # poll() + GC_ref(pcd.ovl) + ) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` + # will be signaled when appropriate `mask` events will be triggered. + if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc addRead*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for read availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addRead` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.recv() + ## or asyncdispatch.accept(), because they are using IOCP, please use + ## nativesockets.recv() and nativesockets.accept() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `read` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for write availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addWrite` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.send() + ## or asyncdispatch.connect(), because they are using IOCP, please use + ## nativesockets.send() and nativesockets.connect() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `write` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE) + + template registerWaitableHandle(p, hEvent, flags, pcd, handleCallback) = + let handleFD = AsyncFD(hEvent) + pcd.ioPort = p.ioPort + pcd.handleFd = handleFD + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: handleFD, cb: handleCallback) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(handleFD) + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Registers callback ``cb`` to be called when timer expired. + ## ``timeout`` - timeout value in milliseconds. + ## ``oneshot`` - `true`, to generate only one timeout event, `false`, to + ## generate timeout events periodically. + + doAssert(timeout > 0) + let p = getGlobalDispatcher() + + var hEvent = createEvent(nil, 1, 0, nil) + if hEvent == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + if oneshot: flags = flags or WT_EXECUTEONLYONCE + + proc timercb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + let res = cb(fd) + if res or oneshot: + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + discard closeHandle(hEvent) + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + + registerWaitableHandle(p, hEvent, flags, pcd, timercb) + + proc addProcess*(pid: int, cb: Callback) = + ## Registers callback ``cb`` to be called when process with pid ``pid`` + ## exited. + let p = getGlobalDispatcher() + let procFlags = SYNCHRONIZE + var hProcess = openProcess(procFlags, 0, pid.Dword) + if hProcess == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc proccb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + discard closeHandle(hProcess) + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + discard cb(fd) + + registerWaitableHandle(p, hProcess, flags, pcd, proccb) + + proc newEvent*(): AsyncEvent = + ## Creates new ``AsyncEvent`` object. + var sa = SECURITY_ATTRIBUTES( + nLength: sizeof(SECURITY_ATTRIBUTES).cint, + bInheritHandle: 1 + ) + var event = createEvent(addr(sa), 0'i32, 0'i32, nil) + if event == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) + + proc setEvent*(ev: AsyncEvent) = + ## Set event ``ev`` to signaled state. + if setEvent(ev.hEvent) == 0: + raiseOSError(osLastError()) + + proc close*(ev: AsyncEvent) = + ## Closes event ``ev``. + if ev.hWaiter != 0: + let p = getGlobalDispatcher() + if unregisterWait(ev.hWaiter) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + p.handles.excl(AsyncFD(ev.hEvent)) + + if closeHandle(ev.hEvent) == 0: + raiseOSError(osLastError()) + deallocShared(cast[pointer](ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Registers callback ``cb`` to be called when ``ev`` will be signaled + if ev.hWaiter != 0: + raise newException(ValueError, "Event is already registered!") + + let p = getGlobalDispatcher() + let hEvent = ev.hEvent + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if cb(fd): + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + ev.hWaiter = 0 + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + + registerWaitableHandle(p, hEvent, flags, pcd, eventcb) + ev.hWaiter = pcd.waitFd + + initAll() +else: + import ioselectors + when defined(windows): + import winlean + const + EINTR = WSAEINPROGRESS + EINPROGRESS = WSAEINPROGRESS + EWOULDBLOCK = WSAEWOULDBLOCK + EAGAIN = EINPROGRESS + MSG_NOSIGNAL = 0 + else: + from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + MSG_NOSIGNAL + + const supportedPlatform = defined(linux) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(macosx) + + type + AsyncFD* = distinct cint + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + + AsyncData = object + readCB: Callback + writeCB: Callback + + AsyncEvent = SelectEvent + + PDispatcher* = ref object of PDispatcherBase + selector: Selector[AsyncData] + {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} + + proc `==`*(x, y: AsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + new result + result.selector = newSelector[AsyncData]() + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc register*(fd: AsyncFD) = + let p = getGlobalDispatcher() + var data = AsyncData() + p.selector.registerHandle(fd.SocketHandle, {}, data) + + proc newAsyncNativeSocket*(domain: cint, sockType: cint, + protocol: cint): AsyncFD = + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + register(result) + + proc newAsyncNativeSocket*(domain: Domain = AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): AsyncFD = + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + register(result) + + proc closeSocket*(sock: AsyncFD) = + let disp = getGlobalDispatcher() + disp.selector.unregister(sock.SocketHandle) + sock.SocketHandle.close() + + proc unregister*(fd: AsyncFD) = + getGlobalDispatcher().selector.unregister(fd.SocketHandle) + + # proc unregister*(ev: AsyncEvent) = + # getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + + proc addRead*(fd: AsyncFD, cb: Callback) = + let p = getGlobalDispatcher() + withData(p.selector, fd.SocketHandle, adata) do: + adata.readCB = cb + do: + raise newException(ValueError, "File descriptor not registered.") + p.selector.updateHandle(fd.SocketHandle, {Event.Read}) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + let p = getGlobalDispatcher() + withData(p.selector, fd.SocketHandle, adata) do: + adata.writeCB = cb + do: + raise newException(ValueError, "File descriptor not registered.") + p.selector.updateHandle(fd.SocketHandle, {Event.Write}) + + proc poll*(timeout = 500) = + var keys: array[64, ReadyKey[AsyncData]] + + let p = getGlobalDispatcher() + when supportedPlatform: + let customSet = {Event.Timer, Event.Signal, Event.Process, + Event.Vnode, Event.User} + + if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: + raise newException(ValueError, + "No handles or timers registered in dispatcher.") + + if not p.selector.isEmpty(): + var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) + var i = 0 + while i < count: + var update = false + var fd = keys[i].fd.SocketHandle + let events = keys[i].events + + if Event.Read in events: + let cb = keys[i].data.readCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.readCB == cb: + adata.readCB = nil + update = true + + if Event.Write in events: + let cb = keys[i].data.writeCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.writeCB == cb: + adata.writeCB = nil + update = true + + when supportedPlatform: + if (customSet * events) != {}: + let cb = keys[i].data.readCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.readCB == cb: + adata.readCB = nil + p.selector.unregister(fd) + + if update: + var newEvents: set[Event] = {} + p.selector.withData(fd, adata) do: + if adata.readCB != nil: incl(newEvents, Event.Read) + if adata.writeCB != nil: incl(newEvents, Event.Write) + p.selector.updateHandle(fd, newEvents) + inc(i) + + # Timer processing. + processTimers(p) + # Callback queue processing + processPendingCallbacks(p) + + proc connect*(socket: AsyncFD, address: string, port: Port, + domain = AF_INET): Future[void] = + var retFuture = newFuture[void]("connect") + + proc cb(fd: AsyncFD): bool = + var ret = SocketHandle(fd).getSockOptInt(cint(SOL_SOCKET), cint(SO_ERROR)) + if ret == 0: + # We have connected. + retFuture.complete() + return true + elif ret == EINTR: + # interrupted, keep waiting + return false + else: + retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + return true + + assert getSockDomain(socket.SocketHandle) == domain + var aiList = getAddrInfo(address, port, domain) + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + var ret = connect(socket.SocketHandle, it.ai_addr, it.ai_addrlen.Socklen) + if ret == 0: + # Request to connect completed immediately. + success = true + retFuture.complete() + break + else: + lastError = osLastError() + if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: + success = true + addWrite(socket, cb) + break + else: + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: AsyncFD, size: int, + flags = {SocketFlag.SafeDisconn}): Future[string] = + var retFuture = newFuture[string]("recv") + + var readBuffer = newString(size) + + proc cb(sock: AsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, addr readBuffer[0], size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + elif res == 0: + # Disconnected + retFuture.complete("") + else: + readBuffer.setLen(res) + retFuture.complete(readBuffer) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + var retFuture = newFuture[int]("recvInto") + + proc cb(sock: AsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, buf, size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete(res) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + var retFuture = newFuture[void]("send") + + var written = 0 + + proc cb(sock: AsyncFD): bool = + result = true + let netSize = data.len-written + var d = data.cstring + let res = send(sock.SocketHandle, addr d[written], netSize.cint, + MSG_NOSIGNAL) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + written.inc(res) + if res != netSize: + result = false # We still have data to send. + else: + retFuture.complete() + # TODO: The following causes crashes. + #if not cb(socket): + addWrite(socket, cb) + return retFuture + + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: SockLen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` of size ``size`` in bytes to specified destination + ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. + ## The returned future will complete once all data has been sent. + var retFuture = newFuture[void]("sendTo") + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen = saddrLen + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + proc cb(sock: AsyncFD): bool = + result = true + let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, + cast[ptr SockAddr](addr(staddr[0])), stalen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete() + + addWrite(socket, cb) + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``data``, which must + ## be at least of size ``size`` in bytes, address of datagram's sender + ## will be stored into ``saddr`` and ``saddrLen``. Returned future will + ## complete once one datagram has been received, and will return size + ## of packet received. + var retFuture = newFuture[int]("recvFromInto") + proc cb(sock: AsyncFD): bool = + result = true + let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), + saddr, saddrLen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false + else: + retFuture.complete(res) + addRead(socket, cb) + return retFuture + + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + Future[tuple[address: string, client: AsyncFD]] = + var retFuture = newFuture[tuple[address: string, + client: AsyncFD]]("acceptAddr") + proc cb(sock: AsyncFD): bool = + result = true + var sockAddress: Sockaddr_storage + var addrLen = sizeof(sockAddress).Socklen + var client = accept(sock.SocketHandle, + cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + if client == osInvalidSocket: + let lastError = osLastError() + assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} + if lastError.int32 == EINTR: + return false + else: + if flags.isDisconnectionError(lastError): + return false + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + register(client.AsyncFD) + retFuture.complete((getAddrString(cast[ptr SockAddr](addr sockAddress)), + client.AsyncFD)) + addRead(socket, cb) + return retFuture + + when supportedPlatform: + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Start watching for timeout expiration, and then call the + ## callback ``cb``. + ## ``timeout`` - time in milliseconds, + ## ``oneshot`` - if ``true`` only one event will be dispatched, + ## if ``false`` continuous events every ``timeout`` milliseconds. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerTimer(timeout, oneshot, data) + + proc addSignal*(signal: int, cb: Callback) = + ## Start watching signal ``signal``, and when signal appears, call the + ## callback ``cb``. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerSignal(signal, data) + + proc addProcess*(pid: int, cb: Callback) = + ## Start watching for process exit with pid ``pid``, and then call + ## the callback ``cb``. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerProcess(pid, data) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates new ``AsyncEvent``. + result = AsyncEvent(ioselectors.newSelectEvent()) + + proc setEvent*(ev: AsyncEvent) = + ## Sets new ``AsyncEvent`` to signaled state. + setEvent(SelectEvent(ev)) + + proc close*(ev: AsyncEvent) = + ## Closes ``AsyncEvent`` + close(SelectEvent(ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Start watching for event ``ev``, and call callback ``cb``, when + ## ev will be set to signaled state. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerEvent(SelectEvent(ev), data) + +proc sleepAsync*(ms: int): Future[void] = + ## Suspends the execution of the current async procedure for the next + ## ``ms`` milliseconds. + var retFuture = newFuture[void]("sleepAsync") + let p = getGlobalDispatcher() + p.timers.push((epochTime() + (ms / 1000), retFuture)) + return retFuture + +proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = + ## Returns a future which will complete once ``fut`` completes or after + ## ``timeout`` milliseconds has elapsed. + ## + ## If ``fut`` completes first the returned future will hold true, + ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned + ## future will hold false. + + var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") + var timeoutFuture = sleepAsync(timeout) + fut.callback = + proc () = + if not retFuture.finished: retFuture.complete(true) + timeoutFuture.callback = + proc () = + if not retFuture.finished: retFuture.complete(false) + return retFuture + +proc accept*(socket: AsyncFD, + flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection. + ## The future will complete when the connection is successfully accepted. + var retFut = newFuture[AsyncFD]("accept") + var fut = acceptAddr(socket, flags) + fut.callback = + proc (future: Future[tuple[address: string, client: AsyncFD]]) = + assert future.finished + if future.failed: + retFut.fail(future.error) + else: + retFut.complete(future.read.client) + return retFut + +# -- Await Macro + +proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = + # Skips a nest of StmtList's. + result = node + if node[0].kind == nnkStmtList: + result = skipUntilStmtList(node[0]) + +proc skipStmtList(node: NimNode): NimNode {.compileTime.} = + result = node + if node[0].kind == nnkStmtList: + result = node[0] + +template createCb(retFutureSym, iteratorNameSym, + name: expr): stmt {.immediate.} = + var nameIterVar = iteratorNameSym + #{.push stackTrace: off.} + proc cb {.closure,gcsafe.} = + try: + if not nameIterVar.finished: + var next = nameIterVar() + if next == nil: + assert retFutureSym.finished, "Async procedure's (" & + name & ") return Future was not finished." + else: + next.callback = cb + except: + if retFutureSym.finished: + # Take a look at tasyncexceptions for the bug which this fixes. + # That test explains it better than I can here. + raise + else: + retFutureSym.fail(getCurrentException()) + cb() + #{.pop.} +proc generateExceptionCheck(futSym, + tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = + if tryStmt.kind == nnkNilLit: + result = rootReceiver + else: + var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] + let errorNode = newDotExpr(futSym, newIdentNode("error")) + for i in 1 .. -> else: raise futSym.error + exceptionChecks.add((newIdentNode("true"), + newNimNode(nnkRaiseStmt).add(errorNode))) + # Read the future if there is no error. + # -> else: futSym.read + let elseNode = newNimNode(nnkElse, fromNode) + elseNode.add newNimNode(nnkStmtList, fromNode) + elseNode[0].add rootReceiver + + let ifBody = newStmtList() + ifBody.add newCall(newIdentNode("setCurrentException"), errorNode) + ifBody.add newIfStmt(exceptionChecks) + ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit()) + + result = newIfStmt( + (newDotExpr(futSym, newIdentNode("failed")), ifBody) + ) + result.add elseNode + +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + +template createVar(result: var NimNode, futSymName: string, + asyncProc: NimNode, + valueReceiver, rootReceiver: expr, + fromNode: NimNode) = + result = newNimNode(nnkStmtList, fromNode) + var futSym = genSym(nskVar, "future") + result.add newVarStmt(futSym, asyncProc) # -> var future = y + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) + +proc processBody(node, retFutureSym: NimNode, + subTypeIsVoid: bool, + tryStmt: NimNode): NimNode {.compileTime.} = + #echo(node.treeRepr) + result = node + case node.kind + of nnkReturnStmt: + result = newNimNode(nnkStmtList, node) + if node[0].kind == nnkEmpty: + if not subTypeIsVoid: + result.add newCall(newIdentNode("complete"), retFutureSym, + newIdentNode("result")) + else: + result.add newCall(newIdentNode("complete"), retFutureSym) + else: + result.add newCall(newIdentNode("complete"), retFutureSym, + node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) + + result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) + return # Don't process the children of this return stmt + of nnkCommand, nnkCall: + if node[0].kind == nnkIdent and node[0].ident == !"await": + case node[1].kind + of nnkIdent, nnkInfix, nnkDotExpr: + # await x + # await x or y + result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x + of nnkCall, nnkCommand: + # await foo(p, x) + # await foo p, x + var futureValue: NimNode + result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, + futureValue, node) + else: + error("Invalid node kind in 'await', got: " & $node[1].kind) + elif node.len > 1 and node[1].kind == nnkCommand and + node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + # foo await x + var newCommand = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], + newCommand, node) + + of nnkVarSection, nnkLetSection: + case node[0][2].kind + of nnkCommand: + if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + # var x = await y + var newVarSection = node # TODO: Should this use copyNimNode? + result.createVar("future" & $node[0][0].ident, node[0][2][1], + newVarSection[0][2], newVarSection, node) + else: discard + of nnkAsgn: + case node[1].kind + of nnkCommand: + if node[1][0].ident == !"await": + # x = await y + var newAsgn = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) + else: discard + of nnkDiscardStmt: + # discard await x + if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and + node[0][0].ident == !"await": + var newDiscard = node + result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], + newDiscard[0], newDiscard, node) + of nnkTryStmt: + # try: await x; except: ... + result = newNimNode(nnkStmtList, node) + template wrapInTry(n, tryBody: expr) = + var temp = n + n[0] = tryBody + tryBody = temp + + # Transform ``except`` body. + # TODO: Could we perform some ``await`` transformation here to get it + # working in ``except``? + tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil) + + proc processForTry(n: NimNode, i: var int, + res: NimNode): bool {.compileTime.} = + ## Transforms the body of the tryStmt. Does not transform the + ## body in ``except``. + ## Returns true if the tryStmt node was transformed into an ifStmt. + result = false + var skipped = n.skipStmtList() + while i < skipped.len: + var processed = processBody(skipped[i], retFutureSym, + subTypeIsVoid, n) + + # Check if we transformed the node into an exception check. + # This suggests skipped[i] contains ``await``. + if processed.kind != skipped[i].kind or processed.len != skipped[i].len: + processed = processed.skipUntilStmtList() + expectKind(processed, nnkStmtList) + expectKind(processed[2][1], nnkElse) + i.inc + + if not processForTry(n, i, processed[2][1][0]): + # We need to wrap the nnkElse nodes back into a tryStmt. + # As they are executed if an exception does not happen + # inside the awaited future. + # The following code will wrap the nodes inside the + # original tryStmt. + wrapInTry(n, processed[2][1][0]) + + res.add processed + result = true + else: + res.add skipped[i] + i.inc + var i = 0 + if not processForTry(node, i, result): + # If the tryStmt hasn't been transformed we can just put the body + # back into it. + wrapInTry(node, result) + return + else: discard + + for i in 0 .. var retFuture = newFuture[T]() + var retFutureSym = genSym(nskVar, "retFuture") + var subRetType = + if returnType.kind == nnkEmpty: newIdentNode("void") + else: baseType + outerProcBody.add( + newVarStmt(retFutureSym, + newCall( + newNimNode(nnkBracketExpr, prc[6]).add( + newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. + subRetType), + newLit(prc[0].getName)))) # Get type from return type of this proc + + # -> iterator nameIter(): FutureBase {.closure.} = + # -> {.push warning[resultshadowed]: off.} + # -> var result: T + # -> {.pop.} + # -> + # -> complete(retFuture, result) + var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") + var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) + if not subtypeIsVoid: + procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), + newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( + newIdentNode("warning"), newIdentNode("resultshadowed")), + newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + + procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( + newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T + + procBody.insert(2, newNimNode(nnkPragma).add( + newIdentNode("pop"))) # -> {.pop.}) + + procBody.add( + newCall(newIdentNode("complete"), + retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) + else: + # -> complete(retFuture) + procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], + procBody, nnkIteratorDef) + closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) + outerProcBody.add(closureIterator) + + # -> createCb(retFuture) + #var cbName = newIdentNode("cb") + var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, + newStrLitNode(prc[0].getName)) + outerProcBody.add procCb + + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) + + result = prc + + # Remove the 'async' pragma. + for i in 0 .. Date: Tue, 5 Jul 2016 13:48:03 +0300 Subject: [PATCH 18/48] Resolved path problems --- lib/pure/{ioselectors => }/ioselectors.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename lib/pure/{ioselectors => }/ioselectors.nim (97%) diff --git a/lib/pure/ioselectors/ioselectors.nim b/lib/pure/ioselectors.nim similarity index 97% rename from lib/pure/ioselectors/ioselectors.nim rename to lib/pure/ioselectors.nim index 1662c1d785..9d5beaf302 100644 --- a/lib/pure/ioselectors/ioselectors.nim +++ b/lib/pure/ioselectors.nim @@ -250,12 +250,12 @@ else: raiseOSError(osLastError()) when defined(linux): - include ioselectors_epoll + include ioselectors/ioselectors_epoll elif bsdPlatform: - include ioselectors_kqueue + include ioselectors/ioselectors_kqueue elif defined(windows): - include ioselectors_select + include ioselectors/ioselectors_select elif defined(solaris): - include ioselectors_poll # need to replace it with event ports + include ioselectors/ioselectors_poll # need to replace it with event ports else: - include ioselectors_poll + include ioselectors/ioselectors_poll From 7724336d73ec580a442357df798771e524ece3a3 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 5 Jul 2016 14:35:55 +0300 Subject: [PATCH 19/48] Patch one more path problem --- lib/pure/ioselectors.nim | 10 +++++----- .../{ioselectors => ioselects}/ioselectors_epoll.nim | 0 .../{ioselectors => ioselects}/ioselectors_kqueue.nim | 0 .../{ioselectors => ioselects}/ioselectors_poll.nim | 0 .../{ioselectors => ioselects}/ioselectors_select.nim | 0 5 files changed, 5 insertions(+), 5 deletions(-) rename lib/pure/{ioselectors => ioselects}/ioselectors_epoll.nim (100%) rename lib/pure/{ioselectors => ioselects}/ioselectors_kqueue.nim (100%) rename lib/pure/{ioselectors => ioselects}/ioselectors_poll.nim (100%) rename lib/pure/{ioselectors => ioselects}/ioselectors_select.nim (100%) diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim index 9d5beaf302..8741f7aba7 100644 --- a/lib/pure/ioselectors.nim +++ b/lib/pure/ioselectors.nim @@ -250,12 +250,12 @@ else: raiseOSError(osLastError()) when defined(linux): - include ioselectors/ioselectors_epoll + include ioselects/ioselectors_epoll elif bsdPlatform: - include ioselectors/ioselectors_kqueue + include ioselects/ioselectors_kqueue elif defined(windows): - include ioselectors/ioselectors_select + include ioselects/ioselectors_select elif defined(solaris): - include ioselectors/ioselectors_poll # need to replace it with event ports + include ioselects/ioselectors_poll # need to replace it with event ports else: - include ioselectors/ioselectors_poll + include ioselects/ioselectors_poll diff --git a/lib/pure/ioselectors/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim similarity index 100% rename from lib/pure/ioselectors/ioselectors_epoll.nim rename to lib/pure/ioselects/ioselectors_epoll.nim diff --git a/lib/pure/ioselectors/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim similarity index 100% rename from lib/pure/ioselectors/ioselectors_kqueue.nim rename to lib/pure/ioselects/ioselectors_kqueue.nim diff --git a/lib/pure/ioselectors/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim similarity index 100% rename from lib/pure/ioselectors/ioselectors_poll.nim rename to lib/pure/ioselects/ioselectors_poll.nim diff --git a/lib/pure/ioselectors/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim similarity index 100% rename from lib/pure/ioselectors/ioselectors_select.nim rename to lib/pure/ioselects/ioselectors_select.nim From 16f280843960d51cdde259e6bb25f223d5b14318 Mon Sep 17 00:00:00 2001 From: Jeff Ciesielski Date: Tue, 5 Jul 2016 07:58:26 -0400 Subject: [PATCH 20/48] Fix typo. Remove unnecessary proc --- compiler/commands.nim | 9 +-------- compiler/pragmas.nim | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 640e07b1b7..cf1e16b283 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -122,13 +122,6 @@ proc splitSwitch(switch: string, cmd, arg: var string, pass: TCmdLinePass, elif switch[i] in {':', '=', '['}: arg = substr(switch, i + 1) else: invalidCmdLineOption(pass, switch, info) -proc hasKeyValuePair(arg: string): bool = - for i in 0..arg.high: - if arg[i] in {':', '='}: - return true - - return false - proc processOnOffSwitch(op: TOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = case whichKeyword(arg) @@ -349,7 +342,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = discard "allow for backwards compatibility, but don't do anything" of "define", "d": expectArg(switch, arg, pass, info) - if hasKeyValuePair(arg): + if {':', '='} in arg: splitSwitch(arg, key, val, pass, info) defineSymbol(key, val) else: diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index fffb69ca8d..c15794b998 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -902,7 +902,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wIntDefine: sym.magic = mIntDefine of wStrDefine: - sym.magic = mIntDefine + sym.magic = mStrDefine else: invalidPragma(it) else: invalidPragma(it) From 109c9d551f7a00394b6211d8983ca0169a0e7fdf Mon Sep 17 00:00:00 2001 From: Matthew Baulch Date: Tue, 5 Jul 2016 22:56:22 +1000 Subject: [PATCH 21/48] Regard nil nodes as having equal type constraints. --- compiler/types.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/types.nim b/compiler/types.nim index bada470756..107bd9f754 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -727,6 +727,7 @@ proc equalParam(a, b: PSym): TParamsEquality = result = paramsNotEqual proc sameConstraints(a, b: PNode): bool = + if isNil(a) and isNil(b): return true internalAssert a.len == b.len for i in 1 .. Date: Tue, 5 Jul 2016 09:06:40 -0400 Subject: [PATCH 22/48] Update documentation and news --- doc/basicopt.txt | 3 ++- doc/manual/pragmas.txt | 27 +++++++++++++++++++++++++++ doc/nimc.rst | 10 +++++++++- web/news/version_0_15_released.rst | 6 ++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/doc/basicopt.txt b/doc/basicopt.txt index c68d731e6c..9a1cfd9565 100644 --- a/doc/basicopt.txt +++ b/doc/basicopt.txt @@ -11,7 +11,8 @@ Arguments: arguments are passed to the program being run (if --run option is selected) Options: -p, --path:PATH add path to search paths - -d, --define:SYMBOL(:VAL) define a conditional symbol + -d, --define:SYMBOL(:VAL) + define a conditional symbol (Optionally: Define the value for that symbol) -u, --undef:SYMBOL undefine a conditional symbol -f, --forceBuild force rebuilding of all modules diff --git a/doc/manual/pragmas.txt b/doc/manual/pragmas.txt index 1a1f0b734a..88ddabef80 100644 --- a/doc/manual/pragmas.txt +++ b/doc/manual/pragmas.txt @@ -1011,3 +1011,30 @@ debugging: # ... complex code here that produces crashes ... +compile time define pragmas +--------------------------- + +The pragmas listed here can be used to optionally accept values from +the -d/--define option at compile time. + +The implementation currently provides the following possible options (various +others may be added later). + +=============== ============================================ +pragma description +=============== ============================================ +intdefine Reads in a build-time define as an integer +strdefine Reads in a build-time define as a string +=============== ============================================ + +.. code-block:: nim + const FooBar {.intdefine.}: int = 5 + echo FooBar + +.. code-block:: bash + nim c -d:FooBar=42 foobar.c + +In the above example, providing the -d flag causes the symbol +``FooBar`` to be overwritten at compile time, printing out 42. If the +``-d:FooBar=42`` were to be omitted, the default value of 5 would be +used. diff --git a/doc/nimc.rst b/doc/nimc.rst index 48dbaeb219..eb1beb549f 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -98,6 +98,11 @@ enable builds in release mode (``-d:release``) where certain safety checks are omitted for better performance. Another common use is the ``-d:ssl`` switch to activate `SSL sockets `_. +Additionally, you may pass a value along with the symbol: ``-d:x=y`` +which may be used in conjunction with the `compile time define +pragmas`_ +to override symbols during build time. + Configuration files ------------------- @@ -370,7 +375,10 @@ For example, to generate code for an `AVR`:idx: processor use this command:: For the ``standalone`` target one needs to provide a file ``panicoverride.nim``. See ``tests/manyloc/standalone/panicoverride.nim`` for an example -implementation. +implementation. Additionally, users should specify the +amount of heap space to use with the ``-d:StandaloneHeapSize=`` +command line switch. Note that the total heap size will be +`` * sizeof(float64)``. Nim for realtime systems diff --git a/web/news/version_0_15_released.rst b/web/news/version_0_15_released.rst index ecda59fcda..5c58e138b3 100644 --- a/web/news/version_0_15_released.rst +++ b/web/news/version_0_15_released.rst @@ -38,8 +38,14 @@ Library Additions Compiler Additions ------------------ +- The ``-d/--define`` flag can now optionally take a value to be used + by code at compile time. + Language Additions ------------------ +- Added ``{.intdefine.}`` and ``{.strdefine.}`` macros to make use of + (optional) compile time defines. + Bugfixes -------- From a2301f64cdbb0efaf88dc4f5398c53ef95ba4a57 Mon Sep 17 00:00:00 2001 From: Matthew Baulch Date: Tue, 5 Jul 2016 23:16:57 +1000 Subject: [PATCH 23/48] Return nil from genOtherArg after error. --- compiler/ccgcalls.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 879b574235..bd1a4101ad 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -262,7 +262,9 @@ proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope = else: if tfVarargs notin typ.flags: localError(ri.info, "wrong argument count") - result = genArgNoParam(p, ri.sons[i]) + result = nil + else: + result = genArgNoParam(p, ri.sons[i]) discard """ Dot call syntax in C++ From e94c0ea4c84f5bdbe3fb2d40da35c7ae92d66cd7 Mon Sep 17 00:00:00 2001 From: Rostyslav Dzinko Date: Tue, 5 Jul 2016 19:05:31 +0300 Subject: [PATCH 24/48] Fixed reprEnum function on 32-bit systems --- lib/system/repr.nim | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 4da4781ef3..1b80cf2f8b 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -73,23 +73,20 @@ proc reprChar(x: char): string {.compilerRtl.} = add result, "\'" proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = - # we read an 'int' but this may have been too large, so mask the other bits: - let b = (sizeof(int)-typ.size)*8 # bits - let m = 1 shl (b-1) # mask - var o = e and ((1 shl b)-1) # clear upper bits - o = (o xor m) - m # sign extend - # XXX we need a proper narrowing based on signedness here - #e and ((1 shl (typ.size*8)) - 1) + ## Return string representation for enumeration values + var n = typ.node if ntfEnumHole notin typ.flags: - if o <% typ.node.len: - return $typ.node.sons[o].name + let o = e - n.sons[0].offset + if o >= 0 and o <% typ.node.len: + return $n.sons[o].name else: # ugh we need a slow linear search: - var n = typ.node var s = n.sons for i in 0 .. n.len-1: - if s[i].offset == o: return $s[i].name - result = $o & " (invalid data!)" + if s[i].offset == e: + return $s[i].name + + result = $e & " (invalid data!)" type PByteArray = ptr array[0.. 0xffff, int8] From 80ae938ddfe6b20072849b711e44c87343dd8a85 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 5 Jul 2016 19:56:18 +0300 Subject: [PATCH 25/48] Simplify SharedArray. --- lib/pure/ioselectors.nim | 8 +------- lib/pure/ioselects/ioselectors_kqueue.nim | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim index 8741f7aba7..d05e3117c0 100644 --- a/lib/pure/ioselectors.nim +++ b/lib/pure/ioselectors.nim @@ -185,19 +185,13 @@ else: import locks type - SharedArrayHolder[T] = object - part: array[1, T] SharedArray {.unchecked.}[T] = array[0..100_000_000, T] proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = - let holder = cast[ptr SharedArrayHolder[T]]( - allocShared0(sizeof(T) * nsize) - ) - result = cast[ptr SharedArray[T]](addr(holder.part[0])) + result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) & nsize)) proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) - type Event* {.pure.} = enum Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 78823dea4e..29a2018638 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -25,7 +25,7 @@ when defined(macosx) or defined(freebsd): proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, newp: pointer, newplen: int): cint {.importc: "sysctl",header: """#include - #include """} + #include """} elif defined(netbsd) or defined(openbsd): # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using # KERN_MAXFILES, because KERN_MAXFILES is always bigger, From 2cbdf6088a03519e8c63500de12058bbb7996f56 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 5 Jul 2016 20:00:26 +0300 Subject: [PATCH 26/48] Misplaced & --- lib/pure/ioselectors.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim index d05e3117c0..0583c275a2 100644 --- a/lib/pure/ioselectors.nim +++ b/lib/pure/ioselectors.nim @@ -188,7 +188,7 @@ else: SharedArray {.unchecked.}[T] = array[0..100_000_000, T] proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = - result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) & nsize)) + result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) From ffb975f474aabc4573a05d8fd9a9f45ebf9168a0 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 5 Jul 2016 20:01:21 +0300 Subject: [PATCH 27/48] Lower numbers for unchecked array --- lib/pure/ioselectors.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim index 0583c275a2..a5d5d2c01b 100644 --- a/lib/pure/ioselectors.nim +++ b/lib/pure/ioselectors.nim @@ -185,7 +185,7 @@ else: import locks type - SharedArray {.unchecked.}[T] = array[0..100_000_000, T] + SharedArray {.unchecked.}[T] = array[0..100, T] proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) From a76b74ed74c98eff41752fc9cb71676f527bab21 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 5 Jul 2016 22:43:59 +0300 Subject: [PATCH 28/48] Repr now works in js for enums starting with non-zero. --- compiler/jsgen.nim | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 8a631bb66e..d7aa7f0f49 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -1557,8 +1557,16 @@ proc genRepr(p: PProc, n: PNode, r: var TCompRes) = of tyEnum, tyOrdinal: gen(p, n.sons[1], r) useMagic(p, "cstrToNimstr") + var offset = "" + if t.kind == tyEnum: + let firstFieldOffset = t.n.sons[0].sym.position + if firstFieldOffset < 0: + offset = "+" & $(-firstFieldOffset) + elif firstFieldOffset > 0: + offset = "-" & $firstFieldOffset + r.kind = resExpr - r.res = "cstrToNimstr($1.node.sons[$2].name)" % [genTypeInfo(p, t), r.res] + r.res = "cstrToNimstr($1.node.sons[$2$3].name)" % [genTypeInfo(p, t), r.res, rope(offset)] else: # XXX: internalError(n.info, "genRepr: Not implemented") From 6d9177c6f1c668a517eaaa1e6946a4885ef531de Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 6 Jul 2016 11:59:20 +0200 Subject: [PATCH 29/48] added strutils.splitWhitespace --- lib/pure/strutils.nim | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 757268cd11..9b6cf45c55 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -513,21 +513,19 @@ template splitCommon(s, sep, maxsplit, sepLen) = dec(splits) inc(last, sepLen) -when defined(nimOldSplit): - template oldSplit(s, seps, maxsplit) = - ## Deprecated split[char] for transition period - var last = 0 - var splits = maxsplit - assert(not ('\0' in seps)) - while last < len(s): - while s[last] in seps: inc(last) - var first = last - while last < len(s) and s[last] notin seps: inc(last) # BUGFIX! - if first <= last-1: - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) +template oldSplit(s, seps, maxsplit) = + var last = 0 + var splits = maxsplit + assert(not ('\0' in seps)) + while last < len(s): + while s[last] in seps: inc(last) + var first = last + while last < len(s) and s[last] notin seps: inc(last) + if first <= last-1: + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) iterator split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = @@ -576,6 +574,16 @@ iterator split*(s: string, seps: set[char] = Whitespace, else: splitCommon(s, seps, maxsplit, 1) +iterator splitWhitespace*(s: string): string = + ## Splits at whitespace. + oldSplit(s, Whitespace, -1) + +proc splitWhitespace*(s: string): seq[string] {.noSideEffect, + rtl, extern: "nsuSplitWhitespace".} = + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accumulateResult(splitWhitespace(s)) + iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a single separator. ## From a6c66139fa2bf3b1481050c90ad7acbaf9dcec29 Mon Sep 17 00:00:00 2001 From: Matthew Baulch Date: Wed, 6 Jul 2016 20:30:57 +1000 Subject: [PATCH 30/48] Update sets examples so they work again. --- lib/pure/collections/sets.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index e2081e5bfb..20e06aaae6 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -58,7 +58,7 @@ proc isValid*[A](s: HashSet[A]): bool = ## initialized. Example: ## ## .. code-block :: - ## proc savePreferences(options: Set[string]) = + ## proc savePreferences(options: HashSet[string]) = ## assert options.isValid, "Pass an initialized set!" ## # Do stuff here, may crash in release builds! result = not s.data.isNil @@ -72,7 +72,7 @@ proc len*[A](s: HashSet[A]): int = ## ## .. code-block:: ## - ## var values: Set[int] + ## var values: HashSet[int] ## assert(not values.isValid) ## assert values.len == 0 result = s.counter @@ -338,7 +338,7 @@ proc init*[A](s: var HashSet[A], initialSize=64) = ## existing values and calling `excl() <#excl,TSet[A],A>`_ on them. Example: ## ## .. code-block :: - ## var a: Set[int] + ## var a: HashSet[int] ## a.init(4) ## a.incl(2) ## a.init From 09783c3fd0cb98c08aa65d27a3fa630cdc99b673 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 6 Jul 2016 14:39:42 +0300 Subject: [PATCH 31/48] Fixed vm codegen for a call with compile-time args. Fixes #4412. --- compiler/vmgen.nim | 5 ++++- tests/vm/tvmmisc.nim | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/vm/tvmmisc.nim diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index d4966b3e33..c63182b0c1 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -501,8 +501,11 @@ proc genCall(c: PCtx; n: PNode; dest: var TDest) = if dest < 0 and not isEmptyType(n.typ): dest = getTemp(c, n.typ) let x = c.getTempRange(n.len, slotTempUnknown) # varargs need 'opcSetType' for the FFI support: - let fntyp = n.sons[0].typ + let fntyp = skipTypes(n.sons[0].typ, abstractInst) for i in 0.. 0 and i < sonsLen(fntyp): + let paramType = fntyp.n.sons[i] + if paramType.typ.isCompileTimeOnly: continue var r: TRegister = x+i c.gen(n.sons[i], r) if i >= fntyp.len: diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim new file mode 100644 index 0000000000..e935013c4f --- /dev/null +++ b/tests/vm/tvmmisc.nim @@ -0,0 +1,6 @@ + +# 4412 +proc default[T](t: typedesc[T]): T {.inline.} = discard + +static: + var x = default(type(0)) From e73fd64f3841bcbc7fd4b88c7943b96fd6efc339 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 6 Jul 2016 16:48:00 +0200 Subject: [PATCH 32/48] fixes #537 --- compiler/sem.nim | 9 +++++++++ compiler/semexprs.nim | 3 +++ compiler/vmgen.nim | 26 ++++++++++++++++---------- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index a8ec2229f2..6e462dec46 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -307,6 +307,13 @@ proc semConstExpr(c: PContext, n: PNode): PNode = include hlo, seminst, semcall +when false: + # hopefully not required: + proc resetSemFlag(n: PNode) = + excl n.flags, nfSem + for i in 0 ..< n.safeLen: + resetSemFlag(n[i]) + proc semAfterMacroCall(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = ## Semantically check the output of a macro. @@ -320,6 +327,8 @@ proc semAfterMacroCall(c: PContext, n: PNode, s: PSym, c.friendModules.add(s.owner.getModule) result = n + excl(n.flags, nfSem) + #resetSemFlag n if s.typ.sons[0] == nil: result = semStmt(c, result) else: diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 57fb3ceed1..7cf0755b42 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -601,6 +601,9 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode = result = n if n.kind notin nkCallKinds or n.sons[0].kind != nkSym: return var callee = n.sons[0].sym + # workaround for bug #537 (overly aggressive inlining leading to + # wrong NimNode semantics): + if n.typ != nil and tfTriggersCompileTime in n.typ.flags: return # constant folding that is necessary for correctness of semantic pass: if callee.magic != mNone and callee.magic in ctfeWhitelist and n.typ != nil: diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index d4966b3e33..5aca6a2f1a 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -374,7 +374,7 @@ proc canonValue*(n: PNode): PNode = proc rawGenLiteral(c: PCtx; n: PNode): int = result = c.constants.len - assert(n.kind != nkCall) + #assert(n.kind != nkCall) n.flags.incl nfAllConst c.constants.add n.canonValue internalAssert result < 0x7fff @@ -497,7 +497,22 @@ proc genReturn(c: PCtx; n: PNode) = gen(c, n.sons[0]) c.gABC(n, opcRet) + +proc genLit(c: PCtx; n: PNode; dest: var TDest) = + # opcLdConst is now always valid. We produce the necessary copy in the + # assignments now: + #var opc = opcLdConst + if dest < 0: dest = c.getTemp(n.typ) + #elif c.prc.slots[dest].kind == slotFixedVar: opc = opcAsgnConst + let lit = genLiteral(c, n) + c.gABx(n, opcLdConst, dest, lit) + proc genCall(c: PCtx; n: PNode; dest: var TDest) = + # it can happen that due to inlining we have a 'n' that should be + # treated as a constant (see issue #537). + #if n.typ != nil and n.typ.sym != nil and n.typ.sym.magic == mPNimrodNode: + # genLit(c, n, dest) + # return if dest < 0 and not isEmptyType(n.typ): dest = getTemp(c, n.typ) let x = c.getTempRange(n.len, slotTempUnknown) # varargs need 'opcSetType' for the FFI support: @@ -1307,15 +1322,6 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) = let dest = c.genx(le, {gfAddrOf}) genAsgn(c, dest, ri, requiresCopy) -proc genLit(c: PCtx; n: PNode; dest: var TDest) = - # opcLdConst is now always valid. We produce the necessary copy in the - # assignments now: - #var opc = opcLdConst - if dest < 0: dest = c.getTemp(n.typ) - #elif c.prc.slots[dest].kind == slotFixedVar: opc = opcAsgnConst - let lit = genLiteral(c, n) - c.gABx(n, opcLdConst, dest, lit) - proc genTypeLit(c: PCtx; t: PType; dest: var TDest) = var n = newNode(nkType) n.typ = t From 389f500226495c335881c846c39ff938f93e693a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 6 Jul 2016 16:52:41 +0200 Subject: [PATCH 33/48] added test case for #537 --- tests/macros/tcomplexecho.nim | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/macros/tcomplexecho.nim diff --git a/tests/macros/tcomplexecho.nim b/tests/macros/tcomplexecho.nim new file mode 100644 index 0000000000..f7f933c1cd --- /dev/null +++ b/tests/macros/tcomplexecho.nim @@ -0,0 +1,42 @@ +discard """ + output: '''3 +OK +56 +123 +56 +61''' +""" + +import macros + +# Bug from the forum +macro addEcho1(s: untyped): stmt = + s.body.add(newCall("echo", newStrLitNode("OK"))) + result = s + +proc f1() {.addEcho1.} = + let i = 1+2 + echo i + +f1() + +# bug #537 +proc test(): seq[NimNode] {.compiletime.} = + result = @[] + result.add parseExpr("echo 56") + result.add parseExpr("echo 123") + result.add parseExpr("echo 56") + +proc foo(): seq[NimNode] {.compiletime.} = + result = @[] + result.add test() + result.add parseExpr("echo(5+56)") + +macro bar(): stmt = + result = newNimNode(nnkStmtList) + let x = foo() + for xx in x: + result.add xx + echo treeRepr(result) + +bar() From 32e3e01dac719a50291dd248abeb92d075005ab7 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 6 Jul 2016 18:02:59 +0300 Subject: [PATCH 34/48] Resolve problems with test on macosx. --- tests/async/tioselectors.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim index 3f08ac59df..ed2fea84fa 100644 --- a/tests/async/tioselectors.nim +++ b/tests/async/tioselectors.nim @@ -58,6 +58,7 @@ when not defined(windows): discard posix.connect(client_socket, aiList.ai_addr, aiList.ai_addrlen.Socklen) dealloc(aiList) + discard selector.select(100) var rc1 = selector.select(100) assert(len(rc1) == 2) From 28940ce457e7a7f3e2d1d6cc0223bc0282ce509e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 6 Jul 2016 20:16:41 +0200 Subject: [PATCH 35/48] ospaths can always be imported; fixes #4249 --- lib/pure/ospaths.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index 9fc816f2f5..65588529e0 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -10,7 +10,7 @@ # Included by the ``os`` module but a module in its own right for NimScript # support. -when isMainModule: +when not declared(os): {.pragma: rtl.} import strutils From e9eab32e54815bfad09fb0120aa4b024106f00ba Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 7 Jul 2016 01:02:12 +0200 Subject: [PATCH 36/48] new language feature: explicit 'import system' statements are allowed --- compiler/modules.nim | 4 ++-- compiler/sem.nim | 25 ++++++++++++++++++++--- compiler/semdata.nim | 1 + tests/modules/texplicit_system_import.nim | 9 ++++++++ web/news/version_0_15_released.rst | 3 +++ 5 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 tests/modules/texplicit_system_import.nim diff --git a/compiler/modules.nim b/compiler/modules.nim index 8ac964321e..03b9dc9f0a 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -184,8 +184,8 @@ proc importModule*(s: PSym, fileIdx: int32): PSym {.procvar.} = # this is called by the semantic checking phase result = compileModule(fileIdx, {}) if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx) - if sfSystemModule in result.flags: - localError(result.info, errAttemptToRedefine, result.name.s) + #if sfSystemModule in result.flags: + # localError(result.info, errAttemptToRedefine, result.name.s) # restore the notes for outer module: gNotes = if s.owner.id == gMainPackageId: gMainPackageNotes else: ForeignPackageNotes diff --git a/compiler/sem.nim b/compiler/sem.nim index 6e462dec46..bd7c97fcb2 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -418,9 +418,6 @@ proc myOpen(module: PSym): PPassContext = c.importTable.addSym(module) # a module knows itself if sfSystemModule in module.flags: magicsys.systemModule = module # set global variable! - else: - c.importTable.addSym magicsys.systemModule # import the "System" identifier - importAllSymbols(c, magicsys.systemModule) c.topLevelScope = openScope(c) # don't be verbose unless the module belongs to the main package: if module.owner.id == gMainPackageId: @@ -434,7 +431,29 @@ proc myOpenCached(module: PSym, rd: PRodReader): PPassContext = result = myOpen(module) for m in items(rd.methods): methodDef(m, true) +proc isImportSystemStmt(n: PNode): bool = + if magicsys.systemModule == nil: return false + case n.kind + of nkImportStmt: + for x in n: + let f = checkModuleName(x) + if f == magicsys.systemModule.info.fileIndex: + return true + of nkImportExceptStmt, nkFromStmt: + let f = checkModuleName(n[0]) + if f == magicsys.systemModule.info.fileIndex: + return true + else: discard + proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = + if c.topStmts == 0 and not isImportSystemStmt(n): + if sfSystemModule notin c.module.flags and + n.kind notin {nkEmpty, nkCommentStmt}: + c.importTable.addSym magicsys.systemModule # import the "System" identifier + importAllSymbols(c, magicsys.systemModule) + inc c.topStmts + else: + inc c.topStmts if sfNoForward in c.module.flags: result = semAllTypeSections(c, n) else: diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 91a1ebf86a..52b5dc2740 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -99,6 +99,7 @@ type unknownIdents*: IntSet # ids of all unknown identifiers to prevent # naming it multiple times generics*: seq[TInstantiationPair] # pending list of instantiated generics to compile + topStmts*: int # counts the number of encountered top level statements lastGenericIdx*: int # used for the generics stack hloLoopDetector*: int # used to prevent endless loops in the HLO inParallelStmt*: int diff --git a/tests/modules/texplicit_system_import.nim b/tests/modules/texplicit_system_import.nim new file mode 100644 index 0000000000..bc4d018bf9 --- /dev/null +++ b/tests/modules/texplicit_system_import.nim @@ -0,0 +1,9 @@ +##. +import system except `+` +discard """ + errormsg: "undeclared identifier: '+'" + line: 9 +""" +# Testament requires that the initial """ occurs before the 40th byte +# in the file. No kidding... +echo 4+5 diff --git a/web/news/version_0_15_released.rst b/web/news/version_0_15_released.rst index e85582d978..e940a2da8e 100644 --- a/web/news/version_0_15_released.rst +++ b/web/news/version_0_15_released.rst @@ -54,6 +54,9 @@ Language Additions - Added ``{.intdefine.}`` and ``{.strdefine.}`` macros to make use of (optional) compile time defines. +- If the first statement is an ``import system`` statement then ``system`` + is not imported implicitly anymore. This allows for code like + ``import system except echo`` or ``from system import nil``. Bugfixes -------- From caa7f42e8e75e273ab3f52ada30aed4e06a817d9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 7 Jul 2016 01:35:42 +0200 Subject: [PATCH 37/48] fixes #4340 --- lib/system/deepcopy.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 6e512c9b17..5445a067c3 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -124,7 +124,9 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = copyMem(dest, src, mt.size) proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = + GC_disable() genericDeepCopyAux(dest, src, mt) + GC_enable() proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = # also invoked for 'string' From b47d9b7b917202ab3b7f1632f8d9462c5f76e869 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 10:34:12 +0200 Subject: [PATCH 38/48] fixes #4371 --- compiler/ast.nim | 7 ++++--- compiler/ccgcalls.nim | 2 +- compiler/ccgexprs.nim | 2 +- compiler/ccgtypes.nim | 6 +++--- compiler/ccgutils.nim | 2 +- compiler/jsgen.nim | 2 +- compiler/patterns.nim | 2 +- compiler/semasgn.nim | 2 +- compiler/semexprs.nim | 3 +-- compiler/semstmts.nim | 4 ++-- compiler/semtypes.nim | 9 +++------ compiler/semtypinst.nim | 8 ++++---- compiler/sigmatch.nim | 4 ++-- compiler/types.nim | 9 +++++---- compiler/vmdeps.nim | 5 +++-- compiler/vmgen.nim | 2 +- lib/pure/asyncdispatch.nim | 6 ++++-- tests/async/treturn_await.nim | 23 +++++++++++++++++++++++ tests/metatype/tautoproc.nim | 2 +- tests/metatype/tvoid_must_not_match.nim | 21 +++++++++++++++++++++ tests/typerel/temptynode.nim | 2 +- 21 files changed, 84 insertions(+), 39 deletions(-) create mode 100644 tests/async/treturn_await.nim create mode 100644 tests/metatype/tvoid_must_not_match.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 34ffd6f583..2b4de75cc9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -395,6 +395,9 @@ type # sons[1]: field type # .n: nkDotExpr storing the field name + tyVoid #\ + # now different from tyEmpty, hurray! + static: # remind us when TTypeKind stops to fit in a single 64-bit word assert TTypeKind.high.ord <= 63 @@ -528,8 +531,6 @@ const tfOldSchoolExprStmt* = tfVarargs # for now used to distinguish \ # 'varargs[expr]' from 'varargs[untyped]'. Eventually 'expr' will be # deprecated and this mess can be cleaned up. - tfVoid* = tfVarargs # for historical reasons we conflated 'void' with - # 'empty' ('@[]' has the type 'seq[empty]'). tfReturnsNew* = tfInheritable skError* = skUnknown @@ -1588,7 +1589,7 @@ proc isAtom*(n: PNode): bool {.inline.} = proc isEmptyType*(t: PType): bool {.inline.} = ## 'void' and 'stmt' types are often equivalent to 'nil' these days: - result = t == nil or t.kind in {tyEmpty, tyStmt} + result = t == nil or t.kind in {tyVoid, tyStmt} proc makeStmtList*(n: PNode): PNode = if n.kind == nkStmtList: diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index bd1a4101ad..dffb8a9a5e 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -124,7 +124,7 @@ proc openArrayLoc(p: BProc, n: PNode): Rope = result = "(*$1)->data, (*$1)->$2" % [a.rdLoc, lenField(p)] of tyArray, tyArrayConstr: result = "$1, $2" % [rdLoc(a), rope(lengthOrd(lastSon(a.t)))] - else: + else: internalError("openArrayLoc: " & typeToString(a.t)) else: internalError("openArrayLoc: " & typeToString(a.t)) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 0a67ca0af0..179119badd 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1320,7 +1320,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = putIntoDest(p, d, e.typ, ropecg(p.module, "#reprAny($1, $2)", [ rdLoc(a), genTypeInfo(p.module, t)]), a.s) - of tyEmpty: + of tyEmpty, tyVoid: localError(e.info, "'repr' doesn't support 'void' type") else: putIntoDest(p, d, e.typ, ropecg(p.module, "#reprAny($1, $2)", diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index e637b6c241..a10f3b8383 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -416,7 +416,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode, addf(result, "union{$n$1} $2;$n", [unionBody, uname]) of nkSym: field = n.sym - if field.typ.kind == tyEmpty: return + if field.typ.kind == tyVoid: return #assert(field.ast == nil) sname = mangleRecFieldName(field, rectype) if accessExpr != nil: ae = "$1.$2" % [accessExpr, sname] @@ -663,7 +663,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = chunkStart = i let typeInSlot = resolveStarsInCppType(typ, idx + 1, stars) - if typeInSlot == nil or typeInSlot.kind == tyEmpty: + if typeInSlot == nil or typeInSlot.kind == tyVoid: result.add(~"void") else: result.add getTypeDescAux(m, typeInSlot, check) @@ -1022,7 +1022,7 @@ proc genTypeInfo(m: BModule, t: PType): Rope = [result, rope(typeToString(t))]) return "(&".rope & result & ")".rope case t.kind - of tyEmpty: result = rope"0" + of tyEmpty, tyVoid: result = rope"0" of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar: genTypeInfoAuxBase(m, t, t, result, rope"0") of tyProc: diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index 6dfd7b52c3..27b432c2ca 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -93,7 +93,7 @@ proc getUniqueType*(key: PType): PType = # produced instead of ``NI``. result = key of tyEmpty, tyNil, tyExpr, tyStmt, tyPointer, tyString, - tyCString, tyNone, tyBigNum: + tyCString, tyNone, tyBigNum, tyVoid: result = gCanonicalTypes[k] if result == nil: gCanonicalTypes[k] = key diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index d7aa7f0f49..0b62b84807 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -163,7 +163,7 @@ proc mapType(typ: PType): TJSTypeKind = of tyNil: result = etyNull of tyGenericInst, tyGenericParam, tyGenericBody, tyGenericInvocation, tyNone, tyFromExpr, tyForward, tyEmpty, tyFieldAccessor, - tyExpr, tyStmt, tyStatic, tyTypeDesc, tyTypeClasses: + tyExpr, tyStmt, tyStatic, tyTypeDesc, tyTypeClasses, tyVoid: result = etyNone of tyProc: result = etyProc of tyCString: result = etyString diff --git a/compiler/patterns.nim b/compiler/patterns.nim index 2336e44e76..09b8d33059 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -75,7 +75,7 @@ proc checkTypes(c: PPatternContext, p: PSym, n: PNode): bool = result = matchNodeKinds(p.constraint, n) if not result: return if isNil(n.typ): - result = p.typ.kind in {tyEmpty, tyStmt} + result = p.typ.kind in {tyVoid, tyStmt} else: result = sigmatch.argtypeMatches(c.c, p.typ, n.typ) diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index a1e209263d..9702128ba6 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -183,7 +183,7 @@ proc newSeqCall(c: PContext; x, y: PNode): PNode = proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind - of tyNone, tyEmpty: discard + of tyNone, tyEmpty, tyVoid: discard of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString, tyPtr, tyString, tyRef: defaultOp(c, t, body, x, y) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 7cf0755b42..f8723bf64c 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -32,8 +32,7 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = # XXX tyGenericInst here? if result.typ.kind == tyVar: result = newDeref(result) elif {efWantStmt, efAllowStmt} * flags != {}: - result.typ = newTypeS(tyEmpty, c) - result.typ.flags.incl tfVoid + result.typ = newTypeS(tyVoid, c) else: localError(n.info, errExprXHasNoType, renderTree(result, {renderNoComments})) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 7155f9c7a9..0fb7708756 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -138,7 +138,7 @@ proc fixNilType(n: PNode) = proc discardCheck(c: PContext, result: PNode) = if c.inTypeClass > 0: return - if result.typ != nil and result.typ.kind notin {tyStmt, tyEmpty}: + if result.typ != nil and result.typ.kind notin {tyStmt, tyVoid}: if result.kind == nkNilLit: result.typ = nil message(result.info, warnNilStatement) @@ -711,7 +711,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) = a.sons[1] = s.typ.n s.typ.size = -1 # could not be computed properly # we fill it out later. For magic generics like 'seq', it won't be filled - # so we use tyEmpty instead of nil to not crash for strange conversions + # so we use tyNone instead of nil to not crash for strange conversions # like: mydata.seq rawAddSon(s.typ, newTypeS(tyNone, c)) s.ast = a diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index e9a2be1bd0..bef1049142 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -945,7 +945,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, if isType: localError(a.info, "':' expected") if kind in {skTemplate, skMacro}: typ = newTypeS(tyExpr, c) - elif skipTypes(typ, {tyGenericInst}).kind == tyEmpty: + elif skipTypes(typ, {tyGenericInst}).kind == tyVoid: continue for j in countup(0, length-3): var arg = newSymG(skParam, a.sons[j], c) @@ -977,7 +977,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, if r != nil: # turn explicit 'void' return type into 'nil' because the rest of the # compiler only checks for 'nil': - if skipTypes(r, {tyGenericInst}).kind != tyEmpty: + if skipTypes(r, {tyGenericInst}).kind != tyVoid: # 'auto' as a return type does not imply a generic: if r.kind == tyAnything: # 'p(): auto' and 'p(): expr' are equivalent, but the rest of the @@ -1390,10 +1390,7 @@ proc processMagicType(c: PContext, m: PSym) = setMagicType(m, tyTypeDesc, 0) rawAddSon(m.typ, newTypeS(tyNone, c)) of mVoidType: - setMagicType(m, tyEmpty, 0) - # for historical reasons we conflate 'void' with 'empty' so that '@[]' - # has the type 'seq[void]'. - m.typ.flags.incl tfVoid + setMagicType(m, tyVoid, 0) of mArray: setMagicType(m, tyArray, 0) of mOpenArray: diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 7ff33f9186..922071ba3a 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -162,7 +162,7 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode = discard of nkSym: result.sym = replaceTypeVarsS(cl, n.sym) - if result.sym.typ.kind == tyEmpty: + if result.sym.typ.kind == tyVoid: # don't add the 'void' field result = newNode(nkRecList, n.info) of nkRecWhen: @@ -316,15 +316,15 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = proc eraseVoidParams*(t: PType) = # transform '(): void' into '()' because old parts of the compiler really # don't deal with '(): void': - if t.sons[0] != nil and t.sons[0].kind == tyEmpty: + if t.sons[0] != nil and t.sons[0].kind == tyVoid: t.sons[0] = nil for i in 1 .. = 0 and (n.typ.isNil or n.typ.kind == tyEmpty): + if dest >= 0 and (n.typ.isNil or n.typ.kind == tyVoid): c.freeTemp(dest) dest = -1 diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index be6522c363..f47bfeee33 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1699,8 +1699,10 @@ proc processBody(node, retFutureSym: NimNode, else: result.add newCall(newIdentNode("complete"), retFutureSym) else: - result.add newCall(newIdentNode("complete"), retFutureSym, - node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) + let x = node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt) + if x.kind == nnkYieldStmt: result.add x + else: + result.add newCall(newIdentNode("complete"), retFutureSym, x) result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt diff --git a/tests/async/treturn_await.nim b/tests/async/treturn_await.nim new file mode 100644 index 0000000000..8d266d665d --- /dev/null +++ b/tests/async/treturn_await.nim @@ -0,0 +1,23 @@ + +# bug #4371 + +import strutils, asyncdispatch, asynchttpserver + +type + List[A] = ref object + value: A + next: List[A] + StrPair* = tuple[k, v: string] + Context* = object + position*: int + accept*: bool + headers*: List[StrPair] + Handler* = proc(req: ref Request, ctx: Context): Future[Context] + +proc logging*(handler: Handler): auto = + proc h(req: ref Request, ctx: Context): Future[Context] {.async.} = + let ret = handler(req, ctx) + debugEcho "$3 $1 $2".format(req.reqMethod, req.url.path, req.hostname) + return await ret + + return h diff --git a/tests/metatype/tautoproc.nim b/tests/metatype/tautoproc.nim index ef5377096c..2c8f6a3f71 100644 --- a/tests/metatype/tautoproc.nim +++ b/tests/metatype/tautoproc.nim @@ -1,5 +1,5 @@ discard """ - output: "empty" + output: "void" """ # bug #898 diff --git a/tests/metatype/tvoid_must_not_match.nim b/tests/metatype/tvoid_must_not_match.nim new file mode 100644 index 0000000000..240c3f7517 --- /dev/null +++ b/tests/metatype/tvoid_must_not_match.nim @@ -0,0 +1,21 @@ +discard """ + errormsg: "type mismatch: got (Future[system.int], void)" + line: 20 +""" + +type + Future[T] = object + value: T + +proc complete[T](x: T) = + echo "completed" + let y = x + + +proc complete*[T](future: var Future[T], val: T) = + future.value = val + +var a: Future[int] + +complete(a): + echo "yielding void" diff --git a/tests/typerel/temptynode.nim b/tests/typerel/temptynode.nim index 91e45f3ca4..b32b16121a 100644 --- a/tests/typerel/temptynode.nim +++ b/tests/typerel/temptynode.nim @@ -1,6 +1,6 @@ discard """ line: 16 - errormsg: "type mismatch: got (empty)" + errormsg: "type mismatch: got (void)" """ # bug #950 From 857b0c8d4c79d44fc0580df4658d1b17121dceed Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 10:52:04 +0200 Subject: [PATCH 39/48] fixes #4462 --- compiler/vmgen.nim | 8 +++++--- tests/vm/tvmmisc.nim | 12 +++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index b8c206bb30..abb88b7b93 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -518,9 +518,9 @@ proc genCall(c: PCtx; n: PNode; dest: var TDest) = # varargs need 'opcSetType' for the FFI support: let fntyp = skipTypes(n.sons[0].typ, abstractInst) for i in 0.. 0 and i < sonsLen(fntyp): - let paramType = fntyp.n.sons[i] - if paramType.typ.isCompileTimeOnly: continue + #if i > 0 and i < sonsLen(fntyp): + # let paramType = fntyp.n.sons[i] + # if paramType.typ.isCompileTimeOnly: continue var r: TRegister = x+i c.gen(n.sons[i], r) if i >= fntyp.len: @@ -1797,6 +1797,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = genConv(c, n, n.sons[1], dest, opcCast) else: globalError(n.info, errGenerated, "VM is not allowed to 'cast'") + of nkTypeOfExpr: + genTypeLit(c, n.typ, dest) else: globalError(n.info, errGenerated, "cannot generate VM code for " & $n) diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index e935013c4f..b7112b0999 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -1,5 +1,15 @@ -# 4412 + +# bug #4462 +import macros + +proc foo(t: typedesc) {.compileTime.} = + echo getType(t).treeRepr + +static: + foo(int) + +# #4412 proc default[T](t: typedesc[T]): T {.inline.} = discard static: From 019ee2260c82681ba5d1b475c6f6f702668e90d0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 11:05:48 +0200 Subject: [PATCH 40/48] fixes #4399 --- lib/pure/json.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index b4eecdf88e..5198f5e008 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -712,6 +712,7 @@ proc `%`*(b: bool): JsonNode = proc `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode = ## Generic constructor for JSON data. Creates a new `JObject JsonNode` + if keyvals.len == 0: return newJArray() result = newJObject() for key, val in items(keyVals): result.fields[key] = val From abf1951ff0e26b5e018e8c87158060b02053fddb Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 15:08:55 +0200 Subject: [PATCH 41/48] docgen: hide pragmas --- compiler/docgen.nim | 10 +++++++++- config/nimdoc.cfg | 40 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index d954b897bb..fab7598485 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -380,8 +380,16 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = "\\spanIdentifier{$1}", [rope(esc(d.target, literal))]) of tkSpaces, tkInvalid: add(result, literal) + of tkCurlyDotLe: + dispA(result, """$1
""", + "\\spanOther{$1}", + [rope(esc(d.target, literal))]) + of tkCurlyDotRi: + dispA(result, "
$1", + "\\spanOther{$1}", + [rope(esc(d.target, literal))]) of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi, - tkBracketDotLe, tkBracketDotRi, tkCurlyDotLe, tkCurlyDotRi, tkParDotLe, + tkBracketDotLe, tkBracketDotRi, tkParDotLe, tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot, tkAccent, tkColonColon, tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr: diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index d973b922a8..112833e583 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -1,5 +1,5 @@ # This is the config file for the documentation generator. -# (c) 2012 Andreas Rumpf +# (c) 2016 Andreas Rumpf # Feel free to edit the templates as you need. If you modify this file, it # might be worth updating the hardcoded values in packages/docutils/rstgen.nim @@ -1235,10 +1235,46 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator background-repeat: no-repeat; background-image: url("data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="); margin-bottom: -5px; } + div.pragma { + display: none; + } + span.pragmabegin { + cursor: pointer; + } + span.pragmaend { + cursor: pointer; + } + + + - +

$title

From 7f752db0e3986c8ba4f010bb0d5e959e748b5248 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 15:55:42 +0200 Subject: [PATCH 42/48] improve error message if C and Nim disagree on pointer size --- lib/nimbase.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/nimbase.h b/lib/nimbase.h index d5a5513522..3635ca006d 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -102,7 +102,7 @@ __clang__ defined __ICL || \ defined __DMC__ || \ defined __BORLANDC__ ) -# define NIM_THREADVAR __declspec(thread) +# define NIM_THREADVAR __declspec(thread) /* note that ICC (linux) and Clang are covered by __GNUC__ */ #elif defined __GNUC__ || \ defined __SUNPRO_C || \ @@ -449,8 +449,8 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } /* Test to see if Nim and the C compiler agree on the size of a pointer. On disagreement, your C compiler will say something like: - "error: 'assert_numbits' declared as an array with a negative size" */ -typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; + "error: 'Nim_and_C_compiler_disagree_on_target_architecture' declared as an array with a negative size" */ +typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; #endif #ifdef __cplusplus From d9e44873ab4a61f79aee33c54ed56656c5446cb7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 18:04:41 +0200 Subject: [PATCH 43/48] gensym'ed symbols are rendered with their ID for much easier debugging --- compiler/renderer.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index e1db327f45..0e733d643a 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -704,7 +704,10 @@ proc gcase(g: var TSrcGen, n: PNode) = proc gproc(g: var TSrcGen, n: PNode) = var c: TContext if n.sons[namePos].kind == nkSym: - put(g, tkSymbol, renderDefinitionName(n.sons[namePos].sym)) + let s = n.sons[namePos].sym + put(g, tkSymbol, renderDefinitionName(s)) + if sfGenSym in s.flags: + put(g, tkIntLit, $s.id) else: gsub(g, n.sons[namePos]) @@ -798,7 +801,8 @@ proc gident(g: var TSrcGen, n: PNode) = else: t = tkOpr put(g, t, s) - if n.kind == nkSym and renderIds in g.flags: put(g, tkIntLit, $n.sym.id) + if n.kind == nkSym and (renderIds in g.flags or sfGenSym in n.sym.flags): + put(g, tkIntLit, $n.sym.id) proc doParamsAux(g: var TSrcGen, params: PNode) = if params.len > 1: From d83eb7064345ae09df927eedf53b4661a2341ffa Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 18:05:36 +0200 Subject: [PATCH 44/48] async: use -d:nimDumpAsync to see what the async macro generates --- lib/pure/asyncdispatch.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index f47bfeee33..15d9ab02ea 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1898,7 +1898,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> createCb(retFuture) #var cbName = newIdentNode("cb") - var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, + var procCb = getAst createCb(retFutureSym, iteratorNameSym, newStrLitNode(prc[0].getName)) outerProcBody.add procCb @@ -1933,6 +1933,8 @@ macro async*(prc: stmt): stmt {.immediate.} = result.add asyncSingleProc(oneProc) else: result = asyncSingleProc(prc) + when defined(nimDumpAsync): + echo repr result proc recvLine*(socket: AsyncFD): Future[string] {.async.} = ## Reads a line of data from ``socket``. Returned future will complete once From 1d186ee9c335a54ae9700110afe4ec9e888b7b9d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 18:08:34 +0200 Subject: [PATCH 45/48] fixes #2377 --- compiler/semgnrc.nim | 7 ++++++- tests/async/tgeneric_async.nim | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/async/tgeneric_async.nim diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 0ba76ccd35..b786794112 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -426,7 +426,12 @@ proc semGenericStmt(c: PContext, n: PNode, n.sons[paramsPos] = semGenericStmt(c, n.sons[paramsPos], flags, ctx) n.sons[pragmasPos] = semGenericStmt(c, n.sons[pragmasPos], flags, ctx) var body: PNode - if n.sons[namePos].kind == nkSym: body = n.sons[namePos].sym.getBody + if n.sons[namePos].kind == nkSym: + let s = n.sons[namePos].sym + if sfGenSym in s.flags and s.ast == nil: + body = n.sons[bodyPos] + else: + body = s.getBody else: body = n.sons[bodyPos] n.sons[bodyPos] = semGenericStmtScope(c, body, flags, ctx) closeScope(c) diff --git a/tests/async/tgeneric_async.nim b/tests/async/tgeneric_async.nim new file mode 100644 index 0000000000..af63701816 --- /dev/null +++ b/tests/async/tgeneric_async.nim @@ -0,0 +1,9 @@ + +import asyncdispatch + +when true: + # bug #2377 + proc test[T](v: T) {.async.} = + echo $v + + asyncCheck test[int](1) From 089c31765fd7d0fc2c4beb4869dd9a42c78b4ab8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 20:11:59 +0200 Subject: [PATCH 46/48] fixes #3055 --- compiler/seminst.nim | 2 +- tests/generics/tforward_generic.nim | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/generics/tforward_generic.nim diff --git a/compiler/seminst.nim b/compiler/seminst.nim index e4ac56cd6b..460db4f7cc 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -205,7 +205,7 @@ proc instantiateProcType(c: PContext, pt: TIdTable, # The solution would be to move this logic into semtypinst, but # at this point semtypinst have to become part of sem, because it # will need to use openScope, addDecl, etc. - addDecl(c, prc) + #addDecl(c, prc) pushInfoContext(info) var cl = initTypeVars(c, pt, info, nil) diff --git a/tests/generics/tforward_generic.nim b/tests/generics/tforward_generic.nim new file mode 100644 index 0000000000..169279cb3d --- /dev/null +++ b/tests/generics/tforward_generic.nim @@ -0,0 +1,28 @@ +discard """ + output: '''b() +720 120.0''' +""" + +# bug #3055 +proc b(t: int | string) +proc a(t: int) = b(t) +proc b(t: int | string) = echo "b()" +a(1) + +# test recursive generics still work: +proc fac[T](x: T): T = + if x == 0: return 1 + else: return fac(x-1)*x + +echo fac(6), " ", fac(5.0) + +when false: + # This still doesn't work... + # test recursive generic with forwarding: + proc fac2[T](x: T): T + + echo fac2(6), " ", fac2(5.0) + + proc fac2[T](x: T): T = + if x == 0: return 1 + else: return fac2(x-1)*x From e2267ef5c95e400e20ac48bea9c662241bc01f18 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 22:24:28 +0200 Subject: [PATCH 47/48] Nimscript supports hint() and warning() procs; refs #3688 --- compiler/commands.nim | 2 +- compiler/scriptconfig.nim | 9 +++++++-- lib/system/nimscript.nim | 13 +++++++++++++ tests/newconfig/tfoo.nims | 3 +++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index cf1e16b283..22512c563e 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -158,7 +158,7 @@ var enableNotes: TNoteKinds disableNotes: TNoteKinds -proc processSpecificNote(arg: string, state: TSpecialWord, pass: TCmdLinePass, +proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, info: TLineInfo; orig: string) = var id = "" # arg = "X]:on|off" var i = 0 diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index d04fd52316..ca42cc8fa7 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -13,7 +13,7 @@ import ast, modules, passes, passaux, condsyms, options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs, - os, times, osproc + os, times, osproc, wordrecg # we support 'cmpIgnoreStyle' natively for efficiency: from strutils import cmpIgnoreStyle @@ -116,7 +116,12 @@ proc setupVM*(module: PSym; scriptName: string): PEvalContext = setResult(a, options.command) cbconf switch: processSwitch(a.getString 0, a.getString 1, passPP, unknownLineInfo()) - + cbconf hintImpl: + processSpecificNote(a.getString 0, wHint, passPP, unknownLineInfo(), + a.getString 1) + cbconf warningImpl: + processSpecificNote(a.getString 0, wWarning, passPP, unknownLineInfo(), + a.getString 1) proc runNimScript*(scriptName: string; freshDefines=true) = passes.gIncludeFile = includeModule diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index d587d772f9..db6d72d7bb 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -39,6 +39,9 @@ proc getCurrentDir(): string = builtin proc rawExec(cmd: string): int {.tags: [ExecIOEffect], raises: [OSError].} = builtin +proc warningImpl(arg, orig: string) = discard +proc hintImpl(arg, orig: string) = discard + proc paramStr*(i: int): string = ## Retrieves the ``i``'th command line parameter. builtin @@ -52,6 +55,16 @@ proc switch*(key: string, val="") = ## example ``switch("checks", "on")``. builtin +proc warning*(name: string; val: bool) = + ## Disables or enables a specific warning. + let v = if val: "on" else: "off" + warningImpl(name & "]:" & v, "warning[" & name & "]:" & v) + +proc hint*(name: string; val: bool) = + ## Disables or enables a specific hint. + let v = if val: "on" else: "off" + hintImpl(name & "]:" & v, "hint[" & name & "]:" & v) + proc getCommand*(): string = ## Gets the Nim command that the compiler has been invoked with, for example ## "c", "js", "build", "help". diff --git a/tests/newconfig/tfoo.nims b/tests/newconfig/tfoo.nims index 519a868d5d..8d1461c78e 100644 --- a/tests/newconfig/tfoo.nims +++ b/tests/newconfig/tfoo.nims @@ -8,6 +8,9 @@ import ospaths --forceBuild +warning("uninit", off) +hint("processing", off) + task listDirs, "lists every subdirectory": for x in listDirs("."): echo "DIR ", x From ba273057e31775dfacbd64719641cec7f3b95891 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jul 2016 23:03:11 +0200 Subject: [PATCH 48/48] Fixes a critical JS codegen bug about @ in call pattern --- compiler/jsgen.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 0b62b84807..09eafe1285 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -1215,6 +1215,7 @@ proc genOtherArg(p: PProc; n: PNode; i: int; typ: PType; genArgNoParam(p, it, r) else: genArg(p, it, paramType.sym, r) + inc generated proc genPatternCall(p: PProc; n: PNode; pat: string; typ: PType; r: var TCompRes) =