From a39b989ce238546eca64e9c37734bc48f61e280c Mon Sep 17 00:00:00 2001 From: data-man Date: Mon, 21 May 2018 23:41:32 +0300 Subject: [PATCH 01/19] fixes #7855 --- compiler/filter_tmpl.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/filter_tmpl.nim b/compiler/filter_tmpl.nim index f93355e6bc..230c4ad32c 100644 --- a/compiler/filter_tmpl.nim +++ b/compiler/filter_tmpl.nim @@ -65,7 +65,7 @@ proc parseLine(p: var TTmplParser) = var j = 0 let hi = p.x.len - 1 - if hi == 0: + if hi < 0: return while j <= hi and p.x[j] == ' ': inc(j) From c12726d41e7df20bce452b6d7f8959d8b9e401a9 Mon Sep 17 00:00:00 2001 From: andri lim Date: Tue, 22 May 2018 18:17:09 +0700 Subject: [PATCH 02/19] fixes #7696, add ConfigRef.arguments initialization (#7858) --- compiler/options.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/options.nim b/compiler/options.nim index 2027897fa4..1c4bb6157d 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -221,7 +221,8 @@ proc newConfigRef*(): ConfigRef = keepComments: true, # whether the parser needs to keep comments implicitImports: @[], # modules that are to be implicitly imported implicitIncludes: @[], # modules that are to be implicitly included - docSeeSrcUrl: "" + docSeeSrcUrl: "", + arguments: "" ) # enable colors by default on terminals if terminal.isatty(stderr): From 4da0135ace7847323f0d1c65e32a2814d9bfc6e6 Mon Sep 17 00:00:00 2001 From: data-man Date: Tue, 22 May 2018 18:56:27 +0300 Subject: [PATCH 03/19] fixes filter bug for empty lines --- compiler/filter_tmpl.nim | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/compiler/filter_tmpl.nim b/compiler/filter_tmpl.nim index 230c4ad32c..6c16a0b4ea 100644 --- a/compiler/filter_tmpl.nim +++ b/compiler/filter_tmpl.nim @@ -42,8 +42,7 @@ proc newLine(p: var TTmplParser) = proc scanPar(p: var TTmplParser, d: int) = var i = d - let hi = p.x.len - 1 - while i <= hi: + while i < p.x.len: case p.x[i] of '(': inc(p.par) of ')': dec(p.par) @@ -63,22 +62,19 @@ const proc parseLine(p: var TTmplParser) = var j = 0 - let hi = p.x.len - 1 + let len = p.x.len - if hi < 0: - return + while j < len and p.x[j] == ' ': inc(j) - while j <= hi and p.x[j] == ' ': inc(j) - - if p.x.len >= 2 and p.x[0] == p.nimDirective and p.x[1] == '?': + if len >= 2 and p.x[0] == p.nimDirective and p.x[1] == '?': newLine(p) - elif j < p.x.len and p.x[j] == p.nimDirective: + elif j < len and p.x[j] == p.nimDirective: newLine(p) inc(j) - while j <= hi and p.x[j] == ' ': inc(j) + while j < len and p.x[j] == ' ': inc(j) let d = j var keyw = "" - while j <= hi and p.x[j] in PatternChars: + while j < len and p.x[j] in PatternChars: add(keyw, p.x[j]) inc(j) @@ -132,7 +128,7 @@ proc parseLine(p: var TTmplParser) = llStreamWrite(p.outp, "(\"") inc(p.emitPar) p.state = psTempl - while j <= hi: + while j < len: case p.x[j] of '\x01'..'\x1F', '\x80'..'\xFF': llStreamWrite(p.outp, "\\x") @@ -160,7 +156,7 @@ proc parseLine(p: var TTmplParser) = llStreamWrite(p.outp, '(') inc(j) var curly = 0 - while j <= hi: + while j < len: case p.x[j] of '{': inc(j) @@ -185,7 +181,7 @@ proc parseLine(p: var TTmplParser) = llStreamWrite(p.outp, p.conc) llStreamWrite(p.outp, p.toStr) llStreamWrite(p.outp, '(') - while j <= hi and p.x[j] in PatternChars: + while j < len and p.x[j] in PatternChars: llStreamWrite(p.outp, p.x[j]) inc(j) llStreamWrite(p.outp, ')') From 1d72ec3f1bd94e20f442cfe418c98a371d3dd9e4 Mon Sep 17 00:00:00 2001 From: genotrance Date: Tue, 22 May 2018 13:53:29 -0500 Subject: [PATCH 04/19] Detect gcc arch in build.bat (#7856) * Detect gcc arch in build.bat * Fix 64-bit test --- tools/niminst/buildbat.tmpl | 58 ++++++++++++++++++++++++++++++------- tools/niminst/niminst.nim | 16 ++++++---- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/tools/niminst/buildbat.tmpl b/tools/niminst/buildbat.tmpl index 278b6caeae..6767461e37 100644 --- a/tools/niminst/buildbat.tmpl +++ b/tools/niminst/buildbat.tmpl @@ -1,5 +1,5 @@ #? stdtmpl(subsChar='?') | standard -#proc generateBuildBatchScript(c: ConfigData, winIndex, cpuIndex: int): string = +#proc generateBuildBatchScript(c: ConfigData, winIndex, cpuIndex32, cpuIndex64: int): string = # result = "@echo off\nREM Generated by niminst\n" SET CC=gcc SET LINKER=gcc @@ -7,27 +7,63 @@ SET COMP_FLAGS=?{c.ccompiler.flags} SET LINK_FLAGS=?{c.linker.flags} SET BIN_DIR=?{firstBinPath(c).toWin} +REM Detect gcc arch +IF DEFINED ARCH ( + ECHO Forcing %CC% arch +) ELSE ( + ECHO Detecting %CC% arch + ECHO int main^(^) { return sizeof^(void *^); } | gcc -xc - -o archtest && archtest + IF ERRORLEVEL 4 SET ARCH=32 + IF ERRORLEVEL 8 SET ARCH=64 + del archtest.* +) +ECHO Building with %ARCH% bit %CC% + if EXIST ..\koch.nim SET BIN_DIR=..\bin if NOT EXIST %BIN_DIR%\nul mkdir %BIN_DIR% REM call the compiler: +IF %ARCH% EQU 32 ( + # block win32: # var linkCmd = "" -# for ff in items(c.cfiles[winIndex][cpuIndex]): -# let f = ff.toWin -ECHO %CC% %COMP_FLAGS% -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} -CALL %CC% %COMP_FLAGS% -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} -# linkCmd.add(" " & changeFileExt(f, "o")) -IF ERRORLEVEL 1 (GOTO:END) -# end for +# if cpuIndex32 != -1: +# for ff in items(c.cfiles[winIndex][cpuIndex32]): +# let f = ff.toWin + ECHO %CC% %COMP_FLAGS% -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} + CALL %CC% %COMP_FLAGS% -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} +# linkCmd.add(" " & changeFileExt(f, "o")) + IF ERRORLEVEL 1 (GOTO:END) +# end for +# end if -ECHO %LINKER% -o ?{"%BIN_DIR%"\toLowerAscii(c.name)}.exe ?linkCmd %LINK_FLAGS% -CALL %LINKER% -o ?{"%BIN_DIR%"\toLowerAscii(c.name)}.exe ?linkCmd %LINK_FLAGS% + ECHO %LINKER% -o ?{"%BIN_DIR%"\toLowerAscii(c.name)}.exe ?linkCmd %LINK_FLAGS% + CALL %LINKER% -o ?{"%BIN_DIR%"\toLowerAscii(c.name)}.exe ?linkCmd %LINK_FLAGS% # end block +) ELSE IF %ARCH% EQU 64 ( + +# block win64: +# var linkCmd = "" +# if cpuIndex64 != -1: +# for ff in items(c.cfiles[winIndex][cpuIndex64]): +# let f = ff.toWin + ECHO %CC% %COMP_FLAGS% -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} + CALL %CC% %COMP_FLAGS% -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} +# linkCmd.add(" " & changeFileExt(f, "o")) + IF ERRORLEVEL 1 (GOTO:END) +# end for +# end if + + ECHO %LINKER% -o ?{"%BIN_DIR%"\toLowerAscii(c.name)}.exe ?linkCmd %LINK_FLAGS% + CALL %LINKER% -o ?{"%BIN_DIR%"\toLowerAscii(c.name)}.exe ?linkCmd %LINK_FLAGS% + +# end block +) + :END IF ERRORLEVEL 1 ( ECHO FAILURE @@ -35,6 +71,8 @@ IF ERRORLEVEL 1 ( ECHO CSource compilation failed. Please check that the gcc compiler is in ECHO the PATH environment variable, and that you are calling the batch script ECHO that matches the target architecture of the compiler. + ECHO. + ECHO Use build.bat to autodetect the compiler architecture. ) ELSE ( ECHO SUCCESS ) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index c2816a0efd..e2cc8cf3a7 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -21,7 +21,8 @@ const maxOS = 20 # max number of OSes maxCPU = 20 # max number of CPUs buildShFile = "build.sh" - buildBatFile32 = "build.bat" + buildBatFile = "build.bat" + buildBatFile32 = "build32.bat" buildBatFile64 = "build64.bat" makeFile = "makefile" installShFile = "install.sh" @@ -542,12 +543,13 @@ proc srcdist(c: var ConfigData) = inclFilePermissions(getOutputDir(c) / buildShFile, {fpUserExec, fpGroupExec, fpOthersExec}) writeFile(getOutputDir(c) / makeFile, generateMakefile(c), "\10") if winIndex >= 0: + if intel32Index >= 0 or intel64Index >= 0: + writeFile(getOutputDir(c) / buildBatFile, + generateBuildBatchScript(c, winIndex, intel32Index, intel64Index), "\13\10") if intel32Index >= 0: - writeFile(getOutputDir(c) / buildBatFile32, - generateBuildBatchScript(c, winIndex, intel32Index), "\13\10") + writeFile(getOutputDir(c) / buildBatFile32, "SET ARCH=32\nCALL build.bat\n") if intel64Index >= 0: - writeFile(getOutputDir(c) / buildBatFile64, - generateBuildBatchScript(c, winIndex, intel64Index), "\13\10") + writeFile(getOutputDir(c) / buildBatFile64, "SET ARCH=64\nCALL build.bat\n") writeInstallScripts(c) # --------------------- generate inno setup ----------------------------------- @@ -593,6 +595,7 @@ when haveZipLib: else: n = c.outdir / n var z: ZipArchive if open(z, n, fmWrite): + addFile(z, proj / buildBatFile, "build" / buildBatFile) addFile(z, proj / buildBatFile32, "build" / buildBatFile32) addFile(z, proj / buildBatFile64, "build" / buildBatFile64) addFile(z, proj / buildShFile, "build" / buildShFile) @@ -631,11 +634,12 @@ proc xzDist(c: var ConfigData; windowsZip=false) = if not dirExists(destDir): createDir(destDir) copyFileWithPermissions(src, dest) - if not windowsZip and not existsFile("build" / buildBatFile32): + if not windowsZip and not existsFile("build" / buildBatFile): quit("No C sources found in ./build/, please build by running " & "./koch csource -d:release.") if not windowsZip: + processFile(proj / buildBatFile, "build" / buildBatFile) processFile(proj / buildBatFile32, "build" / buildBatFile32) processFile(proj / buildBatFile64, "build" / buildBatFile64) processFile(proj / buildShFile, "build" / buildShFile) From c6a9a9c30b2b758f3f5da582b2f1a8c4b0d7d503 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 22 May 2018 23:07:02 +0200 Subject: [PATCH 05/19] hotfix: fixes SSL initialization for httpclient with --threads:on (#7863) * hotfix: fixes SSL initialization for httpclient with --threads:on --- lib/pure/httpclient.nim | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 3f1a3d0671..8530e4c42f 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -362,15 +362,13 @@ when not defined(ssl): type SSLContext = ref object var defaultSSLContext {.threadvar.}: SSLContext -when defined(ssl): - defaultSSLContext = newContext(verifyMode = CVerifyNone) - template contextOrDefault(ctx: SSLContext): SSLContext = - var result = ctx - if ctx == nil: - if defaultSSLContext == nil: - defaultSSLContext = newContext(verifyMode = CVerifyNone) +proc getDefaultSSL(): SSLContext = + result = defaultSslContext + when defined(ssl): + if result == nil: + defaultSSLContext = newContext(verifyMode = CVerifyNone) result = defaultSSLContext - result + doAssert result != nil, "failure to initialize the SSL context" proc newProxy*(url: string, auth = ""): Proxy = ## Constructs a new ``TProxy`` object. @@ -480,9 +478,9 @@ proc format(p: MultipartData): tuple[contentType, body: string] = result.body.add("--" & bound & "--\c\L") proc request*(url: string, httpMethod: string, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, + body = "", sslContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response - {.deprecated.} = + {.deprecated: "use HttpClient.request instead".} = ## | Requests ``url`` with the custom method string specified by the ## | ``httpMethod`` parameter. ## | Extra headers can be specified and must be separated by ``\c\L`` @@ -580,7 +578,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", result = parseResponse(s, httpMethod != "HEAD", timeout) proc request*(url: string, httpMethod = HttpGET, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, + body = "", sslContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response {.deprecated.} = ## | Requests ``url`` with the specified ``httpMethod``. @@ -611,7 +609,7 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string = result = $parsed proc get*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response {.deprecated.} = ## | GETs the ``url`` and returns a ``Response`` object @@ -632,7 +630,7 @@ proc get*(url: string, extraHeaders = "", maxRedirects = 5, lastURL = redirectTo proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): string {.deprecated.} = ## | GETs the body and returns it as a string. @@ -651,7 +649,7 @@ proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, proc post*(url: string, extraHeaders = "", body = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, multipart: MultipartData = nil): Response {.deprecated.} = @@ -694,7 +692,7 @@ proc post*(url: string, extraHeaders = "", body = "", proc postContent*(url: string, extraHeaders = "", body = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, multipart: MultipartData = nil): string @@ -717,7 +715,7 @@ proc postContent*(url: string, extraHeaders = "", body = "", return r.body proc downloadFile*(url: string, outputFilename: string, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil) {.deprecated.} = ## | Downloads ``url`` and saves it to ``outputFilename`` @@ -817,7 +815,7 @@ type HttpClient* = HttpClientBase[Socket] proc newHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext, proxy: Proxy = nil, + maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil, timeout = -1): HttpClient = ## Creates a new HttpClient instance. ## @@ -844,7 +842,7 @@ proc newHttpClient*(userAgent = defUserAgent, result.bodyStream = newStringStream() result.getBody = true when defined(ssl): - result.sslContext = contextOrDefault(sslContext) + result.sslContext = sslContext type AsyncHttpClient* = HttpClientBase[AsyncSocket] @@ -852,7 +850,7 @@ type {.deprecated: [PAsyncHttpClient: AsyncHttpClient].} proc newAsyncHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext, + maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil): AsyncHttpClient = ## Creates a new AsyncHttpClient instance. ## @@ -876,7 +874,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, result.bodyStream = newFutureStream[string]("newAsyncHttpClient") result.getBody = true when defined(ssl): - result.sslContext = contextOrDefault(sslContext) + result.sslContext = sslContext proc close*(client: HttpClient | AsyncHttpClient) = ## Closes any connections held by the HTTP client. From 799674168d969afa73a507d59de7e34b360f1047 Mon Sep 17 00:00:00 2001 From: Tyler Date: Tue, 22 May 2018 19:27:28 -0700 Subject: [PATCH 06/19] Patch the PEG library to work with the JS backend (#7866) * Added compiler check on transformFile --- lib/pure/pegs.nim | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 8304298421..e13bc83298 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1010,14 +1010,18 @@ proc replace*(s: string, sub: Peg, cb: proc( inc(m) add(result, substr(s, i)) -proc transformFile*(infile, outfile: string, - subs: varargs[tuple[pattern: Peg, repl: string]]) {. - rtl, extern: "npegs$1".} = - ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an - ## error occurs. This is supposed to be used for quick scripting. - var x = readFile(infile).string - writeFile(outfile, x.parallelReplace(subs)) +when not defined(js): + proc transformFile*(infile, outfile: string, + subs: varargs[tuple[pattern: Peg, repl: string]]) {. + rtl, extern: "npegs$1".} = + ## reads in the file `infile`, performs a parallel replacement (calls + ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an + ## error occurs. This is supposed to be used for quick scripting. + ## + ## **Note**: this proc does not exist while using the JS backend. + var x = readFile(infile).string + writeFile(outfile, x.parallelReplace(subs)) + iterator split*(s: string, sep: Peg): string = ## Splits the string `s` into substrings. From 39a1ad650cb221ece78980fa1b782638fdc84df5 Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 08:42:22 +0300 Subject: [PATCH 07/19] Faster binarySearch --- lib/pure/algorithm.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 169dcd6023..daec452a0d 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -86,6 +86,9 @@ proc binarySearch*[T, K](a: openArray[T], key: K, var step = len shr 1 while step > 0: let i = result or step + if cmp(a[i], key) == 0: + return i + if cmp(a[i], key) < 1: result = i step = step shr 1 @@ -94,6 +97,9 @@ proc binarySearch*[T, K](a: openArray[T], key: K, var b = len while result < b: var mid = (result + b) shr 1 + if cmp(a[mid], key) == 0: + return mid + if cmp(a[mid], key) < 0: result = mid + 1 else: From 6acbe6fb019b149d73bc702335f2768619ac1ac3 Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 09:45:51 +0300 Subject: [PATCH 08/19] Fixes pegs bugs --- lib/pure/pegs.nim | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index e13bc83298..23ed855a44 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1125,7 +1125,7 @@ proc handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 - if L.buf[result] == '\L': inc(result) + if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result proc handleLF(L: var PegLexer, pos: int): int = @@ -1221,7 +1221,7 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = proc skip(c: var PegLexer) = var pos = c.bufpos var buf = c.buf - while true: + while pos < c.buf.len: case buf[pos] of ' ', '\t': inc(pos) @@ -1242,7 +1242,7 @@ proc getString(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 var buf = c.buf var quote = buf[pos-1] - while true: + while pos < c.buf.len: case buf[pos] of '\\': c.bufpos = pos @@ -1265,7 +1265,7 @@ proc getDollar(c: var PegLexer, tok: var Token) = if buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 - while buf[pos] in {'0'..'9'}: + while pos < c.buf.len and buf[pos] in {'0'..'9'}: tok.index = tok.index * 10 + ord(buf[pos]) - ord('0') inc(pos) else: @@ -1281,7 +1281,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = if buf[pos] == '^': inc(pos) caret = true - while true: + while pos < c.buf.len: var ch: char case buf[pos] of ']': @@ -1300,7 +1300,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = inc(pos) incl(tok.charset, ch) if buf[pos] == '-': - if buf[pos+1] == ']': + if pos+1 < c.buf.len and buf[pos+1] == ']': incl(tok.charset, '-') inc(pos) else: @@ -1326,10 +1326,10 @@ proc getCharSet(c: var PegLexer, tok: var Token) = proc getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos var buf = c.buf - while true: + while pos < c.buf.len: add(tok.literal, buf[pos]) inc(pos) - if buf[pos] notin strutils.IdentChars: break + if pos < buf.len and buf[pos] notin strutils.IdentChars: break c.bufpos = pos tok.kind = tkIdentifier @@ -1451,8 +1451,8 @@ proc getTok(c: var PegLexer, tok: var Token) = proc arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos - while c.buf[pos] in {'\t', ' '}: inc(pos) - result = c.buf[pos] == '<' and c.buf[pos+1] == '-' + while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) + result = c.buf[pos] == '<' and (pos+1 < c.buf.len) and c.buf[pos+1] == '-' # ----------------------------- parser ---------------------------------------- @@ -1475,7 +1475,7 @@ proc pegError(p: PegParser, msg: string, line = -1, col = -1) = proc getTok(p: var PegParser) = getTok(p, p.tok) - if p.tok.kind == tkInvalid: pegError(p, "invalid token") + if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") proc eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) From df37796d88d024e7511ec134dc5405f9d65cd3e1 Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 10:23:19 +0300 Subject: [PATCH 09/19] Fixes pegs bugs, continue --- lib/pure/pegs.nim | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 23ed855a44..39c5790ed8 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1226,7 +1226,8 @@ proc skip(c: var PegLexer) = of ' ', '\t': inc(pos) of '#': - while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) + while (pos < c.buf.len) and + not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) of '\c': pos = handleCR(c, pos) buf = c.buf @@ -1285,7 +1286,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = var ch: char case buf[pos] of ']': - inc(pos) + if pos < c.buf.len: inc(pos) break of '\\': c.bufpos = pos @@ -1304,7 +1305,10 @@ proc getCharSet(c: var PegLexer, tok: var Token) = incl(tok.charset, '-') inc(pos) else: - inc(pos) + if pos+1 < c.buf.len: + inc(pos) + else: + break var ch2: char case buf[pos] of '\\': @@ -1316,8 +1320,11 @@ proc getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid break else: - ch2 = buf[pos] - inc(pos) + if pos+1 < c.buf.len: + ch2 = buf[pos] + inc(pos) + else: + break for i in ord(ch)+1 .. ord(ch2): incl(tok.charset, chr(i)) c.bufpos = pos @@ -1334,7 +1341,7 @@ proc getSymbol(c: var PegLexer, tok: var Token) = tok.kind = tkIdentifier proc getBuiltin(c: var PegLexer, tok: var Token) = - if c.buf[c.bufpos+1] in strutils.Letters: + if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) tok.kind = tkBuiltin @@ -1347,10 +1354,12 @@ proc getTok(c: var PegLexer, tok: var Token) = tok.modifier = modNone setLen(tok.literal, 0) skip(c) + case c.buf[c.bufpos] of '{': inc(c.bufpos) - if c.buf[c.bufpos] == '@' and c.buf[c.bufpos+1] == '}': + if c.buf[c.bufpos] == '@' and c.bufpos+2 < c.buf.len and + c.buf[c.bufpos+1] == '}': tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") @@ -1383,13 +1392,11 @@ proc getTok(c: var PegLexer, tok: var Token) = getBuiltin(c, tok) of '\'', '"': getString(c, tok) of '$': getDollar(c, tok) - of '\0': - tok.kind = tkEof - tok.literal = "[EOF]" of 'a'..'z', 'A'..'Z', '\128'..'\255': getSymbol(c, tok) if c.buf[c.bufpos] in {'\'', '"'} or - c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and + c.buf[c.bufpos+1] in {'0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1410,7 +1417,7 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '+') of '<': - if c.buf[c.bufpos+1] == '-': + if c.bufpos+2 < c.buf.len and c.buf[c.bufpos+1] == '-': inc(c.bufpos, 2) tok.kind = tkArrow add(tok.literal, "<-") @@ -1445,6 +1452,9 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '^') else: + if c.bufpos >= c.buf.len: + tok.kind = tkEof + tok.literal = "[EOF]" add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) From a093605ab06ef851bf177fc444711db8435d7c8f Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 12:40:55 +0300 Subject: [PATCH 10/19] binarySearch became even better --- lib/pure/algorithm.nim | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index daec452a0d..35315947d1 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -84,27 +84,31 @@ proc binarySearch*[T, K](a: openArray[T], key: K, if (len and (len - 1)) == 0: # when `len` is a power of 2, a faster shr can be used. var step = len shr 1 + var cmpRes: int while step > 0: let i = result or step - if cmp(a[i], key) == 0: + cmpRes = cmp(a[i], key) + if cmpRes == 0: return i - if cmp(a[i], key) < 1: + if cmpRes < 1: result = i step = step shr 1 - if cmp(a[result], key) != 0: result = -1 + if cmpRes != 0: result = -1 else: var b = len + var cmpRes: int while result < b: var mid = (result + b) shr 1 - if cmp(a[mid], key) == 0: + cmpRes = cmp(a[mid], key) + if cmpRes == 0: return mid - if cmp(a[mid], key) < 0: + if cmpRes < 0: result = mid + 1 else: b = mid - if result >= len or cmp(a[result], key) != 0: result = -1 + if result >= len or cmpRes != 0: result = -1 proc binarySearch*[T](a: openArray[T], key: T): int = ## binary search for `key` in `a`. Returns -1 if not found. From 17b8bb8b47ef77877c5dac17abaf9998a12eea67 Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 12:58:26 +0300 Subject: [PATCH 11/19] Fixes binarySearch's bug --- lib/pure/algorithm.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 35315947d1..98330b6809 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -94,7 +94,7 @@ proc binarySearch*[T, K](a: openArray[T], key: K, if cmpRes < 1: result = i step = step shr 1 - if cmpRes != 0: result = -1 + if cmp(a[result], key) != 0: result = -1 else: var b = len var cmpRes: int @@ -108,7 +108,7 @@ proc binarySearch*[T, K](a: openArray[T], key: K, result = mid + 1 else: b = mid - if result >= len or cmpRes != 0: result = -1 + if result >= len or cmp(a[result], key) != 0: result = -1 proc binarySearch*[T](a: openArray[T], key: T): int = ## binary search for `key` in `a`. Returns -1 if not found. From 85b7d8fcc4032a0d95af3f25faf1cc850f7d59f5 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 23 May 2018 15:28:53 +0100 Subject: [PATCH 12/19] Rstgen/xml tree fixes (#7823) * Don't prefix lang names with "lang" in rstgen. * Implements ability to render xmltree w/o \n. Fixes <> for `data-*` attrs. * Various rstgen fixes. * Fixes security vulnerabilities due to not escaping some code. * Adds