From 08f1c6100b307e8a6f5eeb8f6647552c10d4481d Mon Sep 17 00:00:00 2001 From: Erwan Ameil Date: Fri, 29 Aug 2014 11:29:19 +0200 Subject: [PATCH 1/8] Add postgresql prepared queries and stop relying on string formatting for sql parameter passing --- lib/impure/db_postgres.nim | 112 +++++++++++++++++++++++++++---------- lib/wrappers/postgres.nim | 2 + 2 files changed, 86 insertions(+), 28 deletions(-) diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim index f6ae933030..c375d91910 100644 --- a/lib/impure/db_postgres.nim +++ b/lib/impure/db_postgres.nim @@ -19,12 +19,13 @@ type EDb* = object of EIO ## exception that is raised if a database error occurs TSqlQuery* = distinct string ## an SQL query string + TSqlPrepared* = distinct string ## a identifier for the prepared queries FDb* = object of FIO ## effect that denotes a database operation FReadDb* = object of FDB ## effect that denotes a read operation FWriteDb* = object of FDB ## effect that denotes a write operation - -proc sql*(query: string): TSqlQuery {.noSideEffect, inline.} = + +proc sql*(query: string): TSqlQuery {.noSideEffect, inline.} = ## constructs a TSqlQuery from the string `query`. This is supposed to be ## used as a raw-string-literal modifier: ## ``sql"update user set counter = counter + 1"`` @@ -33,14 +34,14 @@ proc sql*(query: string): TSqlQuery {.noSideEffect, inline.} = ## on, later versions will check the string for valid syntax. result = TSqlQuery(query) -proc dbError(db: TDbConn) {.noreturn.} = +proc dbError*(db: TDbConn) {.noreturn.} = ## raises an EDb exception. var e: ref EDb new(e) e.msg = $PQerrorMessage(db) raise e -proc dbError*(msg: string) {.noreturn.} = +proc dbError*(msg: string) {.noreturn.} = ## raises an EDb exception with message `msg`. var e: ref EDb new(e) @@ -61,41 +62,70 @@ proc dbFormat(formatstr: TSqlQuery, args: varargs[string]): string = if c == '?': add(result, dbQuote(args[a])) inc(a) - else: + else: add(result, c) -proc tryExec*(db: TDbConn, query: TSqlQuery, +proc tryExec*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]): bool {.tags: [FReadDB, FWriteDb].} = ## tries to execute the query and returns true if successful, false otherwise. - var q = dbFormat(query, args) - var res = PQExec(db, q) + var arr = allocCStringArray(args) + var res = PQexecParams(db, query.string, int32(args.len), nil, arr, + nil, nil, 0) + deallocCStringArray(arr) result = PQresultStatus(res) == PGRES_COMMAND_OK PQclear(res) proc exec*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]) {. tags: [FReadDB, FWriteDb].} = ## executes the query and raises EDB if not successful. - var q = dbFormat(query, args) - var res = PQExec(db, q) + var arr = allocCStringArray(args) + var res = PQexecParams(db, query.string, int32(args.len), nil, arr, + nil, nil, 0) + deallocCStringArray(arr) if PQresultStatus(res) != PGRES_COMMAND_OK: dbError(db) PQclear(res) - + +proc exec*(db: TDbConn, stmtName: TSqlPrepared, + args: varargs[string]) {.tags: [FReadDB, FWriteDb].} = + var arr = allocCStringArray(args) + var res = PQexecPrepared(db, stmtName.string, int32(args.len), arr, + nil, nil, 0) + deallocCStringArray(arr) + if PQResultStatus(res) != PGRES_COMMAND_OK: dbError(db) + PQclear(res) + proc newRow(L: int): TRow = newSeq(result, L) for i in 0..L-1: result[i] = "" -proc setupQuery(db: TDbConn, query: TSqlQuery, - args: varargs[string]): PPGresult = - var q = dbFormat(query, args) - result = PQExec(db, q) - if PQresultStatus(result) != PGRES_TUPLES_OK: dbError(db) - +proc setupQuery(db: TDbConn, query: TSqlQuery, + args: varargs[string]): PPGresult = + var arr = allocCStringArray(args) + result = PQexecParams(db, query.string, int32(args.len), nil, arr, + nil, nil, 0) + deallocCStringArray(arr) + if PQResultStatus(result) != PGRES_TUPLES_OK: dbError(db) + +proc setupQuery(db: TDbConn, stmtName: TSqlPrepared, + args: varargs[string]): PPGresult = + var arr = allocCStringArray(args) + result = PQexecPrepared(db, stmtName.string, int32(args.len), arr, + nil, nil, 0) + deallocCStringArray(arr) + if PQResultStatus(result) != PGRES_TUPLES_OK: dbError(db) + +proc prepare*(db: TDbConn; stmtName: string, query: TSqlQuery; + nParams: int): TSqlPrepared = + var res = PQprepare(db, stmtName, query.string, int32(nParams), nil) + if PQResultStatus(res) != PGRES_COMMAND_OK: dbError(db) + return TSqlPrepared(stmtName) + proc setRow(res: PPGresult, r: var TRow, line, cols: int32) = for col in 0..cols-1: setLen(r[col], 0) var x = PQgetvalue(res, line, col) add(r[col], x) - + iterator fastRows*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]): TRow {.tags: [FReadDB].} = ## executes the query and iterates over the result dataset. This is very @@ -109,6 +139,17 @@ iterator fastRows*(db: TDbConn, query: TSqlQuery, yield result PQclear(res) +iterator fastRows*(db: TDbConn, stmtName: TSqlPrepared, + args: varargs[string, `$`]): TRow {.tags: [FReadDB].} = + ## executes the prepared query and iterates over the result dataset. + var res = setupQuery(db, stmtName, args) + var L = PQnfields(res) + var result = newRow(L) + for i in 0..PQntuples(res)-1: + setRow(res, result, i, L) + yield result + PQclear(res) + proc getRow*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]): TRow {.tags: [FReadDB].} = ## retrieves a single row. If the query doesn't return any rows, this proc @@ -119,40 +160,55 @@ proc getRow*(db: TDbConn, query: TSqlQuery, setRow(res, result, 0, L) PQclear(res) -proc getAllRows*(db: TDbConn, query: TSqlQuery, +proc getRow*(db: TDbConn, stmtName: TSqlPrepared, + args: varargs[string, `$`]): TRow {.tags: [FReadDB].} = + var res = setupQuery(db, stmtName, args) + var L = PQnfields(res) + result = newRow(L) + setRow(res, result, 0, L) + PQclear(res) + +proc getAllRows*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]): seq[TRow] {.tags: [FReadDB].} = ## executes the query and returns the whole result dataset. result = @[] for r in FastRows(db, query, args): result.add(r) -iterator rows*(db: TDbConn, query: TSqlQuery, +proc getAllRows*(db: TDbConn, stmtName: TSqlPrepared, + args: varargs[string, `$`]): seq[TRow] {.tags: [FReadDB].} = + ## executes the prepared query and returns the whole result dataset. + result = @[] + for r in FastRows(db, stmtName, args): + result.add(r) + +iterator rows*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]): TRow {.tags: [FReadDB].} = ## same as `FastRows`, but slower and safe. for r in items(GetAllRows(db, query, args)): yield r -proc getValue*(db: TDbConn, query: TSqlQuery, - args: varargs[string, `$`]): string {.tags: [FReadDB].} = +proc getValue*(db: TDbConn, query: TSqlQuery, + args: varargs[string, `$`]): string {.tags: [FReadDB].} = ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. var x = PQgetvalue(setupQuery(db, query, args), 0, 0) result = if isNil(x): "" else: $x -proc tryInsertID*(db: TDbConn, query: TSqlQuery, +proc tryInsertID*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]): int64 {.tags: [FWriteDb].}= ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. For Postgre this adds ## ``RETURNING id`` to the query, so it only works if your primary key is ## named ``id``. - var x = PQgetvalue(setupQuery(db, TSqlQuery(string(query) & " RETURNING id"), + var x = PQgetvalue(setupQuery(db, TSqlQuery(string(query) & " RETURNING id"), args), 0, 0) if not isNil(x): result = ParseBiggestInt($x) else: result = -1 -proc insertID*(db: TDbConn, query: TSqlQuery, +proc insertID*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row. For Postgre this adds @@ -161,9 +217,9 @@ proc insertID*(db: TDbConn, query: TSqlQuery, result = TryInsertID(db, query, args) if result < 0: dbError(db) -proc execAffectedRows*(db: TDbConn, query: TSqlQuery, +proc execAffectedRows*(db: TDbConn, query: TSqlQuery, args: varargs[string, `$`]): int64 {.tags: [ - FReadDB, FWriteDb].} = + FReadDB, FWriteDb].} = ## executes the query (typically "UPDATE") and returns the ## number of affected rows. var q = dbFormat(query, args) @@ -172,7 +228,7 @@ proc execAffectedRows*(db: TDbConn, query: TSqlQuery, result = parseBiggestInt($PQcmdTuples(res)) PQclear(res) -proc close*(db: TDbConn) {.tags: [FDb].} = +proc close*(db: TDbConn) {.tags: [FDb].} = ## closes the database connection. if db != nil: PQfinish(db) diff --git a/lib/wrappers/postgres.nim b/lib/wrappers/postgres.nim index d99e5651c5..ce78d34352 100644 --- a/lib/wrappers/postgres.nim +++ b/lib/wrappers/postgres.nim @@ -213,6 +213,8 @@ proc PQexecParams*(conn: PPGconn, command: cstring, nParams: int32, paramTypes: POid, paramValues: cstringArray, paramLengths, paramFormats: ptr int32, resultFormat: int32): PPGresult{. cdecl, dynlib: dllName, importc: "PQexecParams".} +proc PQprepare*(conn: PPGconn, stmtName, query: cstring, nParams: int32, + paramTypes: POid): PPGresult{.cdecl, dynlib: dllName, importc: "PQprepare".} proc PQexecPrepared*(conn: PPGconn, stmtName: cstring, nParams: int32, paramValues: cstringArray, paramLengths, paramFormats: ptr int32, resultFormat: int32): PPGresult{. From 80356f1cc76250706f23daa0c729c91127c4812b Mon Sep 17 00:00:00 2001 From: Reimer Behrends Date: Fri, 19 Sep 2014 03:34:00 +0200 Subject: [PATCH 2/8] Avoid unnecessary #include triggered by importc vars. When a C variable or macro is imported via an {.importc.} var or let statement, but no definition is needed and the variable does not have an initializer part, then there is also no need to generate an #include for the associated header until and unless the variable is actually used. The header is already generated upon use, but unnecessarily also when the variable is defined. This is an issue with the posix module in particular, where a lot of unnecessary header files are being included because relevant constants are defined via importc vars, and those header files may not even be available on a given system. This patch omits the generation of the #include directive for those definitions where they aren't needed. --- compiler/ccgstmts.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 3004e353bc..cb9eebb32f 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -171,6 +171,10 @@ proc genSingleVar(p: BProc, a: PNode) = if sfCompileTime in v.flags: return var targetProc = p if sfGlobal in v.flags: + if v.flags * {sfImportc, sfExportc} == {sfImportc} and + a.sons[2].kind == nkEmpty and + v.loc.flags * {lfHeader, lfNoDecl} != {}: + return if sfPure in v.flags: # v.owner.kind != skModule: targetProc = p.module.preInitProc From d0b292b4668e3837c090aefc056867f09fa92ee2 Mon Sep 17 00:00:00 2001 From: Reimer Behrends Date: Mon, 22 Sep 2014 23:18:14 +0200 Subject: [PATCH 3/8] Fix permissions for createDir() on Unix systems. Permissions were set to 0o711 by default; they should be 0o777, with umask being responsible for restricting permissions further. --- lib/pure/os.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index cfff58eb09..71089494f0 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1332,10 +1332,10 @@ proc removeDir*(dir: string) {.rtl, extern: "nos$1", tags: [ proc rawCreateDir(dir: string) = when defined(solaris): - if mkdir(dir, 0o711) != 0'i32 and errno != EEXIST and errno != ENOSYS: + if mkdir(dir, 0o777) != 0'i32 and errno != EEXIST and errno != ENOSYS: osError(osLastError()) elif defined(unix): - if mkdir(dir, 0o711) != 0'i32 and errno != EEXIST: + if mkdir(dir, 0o777) != 0'i32 and errno != EEXIST: osError(osLastError()) else: when useWinUnicode: From f99c40f61bc47f98390aab42f686fcb697dc9ce8 Mon Sep 17 00:00:00 2001 From: Reimer Behrends Date: Thu, 25 Sep 2014 23:29:02 +0200 Subject: [PATCH 4/8] Improve setjmp()/longjmp() performance. Exception handling for the C backend used setjmp()/longjmp() unconditionally. However, on POSIX systems, these functions save and restore the signal mask, adding considerable overhead to exception handling, even where no exceptions are involved. The compiler and library now try to use either _setjmp()/_longjmp() or sigsetjmp()/siglongjmp() where possible, marked by the defines "nimRawSetjmp" and "nimSigSetjmp", respectively. The define "nimStdSetjmp" can be used to revert to setjmp()/longjmp() instead. --- compiler/ccgstmts.nim | 9 ++++++++- compiler/cgen.nim | 2 +- compiler/condsyms.nim | 2 ++ lib/system/ansi_c.nim | 21 +++++++++++++++++---- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index cb9eebb32f..8b0a8e9bb5 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -832,7 +832,14 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) = discard cgsym(p.module, "E_Base") linefmt(p, cpsLocals, "#TSafePoint $1;$n", safePoint) linefmt(p, cpsStmts, "#pushSafePoint(&$1);$n", safePoint) - linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", safePoint) + if isDefined("nimStdSetjmp"): + linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", safePoint) + elif isDefined("nimSigSetjmp"): + linefmt(p, cpsStmts, "$1.status = sigsetjmp($1.context, 0);$n", safePoint) + elif isDefined("nimRawSetjmp"): + linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", safePoint) + else: + linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", safePoint) startBlock(p, "if ($1.status == 0) {$n", [safePoint]) var length = sonsLen(t) add(p.nestedTryStmts, t) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 315b55801c..359fa33098 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -13,7 +13,7 @@ import ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options, intsets, nversion, nimsets, msgs, crc, bitsets, idents, lists, types, ccgutils, os, - times, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, + times, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, semparallel diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index dc61d2f894..a2d8b35641 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -136,3 +136,5 @@ proc initDefines*() = declareSymbol("emulatedthreadvars") if platform.OS[targetOS].props.contains(ospLacksThreadVars): defineSymbol("emulatedthreadvars") + if isDefined("posix"): + defineSymbol("nimRawSetjmp") diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index e012697ae7..673d55582c 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -78,10 +78,23 @@ when defined(macosx): else: template SIGBUS: expr = SIGSEGV -proc c_longjmp(jmpb: C_JmpBuf, retval: cint) {. - header: "", importc: "longjmp".} -proc c_setjmp(jmpb: C_JmpBuf): cint {. - header: "", importc: "setjmp".} +when defined(nimSigSetjmp) and not defined(nimStdSetjmp): + proc c_longjmp(jmpb: C_JmpBuf, retval: cint) {. + header: "", importc: "siglongjmp".} + template c_setjmp(jmpb: C_JmpBuf): cint = + proc c_sigsetjmp(jmpb: C_JmpBuf, savemask: cint): cint {. + header: "", importc: "sigsetjmp".} + c_sigsetjmp(jmpb, 0) +elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): + proc c_longjmp(jmpb: C_JmpBuf, retval: cint) {. + header: "", importc: "_longjmp".} + proc c_setjmp(jmpb: C_JmpBuf): cint {. + header: "", importc: "_setjmp".} +else: + proc c_longjmp(jmpb: C_JmpBuf, retval: cint) {. + header: "", importc: "longjmp".} + proc c_setjmp(jmpb: C_JmpBuf): cint {. + header: "", importc: "setjmp".} proc c_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {. importc: "signal", header: "".} From cb6441e73d976dcf59b1e55236887fe5d3a1d6ec Mon Sep 17 00:00:00 2001 From: Reimer Behrends Date: Sat, 27 Sep 2014 18:05:30 +0200 Subject: [PATCH 5/8] Use _setjmp()/_longjmp() only on BSD-like systems for now. --- compiler/condsyms.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index a2d8b35641..238dbf5c79 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -136,5 +136,7 @@ proc initDefines*() = declareSymbol("emulatedthreadvars") if platform.OS[targetOS].props.contains(ospLacksThreadVars): defineSymbol("emulatedthreadvars") - if isDefined("posix"): + case targetOS + of osSolaris, osNetbsd, osFreebsd, osOpenbsd, osMacosx: defineSymbol("nimRawSetjmp") + else: discard From b234d311d49df927b709891f5f640b90cf982ee4 Mon Sep 17 00:00:00 2001 From: Varriount Date: Sun, 28 Sep 2014 15:35:35 -0400 Subject: [PATCH 6/8] Disable git hashing in the version command --- compiler/commands.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index c15cc674ca..87e94d9c9d 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -80,9 +80,9 @@ proc writeVersionInfo(pass: TCmdLinePass) = platform.OS[platform.hostOS].name, CPU[platform.hostCPU].name])) - const gitHash = gorge("git log -n 1 --format=%H") + discard """const gitHash = gorge("git log -n 1 --format=%H") if gitHash.strip.len == 40: - msgWriteln("git hash: " & gitHash) + msgWriteln("git hash: " & gitHash)""" msgWriteln("active boot switches:" & usedRelease & usedAvoidTimeMachine & usedTinyC & usedGnuReadline & usedNativeStacktrace & usedNoCaas & From 93d55c077ffc9b9df381b4f921a16960c7141f19 Mon Sep 17 00:00:00 2001 From: Varriount Date: Sun, 28 Sep 2014 15:47:09 -0400 Subject: [PATCH 7/8] Updated Version Number --- compiler/nversion.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/nversion.nim b/compiler/nversion.nim index 3c868ed2af..bd1f79ac6b 100644 --- a/compiler/nversion.nim +++ b/compiler/nversion.nim @@ -14,7 +14,7 @@ const MaxSetElements* = 1 shl 16 # (2^16) to support unicode character sets? VersionMajor* = 0 VersionMinor* = 9 - VersionPatch* = 5 + VersionPatch* = 6 VersionAsString* = $VersionMajor & "." & $VersionMinor & "." & $VersionPatch RodFileVersion* = "1215" # modify this if the rod-format changes! From 55c78af9c0a7a63683ee49044ece0667f699adc8 Mon Sep 17 00:00:00 2001 From: Clay Sweetser Date: Wed, 1 Oct 2014 18:14:06 -0400 Subject: [PATCH 8/8] Fixes #1529 --- compiler/procfind.nim | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/compiler/procfind.nim b/compiler/procfind.nim index 0354d585d3..9f52cc117c 100644 --- a/compiler/procfind.nim +++ b/compiler/procfind.nim @@ -70,8 +70,15 @@ proc searchForProcNew(c: PContext, scope: PScope, fn: PSym): PSym = var it: TIdentIter result = initIdentIter(it, scope.symbols, fn.name) while result != nil: - if result.kind in skProcKinds and - sameType(result.typ, fn.typ, flags): return + if result.kind in skProcKinds and sameType(result.typ, fn.typ, flags): + case equalParams(result.typ.n, fn.typ.n) + of paramsEqual: + return + of paramsIncompatible: + localError(fn.info, errNotOverloadable, fn.name.s) + return + of paramsNotEqual: + discard result = nextIdentIter(it, scope.symbols)