From 188bba2b3cd3106e0c615e712b8f5fdd995137bb Mon Sep 17 00:00:00 2001 From: Flaviu Tamas Date: Sat, 23 May 2015 12:59:15 -0400 Subject: [PATCH 01/20] Clean up crc module Use better names, remove quite a bit of dead code. Change `><` to a name that's actually descriptive. --- compiler/crc.nim | 64 ++++++------------------------------------- compiler/extccomp.nim | 14 +++++----- compiler/modules.nim | 8 +++--- compiler/rodread.nim | 12 ++++---- compiler/rodwrite.nim | 8 +++--- 5 files changed, 30 insertions(+), 76 deletions(-) diff --git a/compiler/crc.nim b/compiler/crc.nim index a8b61f2a58..7a2b6e4c20 100644 --- a/compiler/crc.nim +++ b/compiler/crc.nim @@ -11,18 +11,12 @@ import strutils type - TCrc32* = int32 + SecureHash* = int32 const - InitCrc32* = TCrc32(- 1) - InitAdler32* = int32(1) + InitCrc32 = SecureHash(- 1) -proc updateCrc32*(val: int8, crc: TCrc32): TCrc32 {.inline.} -proc updateCrc32*(val: char, crc: TCrc32): TCrc32 {.inline.} -proc crcFromBuf*(buf: pointer, length: int): TCrc32 -proc strCrc32*(s: string): TCrc32 -proc crcFromFile*(filename: string): TCrc32 -proc updateAdler32*(adler: int32, buf: pointer, length: int): int32 +proc secureHashFile*(filename: string): SecureHash # implementation type @@ -74,31 +68,22 @@ const - 1000256840, 1567103746, 711928724, - 1274298825, - 1022587231, 1510334235, 755167117] -proc updateCrc32(val: int8, crc: TCrc32): TCrc32 = - result = TCrc32(crc32table[(int(crc) xor (int(val) and 0x000000FF)) and - 0x000000FF]) xor (crc shr TCrc32(8)) +proc updateCrc32(val: int8, crc: SecureHash): SecureHash = + result = SecureHash(crc32table[(int(crc) xor (int(val) and 0x000000FF)) and + 0x000000FF]) xor (crc shr SecureHash(8)) -proc updateCrc32(val: char, crc: TCrc32): TCrc32 = +proc updateCrc32(val: char, crc: SecureHash): SecureHash = result = updateCrc32(toU8(ord(val)), crc) -proc strCrc32(s: string): TCrc32 = +proc secureHash*(s: string): SecureHash = result = InitCrc32 - for i in countup(0, len(s) - 1): result = updateCrc32(s[i], result) - -proc `><`*(c: TCrc32, s: string): TCrc32 = - result = c for i in 0..len(s)-1: result = updateCrc32(s[i], result) type TByteArray = array[0..10000000, int8] PByteArray = ref TByteArray -proc crcFromBuf(buf: pointer, length: int): TCrc32 = - var p = cast[PByteArray](buf) - result = InitCrc32 - for i in countup(0, length - 1): result = updateCrc32(p[i], result) - -proc crcFromFile(filename: string): TCrc32 = +proc secureHashFile(filename: string): SecureHash = const bufSize = 8000 # don't use 8K for the memory allocator! var @@ -114,34 +99,3 @@ proc crcFromFile(filename: string): TCrc32 = if readBytes != bufSize: break dealloc(buf) close(bin) - -const - base = int32(65521) # largest prime smaller than 65536 - # NMAX = 5552; original code with unsigned 32 bit integer - # NMAX is the largest n - # such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 - nmax = 3854 # code with signed 32 bit integer - # NMAX is the largest n such that - # 255n(n+1)/2 + (n+1)(BASE-1) <= 2^31-1 - # The penalty is the time loss in the extra MOD-calls. - -proc updateAdler32(adler: int32, buf: pointer, length: int): int32 = - var - s1, s2: int32 - L, k, b: int - s1 = adler and int32(0x0000FFFF) - s2 = (adler shr int32(16)) and int32(0x0000FFFF) - L = length - b = 0 - while (L > 0): - if L < nmax: k = L - else: k = nmax - dec(L, k) - while (k > 0): - s1 = s1 +% int32((cast[cstring](buf))[b]) - s2 = s2 +% s1 - inc(b) - dec(k) - s1 = `%%`(s1, base) - s2 = `%%`(s2, base) - result = (s2 shl int32(16)) or s1 diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 186a3884df..cd7ac2a683 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -572,13 +572,13 @@ proc getCompileCFileCmd*(cfilename: string, isExternal = false): string = "nim", quoteShell(getPrefixDir()), "lib", quoteShell(libpath)]) -proc footprint(filename: string): TCrc32 = - # note, '><' further modifies a crc value with a string. - result = crcFromFile(filename) >< - platform.OS[targetOS].name >< - platform.CPU[targetCPU].name >< - extccomp.CC[extccomp.cCompiler].name >< - getCompileCFileCmd(filename, true) +proc footprint(filename: string): SecureHash = + result = secureHash( + $secureHashFile(filename) & + platform.OS[targetOS].name & + platform.CPU[targetCPU].name & + extccomp.CC[extccomp.cCompiler].name & + getCompileCFileCmd(filename, true)) proc externalFileChanged(filename: string): bool = if gCmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM}: diff --git a/compiler/modules.nim b/compiler/modules.nim index 2fa46f3568..09abd5da67 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -19,7 +19,7 @@ type TModuleInMemory* = object compiledAt*: float - crc*: TCrc32 + crc*: SecureHash deps*: seq[int32] ## XXX: slurped files are currently not tracked needsRecompile*: TNeedRecompile crcStatus*: TCrcStatus @@ -51,19 +51,19 @@ proc crcChanged(fileIdx: int32): bool = of crcNotChanged: result = false of crcCached: - let newCrc = crcFromFile(fileIdx.toFilename) + let newCrc = secureHashFile(fileIdx.toFilename) result = newCrc != gMemCacheData[fileIdx].crc gMemCacheData[fileIdx].crc = newCrc updateStatus() of crcNotTaken: - gMemCacheData[fileIdx].crc = crcFromFile(fileIdx.toFilename) + gMemCacheData[fileIdx].crc = secureHashFile(fileIdx.toFilename) result = true updateStatus() proc doCRC(fileIdx: int32) = if gMemCacheData[fileIdx].crcStatus == crcNotTaken: # echo "FIRST CRC: ", fileIdx.ToFilename - gMemCacheData[fileIdx].crc = crcFromFile(fileIdx.toFilename) + gMemCacheData[fileIdx].crc = secureHashFile(fileIdx.toFilename) proc addDep(x: PSym, dep: int32) = growCache gMemCacheData, dep diff --git a/compiler/rodread.nim b/compiler/rodread.nim index e92f7ecfa4..066a02b1d3 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -538,7 +538,7 @@ proc cmdChangeTriggersRecompilation(old, new: TCommands): bool = # else: trigger recompilation: result = true -proc processRodFile(r: PRodReader, crc: TCrc32) = +proc processRodFile(r: PRodReader, crc: SecureHash) = var w: string d, inclCrc: int @@ -598,7 +598,7 @@ proc processRodFile(r: PRodReader, crc: TCrc32) = inc(r.pos) # skip ' ' inclCrc = decodeVInt(r.s, r.pos) if r.reason == rrNone: - if not existsFile(w) or (inclCrc != int(crcFromFile(w))): + if not existsFile(w) or (inclCrc != int(secureHashFile(w))): r.reason = rrInclDeps if r.s[r.pos] == '\x0A': inc(r.pos) @@ -649,7 +649,7 @@ proc startsWith(buf: cstring, token: string, pos = 0): bool = while s < token.len and buf[pos+s] == token[s]: inc s result = s == token.len -proc newRodReader(modfilename: string, crc: TCrc32, +proc newRodReader(modfilename: string, crc: SecureHash, readerIndex: int): PRodReader = new(result) try: @@ -701,7 +701,7 @@ type filename*: string reason*: TReasonForRecompile rd*: PRodReader - crc*: TCrc32 + crc*: SecureHash crcDone*: bool TFileModuleMap = seq[TFileModuleRec] @@ -794,13 +794,13 @@ proc loadMethods(r: PRodReader) = r.methods.add(rrGetSym(r, d, unknownLineInfo())) if r.s[r.pos] == ' ': inc(r.pos) -proc getCRC*(fileIdx: int32): TCrc32 = +proc getCRC*(fileIdx: int32): SecureHash = internalAssert fileIdx >= 0 and fileIdx < gMods.len if gMods[fileIdx].crcDone: return gMods[fileIdx].crc - result = crcFromFile(fileIdx.toFilename) + result = secureHashFile(fileIdx.toFilename) gMods[fileIdx].crc = result template growCache*(cache, pos) = diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index e178b7ce6e..c83e728db8 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -20,7 +20,7 @@ import type TRodWriter = object of TPassContext module: PSym - crc: TCrc32 + crc: SecureHash options: TOptions defines: string inclDeps: string @@ -38,7 +38,7 @@ type PRodWriter = ref TRodWriter -proc newRodWriter(crc: TCrc32, module: PSym): PRodWriter +proc newRodWriter(crc: SecureHash, module: PSym): PRodWriter proc addModDep(w: PRodWriter, dep: string) proc addInclDep(w: PRodWriter, dep: string) proc addInterfaceSym(w: PRodWriter, s: PSym) @@ -62,7 +62,7 @@ proc fileIdx(w: PRodWriter, filename: string): int = template filename*(w: PRodWriter): string = w.module.filename -proc newRodWriter(crc: TCrc32, module: PSym): PRodWriter = +proc newRodWriter(crc: SecureHash, module: PSym): PRodWriter = new(result) result.sstack = @[] result.tstack = @[] @@ -96,7 +96,7 @@ proc addInclDep(w: PRodWriter, dep: string) = var resolved = dep.findModule(w.module.info.toFullPath) encodeVInt(fileIdx(w, dep), w.inclDeps) add(w.inclDeps, " ") - encodeVInt(crcFromFile(resolved), w.inclDeps) + encodeVInt(secureHashFile(resolved), w.inclDeps) add(w.inclDeps, rodNL) proc pushType(w: PRodWriter, t: PType) = From 4873db4c7218d1abc4cf43a17e1330628362801d Mon Sep 17 00:00:00 2001 From: Flaviu Tamas Date: Sat, 23 May 2015 13:17:19 -0400 Subject: [PATCH 02/20] Rename crc module to "secure_hash" --- compiler/ast.nim | 2 +- compiler/cgen.nim | 11 +++++------ compiler/extccomp.nim | 2 +- compiler/jsgen.nim | 4 ++-- compiler/modules.nim | 2 +- compiler/rodread.nim | 2 +- compiler/rodwrite.nim | 2 +- compiler/{crc.nim => secure_hash.nim} | 0 8 files changed, 12 insertions(+), 13 deletions(-) rename compiler/{crc.nim => secure_hash.nim} (100%) diff --git a/compiler/ast.nim b/compiler/ast.nim index 3798410e87..fd9c5478b1 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -10,7 +10,7 @@ # abstract syntax tree + symbol table import - msgs, hashes, nversion, options, strutils, crc, ropes, idents, lists, + msgs, hashes, nversion, options, strutils, secure_hash, ropes, idents, lists, intsets, idgen type diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 4b0bac28ad..c3e6ac0d65 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -10,12 +10,11 @@ ## This module implements the C code generator. import - ast, astalgo, hashes, trees, platform, magicsys, extccomp, - options, intsets, - nversion, nimsets, msgs, crc, bitsets, idents, lists, types, ccgutils, os, - ropes, math, passes, rodread, wordrecg, treetab, cgmeth, condsyms, - rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, - semparallel + ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets, + nversion, nimsets, msgs, secure_hash, bitsets, idents, lists, types, + ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, + condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, + lowerings, semparallel import strutils except `%` # collides with ropes.`%` diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index cd7ac2a683..db6057b5e8 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -13,7 +13,7 @@ # nim files. import - lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, crc + lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, secure_hash type TSystemCC* = enum diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 5c70714983..ebcaf32c42 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -30,8 +30,8 @@ implements the required case distinction. import - ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, - options, nversion, nimsets, msgs, crc, bitsets, idents, lists, types, os, + ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options, + nversion, nimsets, msgs, secure_hash, bitsets, idents, lists, types, os, times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils, intsets, cgmeth, lowerings diff --git a/compiler/modules.nim b/compiler/modules.nim index 09abd5da67..0573b91f8a 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -10,7 +10,7 @@ ## implements the module handling import - ast, astalgo, magicsys, crc, rodread, msgs, cgendata, sigmatch, options, + ast, astalgo, magicsys, secure_hash, rodread, msgs, cgendata, sigmatch, options, idents, os, lexer, idgen, passes, syntaxes, llstream type diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 066a02b1d3..51d60f5b54 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -90,7 +90,7 @@ import os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms, - ropes, idents, crc, idgen, types, rodutils, memfiles + ropes, idents, secure_hash, idgen, types, rodutils, memfiles type TReasonForRecompile* = enum ## all the reasons that can trigger recompilation diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index c83e728db8..22ecdd8af7 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -13,7 +13,7 @@ import intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform, - condsyms, ropes, idents, crc, rodread, passes, importer, idgen, rodutils + condsyms, ropes, idents, secure_hash, rodread, passes, importer, idgen, rodutils # implementation diff --git a/compiler/crc.nim b/compiler/secure_hash.nim similarity index 100% rename from compiler/crc.nim rename to compiler/secure_hash.nim From e9516e6308687d3c92564ac54fc63f19470ccf2b Mon Sep 17 00:00:00 2001 From: Flaviu Tamas Date: Sat, 23 May 2015 14:34:11 -0400 Subject: [PATCH 03/20] Use sha1 instead of crc --- compiler/extccomp.nim | 6 +- compiler/rodread.nim | 14 +- compiler/rodwrite.nim | 4 +- compiler/secure_hash.nim | 269 +++++++++++++++++++++++++++------------ 4 files changed, 197 insertions(+), 96 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index db6057b5e8..2ddb98abab 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -585,13 +585,11 @@ proc externalFileChanged(filename: string): bool = return false var crcFile = toGeneratedFile(filename.withPackageName, "crc") - var currentCrc = int(footprint(filename)) + var currentCrc = footprint(filename) var f: File if open(f, crcFile, fmRead): - var line = newStringOfCap(40) - if not f.readLine(line): line = "0" + let oldCrc = parseSecureHash(f.readLine()) close(f) - var oldCrc = parseInt(line) result = oldCrc != currentCrc else: result = true diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 51d60f5b54..27cb96ca14 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -541,7 +541,8 @@ proc cmdChangeTriggersRecompilation(old, new: TCommands): bool = proc processRodFile(r: PRodReader, crc: SecureHash) = var w: string - d, inclCrc: int + d: int + var inclCrc: SecureHash while r.s[r.pos] != '\0': var section = rdWord(r) if r.reason != rrNone: @@ -549,7 +550,8 @@ proc processRodFile(r: PRodReader, crc: SecureHash) = case section of "CRC": inc(r.pos) # skip ':' - if int(crc) != decodeVInt(r.s, r.pos): r.reason = rrCrcChange + if crc != parseSecureHash(decodeStr(r.s, r.pos)): + r.reason = rrCrcChange of "ID": inc(r.pos) # skip ':' r.moduleID = decodeVInt(r.s, r.pos) @@ -596,9 +598,9 @@ proc processRodFile(r: PRodReader, crc: SecureHash) = while r.s[r.pos] != ')': w = r.files[decodeVInt(r.s, r.pos)].toFullPath inc(r.pos) # skip ' ' - inclCrc = decodeVInt(r.s, r.pos) + inclCrc = parseSecureHash(decodeStr(r.s, r.pos)) if r.reason == rrNone: - if not existsFile(w) or (inclCrc != int(secureHashFile(w))): + if not existsFile(w) or (inclCrc != secureHashFile(w)): r.reason = rrInclDeps if r.s[r.pos] == '\x0A': inc(r.pos) @@ -649,7 +651,7 @@ proc startsWith(buf: cstring, token: string, pos = 0): bool = while s < token.len and buf[pos+s] == token[s]: inc s result = s == token.len -proc newRodReader(modfilename: string, crc: SecureHash, +proc newRodReader(modfilename: string, crc: SecureHash, readerIndex: int): PRodReader = new(result) try: @@ -1017,7 +1019,7 @@ proc writeType(f: File; t: PType) = f.write("]\n") proc viewFile(rodfile: string) = - var r = newRodReader(rodfile, 0, 0) + var r = newRodReader(rodfile, secureHash(""), 0) if r == nil: rawMessage(errGenerated, "cannot open file (or maybe wrong version):" & rodfile) diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index 22ecdd8af7..3454b9ade6 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -96,7 +96,7 @@ proc addInclDep(w: PRodWriter, dep: string) = var resolved = dep.findModule(w.module.info.toFullPath) encodeVInt(fileIdx(w, dep), w.inclDeps) add(w.inclDeps, " ") - encodeVInt(secureHashFile(resolved), w.inclDeps) + encodeStr($secureHashFile(resolved), w.inclDeps) add(w.inclDeps, rodNL) proc pushType(w: PRodWriter, t: PType) = @@ -440,7 +440,7 @@ proc writeRod(w: PRodWriter) = f.write(rodNL) var crc = "CRC:" - encodeVInt(w.crc, crc) + encodeStr($w.crc, crc) f.write(crc) f.write(rodNL) diff --git a/compiler/secure_hash.nim b/compiler/secure_hash.nim index 7a2b6e4c20..a2ddd02f0a 100644 --- a/compiler/secure_hash.nim +++ b/compiler/secure_hash.nim @@ -1,101 +1,202 @@ # # # The Nim Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Nim Contributers # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -import - strutils +import + strutils, unsigned -type - SecureHash* = int32 +const sha_digest_size = 20 -const - InitCrc32 = SecureHash(- 1) +type + SHA1Digest = array[0 .. sha_digest_size-1, uint8] + SecureHash* = distinct SHA1Digest -proc secureHashFile*(filename: string): SecureHash -# implementation +const emptySecureHash = SecureHash([ + 0u8, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, +]) -type - TCRC_TabEntry = int +proc sha1(src: string) : SHA1Digest -const - crc32table: array[0..255, TCRC_TabEntry] = [0, 1996959894, - 301047508, - - 1727442502, 124634137, 1886057615, - 379345611, - 1637575261, 249268274, - 2044508324, - 522852066, - 1747789432, 162941995, 2125561021, - 407360249, - - 1866523247, 498536548, 1789927666, - 205950648, - 2067906082, 450548861, - 1843258603, - 187386543, - 2083289657, 325883990, 1684777152, - 43845254, - - 1973040660, 335633487, 1661365465, - 99664541, - 1928851979, 997073096, - 1281953886, - 715111964, - 1570279054, 1006888145, 1258607687, - 770865667, - - 1526024853, 901097722, 1119000684, - 608450090, - 1396901568, 853044451, - 1172266101, - 589951537, - 1412350631, 651767980, 1373503546, - 925412992, - - 1076862698, 565507253, 1454621731, - 809855591, - 1195530993, 671266974, - 1594198024, - 972236366, - 1324619484, 795835527, 1483230225, - 1050600021, - - 1234817731, 1994146192, 31158534, - 1731059524, - 271249366, 1907459465, - 112637215, - 1614814043, - 390540237, 2013776290, 251722036, - 1777751922, - - 519137256, 2137656763, 141376813, - 1855689577, - 429695999, 1802195444, - 476864866, - 2056965928, - 228458418, 1812370925, 453092731, - 2113342271, - - 183516073, 1706088902, 314042704, - 1950435094, - 54949764, 1658658271, - 366619977, - 1932296973, - 69972891, 1303535960, 984961486, - 1547960204, - - 725929758, 1256170817, 1037604311, - 1529756563, - 740887301, 1131014506, - 879679996, - 1385723834, - 631195440, 1141124467, 855842277, - 1442165665, - - 586318647, 1342533948, 654459306, - 1106571248, - 921952122, 1466479909, - 544179635, - 1184443383, - 832445281, 1591671054, 702138776, - 1328506846, - - 942167884, 1504918807, 783551873, - 1212326853, - 1061524307, - 306674912, - - 1698712650, 62317068, 1957810842, - 355121351, - 1647151185, 81470997, - 1943803523, - 480048366, - 1805370492, 225274430, 2053790376, - 468791541, - - 1828061283, 167816743, 2097651377, - 267414716, - 2029476910, 503444072, - 1762050814, - 144550051, - 2140837941, 426522225, 1852507879, - 19653770, - - 1982649376, 282753626, 1742555852, - 105259153, - 1900089351, 397917763, - 1622183637, - 690576408, - 1580100738, 953729732, 1340076626, - 776247311, - - 1497606297, 1068828381, 1219638859, - 670225446, - 1358292148, 906185462, - 1090812512, - 547295293, - 1469587627, 829329135, 1181335161, - 882789492, - - 1134132454, 628085408, 1382605366, - 871598187, - 1156888829, 570562233, - 1426400815, - 977650754, - 1296233688, 733239954, 1555261956, - 1026031705, - - 1244606671, 752459403, 1541320221, - 1687895376, - 328994266, 1969922972, - 40735498, - 1677130071, - 351390145, 1913087877, 83908371, - 1782625662, - - 491226604, 2075208622, 213261112, - 1831694693, - 438977011, 2094854071, - 198958881, - 2032938284, - 237706686, 1759359992, 534414190, - 2118248755, - - 155638181, 1873836001, 414664567, - 2012718362, - 15766928, 1711684554, - 285281116, - 1889165569, - 127750551, 1634467795, 376229701, - 1609899400, - - 686959890, 1308918612, 956543938, - 1486412191, - 799009033, 1231636301, - 1047427035, - 1362007478, - 640263460, 1088359270, 936918000, - 1447252397, - - 558129467, 1202900863, 817233897, - 1111625188, - 893730166, 1404277552, - 615818150, - 1160759803, - 841546093, 1423857449, 601450431, - 1285129682, - - 1000256840, 1567103746, 711928724, - 1274298825, - 1022587231, 1510334235, - 755167117] +proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) +proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) +proc `$`*(self: SecureHash): string = + result = "" + for v in SHA1Digest(self): + result.add(toHex(int(v), 2)) -proc updateCrc32(val: int8, crc: SecureHash): SecureHash = - result = SecureHash(crc32table[(int(crc) xor (int(val) and 0x000000FF)) and - 0x000000FF]) xor (crc shr SecureHash(8)) +proc parseSecureHash*(hash: string): SecureHash = + for i in 0.. = 56: + innerHash(state, w) + clearBuffer(w) + + w[15] = uint32(byteLen) shl 3 + innerHash(state, w) + + # Store hash in result pointer, and make sure we get in in the correct order + # on both endian models. + for i in 0 .. sha_digest_size-1: + result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) + +proc sha1(src: string) : SHA1Digest = + ## Calculate SHA1 from input string + computeInternal(src) From badc3a0929affb14415b178a96ee10151b795196 Mon Sep 17 00:00:00 2001 From: Flaviu Tamas Date: Sat, 23 May 2015 14:40:33 -0400 Subject: [PATCH 04/20] Add simple tests to secure_hash --- compiler/secure_hash.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compiler/secure_hash.nim b/compiler/secure_hash.nim index a2ddd02f0a..3383f23a23 100644 --- a/compiler/secure_hash.nim +++ b/compiler/secure_hash.nim @@ -42,6 +42,11 @@ proc `==`*(a, b: SecureHash): bool = SHA1Digest(a) == SHA1Digest(b) +when isMainModule: + let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") + doAssert hash1 == hash1 + doAssert parseSecureHash($hash1) == hash1 + # Copyright (c) 2011, Micael Hildenborg # All rights reserved. From 8beed9610640495185e8dce55ca1efe13a7705fd Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez Date: Sun, 31 May 2015 18:05:34 -0500 Subject: [PATCH 05/20] Colors! --- compiler/commands.nim | 3 ++ compiler/extccomp.nim | 2 +- compiler/msgs.nim | 79 +++++++++++++++++++++++++++---------------- compiler/options.nim | 1 + 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index d30d8326cd..285d73e7ac 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -532,6 +532,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = of "genscript": expectNoArg(switch, arg, pass, info) incl(gGlobalOptions, optGenScript) + of "usecolors": + expectNoArg(switch, arg, pass, info) + incl(gGlobalOptions, optUseColors) of "lib": expectArg(switch, arg, pass, info) libpath = processPath(arg, notRelativeToProj=true) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 186a3884df..ea69f6c25e 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -13,7 +13,7 @@ # nim files. import - lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, crc + lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, crc, terminal type TSystemCC* = enum diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 041a181be7..eb6707d65a 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -8,7 +8,7 @@ # import - options, strutils, os, tables, ropes, platform + options, strutils, os, tables, ropes, platform, terminal type TMsgKind* = enum @@ -601,13 +601,13 @@ proc suggestQuit*() = # this format is understood by many text editors: it is the same that # Borland and Freepascal use const - PosErrorFormat* = "$1($2, $3) Error: $4" - PosWarningFormat* = "$1($2, $3) Warning: $4" - PosHintFormat* = "$1($2, $3) Hint: $4" - PosContextFormat = "$1($2, $3) Info: $4" - RawErrorFormat* = "Error: $1" - RawWarningFormat* = "Warning: $1" - RawHintFormat* = "Hint: $1" + PosErrorFormat* = "$1($2, $3) Error: " + PosWarningFormat* = "$1($2, $3) Warning: " + PosHintFormat* = "$1($2, $3) Hint: " + PosContextFormat = "$1($2, $3) Info: " + RawError* = "Error: " + RawWarning* = "Warning: " + RawHint* = "Hint: " proc getInfoContextLen*(): int = return msgContext.len proc setInfoContextLen*(L: int) = setLen(msgContext, L) @@ -682,17 +682,27 @@ proc outWriteln*(s: string) = ## Writes to stdout. Always. if eStdOut in errorOutputs: writeln(stdout, s) -proc msgWriteln*(s: string) = - ## Writes to stdout. If --stdout option is given, writes to stderr instead. +proc msgWriteln*(s: string, color: ForegroundColor = fgWhite, coloredText: string = "") = + ## Writes to stdout. If --stderr option is given, writes to stderr instead. #if gCmd == cmdIdeTools and optCDebug notin gGlobalOptions: return + var hasColor = optUseColors in gGlobalOptions if not isNil(writelnHook): - writelnHook(s) - elif optStdout in gGlobalOptions: - if eStdErr in errorOutputs: writeln(stderr, s) + writelnHook(coloredText & s) else: - if eStdOut in errorOutputs: writeln(stdout, s) + if optStdout in gGlobalOptions: + if eStdErr in errorOutputs: + if hasColor: setForegroundColor(color) + write(stderr, coloredText) + if hasColor: resetAttributes() + writeln(stderr, s) + else: + if eStdOut in errorOutputs: + if hasColor: setForegroundColor(color) + write(stdout, coloredText) + if hasColor: resetAttributes() + writeln(stdout, s) proc coordToStr(coord: int): string = if coord == -1: result = "???" @@ -714,7 +724,7 @@ proc handleError(msg: TMsgKind, eh: TErrorHandling, s: string) = if stackTraceAvailable() and isNil(writelnHook): writeStackTrace() else: - msgWriteln("No stack traceback available\nTo create a stacktrace, rerun compilation with ./koch temp " & options.command & " ") + msgWriteln("", fgRed, "No stack traceback available\nTo create a stacktrace, rerun compilation with ./koch temp " & options.command & " ") quit 1 if msg >= fatalMin and msg <= fatalMax: @@ -737,34 +747,39 @@ proc writeContext(lastinfo: TLineInfo) = for i in countup(0, len(msgContext) - 1): if msgContext[i] != lastinfo and msgContext[i] != info: msgWriteln(PosContextFormat % [toMsgFilename(msgContext[i]), - coordToStr(msgContext[i].line), - coordToStr(msgContext[i].col+1), - getMessageStr(errInstantiationFrom, "")]) + coordToStr(msgContext[i].line), + coordToStr(msgContext[i].col+1), + getMessageStr(errInstantiationFrom, "")]) info = msgContext[i] proc ignoreMsgBecauseOfIdeTools(msg: TMsgKind): bool = msg >= errGenerated and gCmd == cmdIdeTools and optIdeDebug notin gGlobalOptions proc rawMessage*(msg: TMsgKind, args: openArray[string]) = - var frmt: string + var + frmt: string + color: ForegroundColor case msg of errMin..errMax: writeContext(unknownLineInfo()) - frmt = RawErrorFormat + frmt = RawError + color = fgRed of warnMin..warnMax: if optWarns notin gOptions: return if msg notin gNotes: return writeContext(unknownLineInfo()) - frmt = RawWarningFormat + frmt = RawWarning inc(gWarnCounter) + color = fgYellow of hintMin..hintMax: if optHints notin gOptions: return if msg notin gNotes: return - frmt = RawHintFormat + frmt = RawHint inc(gHintCounter) - let s = `%`(frmt, `%`(msgKindToString(msg), args)) + color = fgGreen + let s = `%`(msgKindToString(msg), args) if not ignoreMsgBecauseOfIdeTools(msg): - msgWriteln(s) + msgWriteln(s, color, frmt) handleError(msg, doAbort, s) proc rawMessage*(msg: TMsgKind, arg: string) = @@ -785,8 +800,10 @@ proc formatMsg*(info: TLineInfo, msg: TMsgKind, arg: string): string = proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string, eh: TErrorHandling) = - var frmt: string - var ignoreMsg = false + var + frmt: string + ignoreMsg = false + color: ForegroundColor case msg of errMin..errMax: writeContext(info) @@ -795,22 +812,26 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string, # in the same file and line are produced: #ignoreMsg = lastError == info and eh != doAbort lastError = info + color = fgRed of warnMin..warnMax: ignoreMsg = optWarns notin gOptions or msg notin gNotes if not ignoreMsg: writeContext(info) frmt = PosWarningFormat inc(gWarnCounter) + color = fgYellow of hintMin..hintMax: ignoreMsg = optHints notin gOptions or msg notin gNotes frmt = PosHintFormat inc(gHintCounter) + color = fgGreen # NOTE: currently line info line numbers start with 1, # but column numbers start with 0, however most editors expect # first column to be 1, so we need to +1 here - let s = frmt % [toMsgFilename(info), coordToStr(info.line), - coordToStr(info.col+1), getMessageStr(msg, arg)] + let x = frmt % [toMsgFilename(info), coordToStr(info.line), + coordToStr(info.col+1)] + let s = getMessageStr(msg, arg) if not ignoreMsg and not ignoreMsgBecauseOfIdeTools(msg): - msgWriteln(s) + msgWriteln(s, color, x) if optPrintSurroundingSrc and msg in errMin..errMax: info.writeSurroundingSrc handleError(msg, eh, s) diff --git a/compiler/options.nim b/compiler/options.nim index b3060a1807..48e2db0c70 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -54,6 +54,7 @@ type # please make sure we have under 32 options optSkipUserConfigFile, # skip the users's config file optSkipParentConfigFiles, # skip parent dir's config files optNoMain, # do not generate a "main" proc + optUseColors, # use colors for hints, warnings, and errors optThreads, # support for multi-threading optStdout, # output to stdout optThreadAnalysis, # thread analysis pass From ab1b1c9251f6d787cf78eefd2c3d1d391f5287fd Mon Sep 17 00:00:00 2001 From: apense Date: Mon, 8 Jun 2015 16:03:57 -0400 Subject: [PATCH 06/20] Note about deprecation in generics.txt I'm happy to change the wording or anything. It does seem like an important thing to mention, though. It might be better not to even mention typeless procs so as not to encourage bad behavior. --- doc/manual/generics.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/manual/generics.txt b/doc/manual/generics.txt index 2736f88fb8..f82cad5313 100644 --- a/doc/manual/generics.txt +++ b/doc/manual/generics.txt @@ -164,11 +164,13 @@ Alternatively, the ``distinct`` type modifier can be applied to the type class to allow each param matching the type class to bind to a different type. If a proc param doesn't have a type specified, Nim will use the -``distinct auto`` type class (also known as ``any``): +``distinct auto`` type class (also known as ``any``). Note this behavior is +deprecated for procs; templates, however, support them: .. code-block:: nim # allow any combination of param types - proc concat(a, b): string = $a & $b + proc concat(a, b): string = $a & $b # deprecated + proc concat(a, b: any): string = $a & $b # preferred Procs written with the implicitly generic style will often need to refer to the type parameters of the matched generic type. They can be easily accessed using From c4009c61820190c188f6bcf7469754b3c40201e5 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 8 Jun 2015 23:10:50 +0100 Subject: [PATCH 07/20] Fixes #2884 Haven't tested this patch but should work. --- lib/pure/httpclient.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index e083d44eab..6a29137139 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -569,7 +569,7 @@ proc downloadFile*(url: string, outputFilename: string, fileError("Unable to open file") proc generateHeaders(r: Uri, httpMethod: string, - headers: StringTableRef): string = + headers: StringTableRef, body: string): string = # TODO: Use this in the blocking HttpClient once it supports proxies. result = substr(httpMethod, len("http")) # TODO: Proxies @@ -582,6 +582,8 @@ proc generateHeaders(r: Uri, httpMethod: string, add(result, "Host: " & r.hostname & "\c\L") add(result, "Connection: Keep-Alive\c\L") + if body.len > 0 and not headers.hasKey("Content-Length"): + add(result, "Content-Length: " & $body.len & "\c\L") for key, val in headers: add(result, key & ": " & val & "\c\L") @@ -786,7 +788,7 @@ proc request*(client: AsyncHttpClient, url: string, httpMethod: string, if not client.headers.hasKey("user-agent") and client.userAgent != "": client.headers["User-Agent"] = client.userAgent - var headers = generateHeaders(r, $httpMethod, client.headers) + var headers = generateHeaders(r, $httpMethod, client.headers, body) await client.socket.send(headers) if body != "": From 0ee1672d69272aa75cf9be15dede34773a4fa487 Mon Sep 17 00:00:00 2001 From: apense Date: Mon, 8 Jun 2015 19:48:57 -0400 Subject: [PATCH 08/20] Updated whitespace ranges Ranges sourced from _. Wikipedia also uses these ranges on its information page _. 0xfeff isn't included in the list, but it is a no-break space, so I guess it makes sense. 0x200b is actually a format character, but it is a zero-width space. To fit Unicode, both 0x200b and 0xfeff would be removed. --- lib/pure/unicode.nim | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 5fd3c2418e..4446eaa0c9 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -372,11 +372,17 @@ const 0xfe74] # spaceRanges = [ - 0x0009, 0x000a, # tab and newline + 0x0009, 0x000d, # tab and newline 0x0020, 0x0020, # space + 0x0085, 0x0085, # next line 0x00a0, 0x00a0, # - 0x2000, 0x200b, # - + 0x1680, 0x1680, # Ogham space mark + 0x2000, 0x200b, # en dash .. zero-width space + 0x200e, 0x200f, # LTR mark .. RTL mark (pattern whitespace) 0x2028, 0x2029, # - 0x3000, 0x3000, # + 0x202f, 0x202f, # narrow no-break space + 0x205f, 0x205f, # medium mathematical space + 0x3000, 0x3000, # ideographic space 0xfeff, 0xfeff] # toupperRanges = [ From 87429071330cc8516dab2612b45689a510528f6a Mon Sep 17 00:00:00 2001 From: Adam Strzelecki Date: Thu, 4 Jun 2015 00:38:49 +0200 Subject: [PATCH 09/20] db: InstantRow and instantRows It is drop-in replacement for Row and fastRows, however instantRows returns a handle, not seq[string], so no Nim string is created until [] operator is called on the given handle. Also there is a len() proc returning number of columns in the handle. In some situations, when we iterate through many rows, but later we just read few columns this solution will be quicker than converting all column to Nim seq[string] on each iteration. --- lib/impure/db_mysql.nim | 27 +++++++++++++++++++++++++++ lib/impure/db_postgres.nim | 21 +++++++++++++++++++++ lib/impure/db_sqlite.nim | 24 +++++++++++++++++++++++- 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index 619c2a6562..7c2901efde 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -16,6 +16,9 @@ type DbConn* = PMySQL ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be ## transformed always to the empty string. + InstantRow* = tuple[row: cstringArray, len: int] ## a handle that can be + ## used to get a row's + ## column text on demand EDb* = object of IOError ## exception that is raised if a database error occurs SqlQuery* = distinct string ## an SQL query string @@ -127,6 +130,30 @@ iterator fastRows*(db: DbConn, query: SqlQuery, yield result properFreeResult(sqlres, row) +iterator instantRows*(db: DbConn, query: SqlQuery, + args: varargs[string, `$`]): InstantRow + {.tags: [FReadDb].} = + ## same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within interator body. + rawExec(db, query, args) + var sqlres = mysql.useResult(db) + if sqlres != nil: + let L = int(mysql.numFields(sqlres)) + var row: cstringArray + while true: + row = mysql.fetchRow(sqlres) + if row == nil: break + yield (row: row, len: L) + properFreeResult(sqlres, row) + +proc `[]`*(row: InstantRow, col: int): string {.inline.} = + ## returns text for given column of the row + $row.row[col] + +proc len*(row: InstantRow): int {.inline.} = + ## returns number of columns in the row + row.len + proc getRow*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {.tags: [FReadDB].} = ## retrieves a single row. If the query doesn't return any rows, this proc diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim index 774cb15106..c88e660f4b 100644 --- a/lib/impure/db_postgres.nim +++ b/lib/impure/db_postgres.nim @@ -16,6 +16,9 @@ type DbConn* = PPGconn ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be ## transformed always to the empty string. + InstantRow* = tuple[res: PPGresult, line: int32] ## a handle that can be + ## used to get a row's + ## column text on demand EDb* = object of IOError ## exception that is raised if a database error occurs SqlQuery* = distinct string ## an SQL query string @@ -159,6 +162,24 @@ iterator fastRows*(db: DbConn, stmtName: SqlPrepared, yield result pqClear(res) +iterator instantRows*(db: DbConn, query: SqlQuery, + args: varargs[string, `$`]): InstantRow + {.tags: [FReadDb].} = + ## same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within interator body. + var res = setupQuery(db, query, args) + for i in 0..pqNtuples(res)-1: + yield (res: res, line: i) + pqClear(res) + +proc `[]`*(row: InstantRow, col: int32): string {.inline.} = + ## returns text for given column of the row + $pqgetvalue(row.res, row.line, col) + +proc len*(row: InstantRow): int32 {.inline.} = + ## returns number of columns in the row + pqNfields(row.res) + proc getRow*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {.tags: [FReadDB].} = ## retrieves a single row. If the query doesn't return any rows, this proc diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 47e7c1900d..1a037beccd 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -16,6 +16,8 @@ type DbConn* = PSqlite3 ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be ## transformed always to the empty string. + InstantRow* = Pstmt ## a handle that can be used to get a row's column + ## text on demand EDb* = object of IOError ## exception that is raised if a database error occurs SqlQuery* = distinct string ## an SQL query string @@ -109,6 +111,24 @@ iterator fastRows*(db: DbConn, query: SqlQuery, yield result if finalize(stmt) != SQLITE_OK: dbError(db) +iterator instantRows*(db: DbConn, query: SqlQuery, + args: varargs[string, `$`]): InstantRow + {.tags: [FReadDb].} = + ## same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within interator body. + var stmt = setupQuery(db, query, args) + while step(stmt) == SQLITE_ROW: + yield stmt + if finalize(stmt) != SQLITE_OK: dbError(db) + +proc `[]`*(row: InstantRow, col: int32): string {.inline.} = + ## returns text for given column of the row + $column_text(row, col) + +proc len*(row: InstantRow): int32 {.inline.} = + ## returns number of columns in the row + column_count(row) + proc getRow*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {.tags: [FReadDb].} = ## retrieves a single row. If the query doesn't return any rows, this proc @@ -216,5 +236,7 @@ when not defined(testing) and isMainModule: #db.query("insert into tbl1 values('goodbye', 20)") for r in db.rows(sql"select * from tbl1", []): echo(r[0], r[1]) - + for r in db.instantRows(sql"select * from tbl1", []): + echo(r[0], r[1]) + db_sqlite.close(db) From 3d266433a75449d76b589aba6edaf488d5a8c921 Mon Sep 17 00:00:00 2001 From: Adam Strzelecki Date: Tue, 9 Jun 2015 22:16:43 +0200 Subject: [PATCH 10/20] securehash: Keep module name consistent 1. All other modules don't use underscore, why this should be different? 2. Wrap some imports to 80 column --- compiler/ast.nim | 2 +- compiler/cgen.nim | 2 +- compiler/extccomp.nim | 3 ++- compiler/jsgen.nim | 2 +- compiler/modules.nim | 2 +- compiler/rodread.nim | 2 +- compiler/rodwrite.nim | 3 ++- compiler/{secure_hash.nim => securehash.nim} | 0 8 files changed, 9 insertions(+), 7 deletions(-) rename compiler/{secure_hash.nim => securehash.nim} (100%) diff --git a/compiler/ast.nim b/compiler/ast.nim index dc190fd7f4..36d29e06ae 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -10,7 +10,7 @@ # abstract syntax tree + symbol table import - msgs, hashes, nversion, options, strutils, secure_hash, ropes, idents, lists, + msgs, hashes, nversion, options, strutils, securehash, ropes, idents, lists, intsets, idgen type diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 03aa0b4d4a..390150cf70 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -11,7 +11,7 @@ import ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets, - nversion, nimsets, msgs, secure_hash, bitsets, idents, lists, types, + nversion, nimsets, msgs, securehash, bitsets, idents, lists, types, ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, semparallel diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 2ddb98abab..38427b367c 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -13,7 +13,8 @@ # nim files. import - lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, secure_hash + lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, + securehash type TSystemCC* = enum diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 346a52cfca..1f82306d26 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -31,7 +31,7 @@ implements the required case distinction. import ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options, - nversion, nimsets, msgs, secure_hash, bitsets, idents, lists, types, os, + nversion, nimsets, msgs, securehash, bitsets, idents, lists, types, os, times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils, intsets, cgmeth, lowerings diff --git a/compiler/modules.nim b/compiler/modules.nim index 0573b91f8a..6cb14c0917 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -10,7 +10,7 @@ ## implements the module handling import - ast, astalgo, magicsys, secure_hash, rodread, msgs, cgendata, sigmatch, options, + ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options, idents, os, lexer, idgen, passes, syntaxes, llstream type diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 27cb96ca14..dad7d111e5 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -90,7 +90,7 @@ import os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms, - ropes, idents, secure_hash, idgen, types, rodutils, memfiles + ropes, idents, securehash, idgen, types, rodutils, memfiles type TReasonForRecompile* = enum ## all the reasons that can trigger recompilation diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index 3454b9ade6..7373875978 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -13,7 +13,8 @@ import intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform, - condsyms, ropes, idents, secure_hash, rodread, passes, importer, idgen, rodutils + condsyms, ropes, idents, securehash, rodread, passes, importer, idgen, + rodutils # implementation diff --git a/compiler/secure_hash.nim b/compiler/securehash.nim similarity index 100% rename from compiler/secure_hash.nim rename to compiler/securehash.nim From eccc2f87e9dfa7e849e027bef857a390de3e9bae Mon Sep 17 00:00:00 2001 From: Adam Strzelecki Date: Tue, 9 Jun 2015 22:15:08 +0200 Subject: [PATCH 11/20] securehash: remove unused emptySecureHash --- compiler/securehash.nim | 8 -------- 1 file changed, 8 deletions(-) diff --git a/compiler/securehash.nim b/compiler/securehash.nim index 3383f23a23..12950f1371 100644 --- a/compiler/securehash.nim +++ b/compiler/securehash.nim @@ -16,14 +16,6 @@ type SHA1Digest = array[0 .. sha_digest_size-1, uint8] SecureHash* = distinct SHA1Digest -const emptySecureHash = SecureHash([ - 0u8, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, -]) - proc sha1(src: string) : SHA1Digest proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) From 54c863d6ae36faa9ba14fa2a5c34ea38d4be6efe Mon Sep 17 00:00:00 2001 From: Adam Strzelecki Date: Tue, 9 Jun 2015 22:15:51 +0200 Subject: [PATCH 12/20] securehash: sha_digest_size is not following style https://github.com/Araq/Nim/wiki/Style-Guide-for-Nim-Code --- compiler/securehash.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/securehash.nim b/compiler/securehash.nim index 12950f1371..ad15c2e8f0 100644 --- a/compiler/securehash.nim +++ b/compiler/securehash.nim @@ -10,10 +10,10 @@ import strutils, unsigned -const sha_digest_size = 20 +const SHA1DigestSize = 20 type - SHA1Digest = array[0 .. sha_digest_size-1, uint8] + SHA1Digest = array[0 .. SHA1DigestSize-1, uint8] SecureHash* = distinct SHA1Digest proc sha1(src: string) : SHA1Digest @@ -26,7 +26,7 @@ proc `$`*(self: SecureHash): string = result.add(toHex(int(v), 2)) proc parseSecureHash*(hash: string): SecureHash = - for i in 0.. Date: Tue, 9 Jun 2015 16:21:33 -0400 Subject: [PATCH 13/20] Removed unnecessary template `-|` only used in the compiler for negative indexing --- lib/system.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 90587f3063..949443dd9d 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2859,9 +2859,6 @@ proc `/`*(x, y: int): float {.inline, noSideEffect.} = ## integer division that results in a float. result = toFloat(x) / toFloat(y) -template `-|`*(b, s: expr): expr = - (if b >= 0: b else: s.len + b) - template spliceImpl(s, a, L, b: expr): stmt {.immediate.} = # make room for additional elements or cut: var slen = s.len From d4d5f95808f0db59dd46a6793fd35d8a32f693fb Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez Date: Tue, 9 Jun 2015 15:21:33 -0500 Subject: [PATCH 14/20] Remove unnecessary import --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index ea69f6c25e..186a3884df 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -13,7 +13,7 @@ # nim files. import - lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, crc, terminal + lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, crc type TSystemCC* = enum From 867b38496b234df22ab34ef15344f848278b096d Mon Sep 17 00:00:00 2001 From: apense Date: Tue, 9 Jun 2015 16:24:13 -0400 Subject: [PATCH 15/20] Moved negative indexing operator here Only used here (not in standard library) --- compiler/ast.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compiler/ast.nim b/compiler/ast.nim index dc190fd7f4..f48003797a 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -948,6 +948,9 @@ proc add*(father, son: PNode) = proc `[]`*(n: PNode, i: int): PNode {.inline.} = result = n.sons[i] +template `-|`*(b, s: expr): expr = + (if b >= 0: b else: s.len + b) + # son access operators with support for negative indices template `{}`*(n: PNode, i: int): expr = n[i -| n] template `{}=`*(n: PNode, i: int, s: PNode): stmt = From 4e14c0ad9fc790a8a8d22ddcd2a28ed90322c4e7 Mon Sep 17 00:00:00 2001 From: Adam Strzelecki Date: Tue, 9 Jun 2015 22:55:29 +0200 Subject: [PATCH 16/20] securehash: SHA1 -> Sha1 according to style guide --- compiler/securehash.nim | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/compiler/securehash.nim b/compiler/securehash.nim index ad15c2e8f0..8ac6acb0e9 100644 --- a/compiler/securehash.nim +++ b/compiler/securehash.nim @@ -10,28 +10,28 @@ import strutils, unsigned -const SHA1DigestSize = 20 +const Sha1DigestSize = 20 type - SHA1Digest = array[0 .. SHA1DigestSize-1, uint8] - SecureHash* = distinct SHA1Digest + Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] + SecureHash* = distinct Sha1Digest -proc sha1(src: string) : SHA1Digest +proc sha1(src: string) : Sha1Digest proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) proc `$`*(self: SecureHash): string = result = "" - for v in SHA1Digest(self): + for v in Sha1Digest(self): result.add(toHex(int(v), 2)) proc parseSecureHash*(hash: string): SecureHash = - for i in 0.. Date: Tue, 9 Jun 2015 22:25:31 -0300 Subject: [PATCH 17/20] Allow to set position at end of stream --- lib/pure/streams.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index e6004b5d0f..50b5c219a2 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -279,7 +279,7 @@ proc ssAtEnd(s: Stream): bool = proc ssSetPosition(s: Stream, pos: int) = var s = StringStream(s) - s.pos = clamp(pos, 0, s.data.high) + s.pos = clamp(pos, 0, s.data.len) proc ssGetPosition(s: Stream): int = var s = StringStream(s) From 1b98d3257988b8878bef5fe637410ea125cfeb44 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 7 Jun 2015 18:15:47 +0200 Subject: [PATCH 18/20] no private deprecated symbols --- bootstrap.sh | 0 lib/pure/pegs.nim | 1 - 2 files changed, 1 deletion(-) mode change 100755 => 100644 bootstrap.sh diff --git a/bootstrap.sh b/bootstrap.sh old mode 100755 new mode 100644 diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index ce837d9d18..e165d7dca1 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1060,7 +1060,6 @@ type lineStart: int ## index of last line start in buffer colOffset: int ## column to add filename: string -{.deprecated: [TTokKind: TokKind, TToken: Token, TModifier: Modifier].} const tokKindToStr: array[TokKind, string] = [ From d4610d7716c4c82d738548d7169aa150b803ef1d Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 10 Jun 2015 11:32:10 +0200 Subject: [PATCH 19/20] terminal doesn't use enormous windows module anymore --- lib/pure/terminal.nim | 103 +++++++++++++++++++++++++++++++++--------- 1 file changed, 81 insertions(+), 22 deletions(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 1e9c40f065..15e2eefec7 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -17,28 +17,87 @@ import macros when defined(windows): - import windows, os + import winlean, os + + const + DUPLICATE_SAME_ACCESS = 2 + FOREGROUND_BLUE = 1 + FOREGROUND_GREEN = 2 + FOREGROUND_RED = 4 + FOREGROUND_INTENSITY = 8 + BACKGROUND_BLUE = 16 + BACKGROUND_GREEN = 32 + BACKGROUND_RED = 64 + BACKGROUND_INTENSITY = 128 + + type + SHORT = int16 + COORD = object + X: SHORT + Y: SHORT + + SMALL_RECT = object + Left: SHORT + Top: SHORT + Right: SHORT + Bottom: SHORT + + CONSOLE_SCREEN_BUFFER_INFO = object + dwSize: COORD + dwCursorPosition: COORD + wAttributes: int16 + srWindow: SMALL_RECT + dwMaximumWindowSize: COORD + + proc duplicateHandle(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, + hTargetProcessHandle: HANDLE, lpTargetHandle: ptr HANDLE, + dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, + dwOptions: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", + importc: "DuplicateHandle".} + proc getCurrentProcess(): HANDLE{.stdcall, dynlib: "kernel32", + importc: "GetCurrentProcess".} + proc getConsoleScreenBufferInfo(hConsoleOutput: HANDLE, + lpConsoleScreenBufferInfo: ptr CONSOLE_SCREEN_BUFFER_INFO): WINBOOL{.stdcall, + dynlib: "kernel32", importc: "GetConsoleScreenBufferInfo".} + + proc setConsoleCursorPosition(hConsoleOutput: HANDLE, + dwCursorPosition: COORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "SetConsoleCursorPosition".} + + proc fillConsoleOutputCharacter(hConsoleOutput: Handle, cCharacter: char, + nLength: DWORD, dwWriteCoord: Coord, + lpNumberOfCharsWritten: ptr DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "FillConsoleOutputCharacterA".} + + proc fillConsoleOutputAttribute(hConsoleOutput: HANDLE, wAttribute: int16, + nLength: DWORD, dwWriteCoord: COORD, + lpNumberOfAttrsWritten: ptr DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "FillConsoleOutputAttribute".} + + proc setConsoleTextAttribute(hConsoleOutput: HANDLE, + wAttributes: int16): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "SetConsoleTextAttribute".} var conHandle: Handle # = createFile("CONOUT$", GENERIC_WRITE, 0, nil, OPEN_ALWAYS, 0, 0) block: - var hTemp = GetStdHandle(STD_OUTPUT_HANDLE) - if DuplicateHandle(GetCurrentProcess(), hTemp, GetCurrentProcess(), + var hTemp = getStdHandle(STD_OUTPUT_HANDLE) + if duplicateHandle(getCurrentProcess(), hTemp, getCurrentProcess(), addr(conHandle), 0, 1, DUPLICATE_SAME_ACCESS) == 0: raiseOSError(osLastError()) proc getCursorPos(): tuple [x,y: int] = var c: CONSOLESCREENBUFFERINFO - if GetConsoleScreenBufferInfo(conHandle, addr(c)) == 0: + if getConsoleScreenBufferInfo(conHandle, addr(c)) == 0: raiseOSError(osLastError()) return (int(c.dwCursorPosition.X), int(c.dwCursorPosition.Y)) proc getAttributes(): int16 = var c: CONSOLESCREENBUFFERINFO # workaround Windows bugs: try several times - if GetConsoleScreenBufferInfo(conHandle, addr(c)) != 0: + if getConsoleScreenBufferInfo(conHandle, addr(c)) != 0: return c.wAttributes return 0x70'i16 # ERROR: return white background, black text @@ -67,7 +126,7 @@ proc setCursorPos*(x, y: int) = var c: COORD c.X = int16(x) c.Y = int16(y) - if SetConsoleCursorPosition(conHandle, c) == 0: raiseOSError(osLastError()) + if setConsoleCursorPosition(conHandle, c) == 0: raiseOSError(osLastError()) else: stdout.write("\e[" & $y & ';' & $x & 'f') @@ -77,11 +136,11 @@ proc setCursorXPos*(x: int) = when defined(windows): var scrbuf: CONSOLESCREENBUFFERINFO var hStdout = conHandle - if GetConsoleScreenBufferInfo(hStdout, addr(scrbuf)) == 0: + if getConsoleScreenBufferInfo(hStdout, addr(scrbuf)) == 0: raiseOSError(osLastError()) var origin = scrbuf.dwCursorPosition origin.X = int16(x) - if SetConsoleCursorPosition(conHandle, origin) == 0: + if setConsoleCursorPosition(conHandle, origin) == 0: raiseOSError(osLastError()) else: stdout.write("\e[" & $x & 'G') @@ -93,11 +152,11 @@ when defined(windows): when defined(windows): var scrbuf: CONSOLESCREENBUFFERINFO var hStdout = conHandle - if GetConsoleScreenBufferInfo(hStdout, addr(scrbuf)) == 0: + if getConsoleScreenBufferInfo(hStdout, addr(scrbuf)) == 0: raiseOSError(osLastError()) var origin = scrbuf.dwCursorPosition origin.Y = int16(y) - if SetConsoleCursorPosition(conHandle, origin) == 0: + if setConsoleCursorPosition(conHandle, origin) == 0: raiseOSError(osLastError()) else: discard @@ -175,18 +234,18 @@ proc eraseLine* = var scrbuf: CONSOLESCREENBUFFERINFO var numwrote: DWORD var hStdout = conHandle - if GetConsoleScreenBufferInfo(hStdout, addr(scrbuf)) == 0: + if getConsoleScreenBufferInfo(hStdout, addr(scrbuf)) == 0: raiseOSError(osLastError()) var origin = scrbuf.dwCursorPosition origin.X = 0'i16 - if SetConsoleCursorPosition(conHandle, origin) == 0: + if setConsoleCursorPosition(conHandle, origin) == 0: raiseOSError(osLastError()) var ht = scrbuf.dwSize.Y - origin.Y var wt = scrbuf.dwSize.X - origin.X - if FillConsoleOutputCharacter(hStdout,' ', ht*wt, + if fillConsoleOutputCharacter(hStdout,' ', ht*wt, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) - if FillConsoleOutputAttribute(hStdout, scrbuf.wAttributes, ht * wt, + if fillConsoleOutputAttribute(hStdout, scrbuf.wAttributes, ht * wt, scrbuf.dwCursorPosition, addr(numwrote)) == 0: raiseOSError(osLastError()) else: @@ -201,14 +260,14 @@ proc eraseScreen* = var origin: COORD # is inititalized to 0, 0 var hStdout = conHandle - if GetConsoleScreenBufferInfo(hStdout, addr(scrbuf)) == 0: + if getConsoleScreenBufferInfo(hStdout, addr(scrbuf)) == 0: raiseOSError(osLastError()) let numChars = int32(scrbuf.dwSize.X)*int32(scrbuf.dwSize.Y) - if FillConsoleOutputCharacter(hStdout, ' ', numChars, + if fillConsoleOutputCharacter(hStdout, ' ', numChars, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) - if FillConsoleOutputAttribute(hStdout, scrbuf.wAttributes, numChars, + if fillConsoleOutputAttribute(hStdout, scrbuf.wAttributes, numChars, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) setCursorXPos(0) @@ -219,7 +278,7 @@ proc resetAttributes* {.noconv.} = ## resets all attributes; it is advisable to register this as a quit proc ## with ``system.addQuitProc(resetAttributes)``. when defined(windows): - discard SetConsoleTextAttribute(conHandle, oldAttr) + discard setConsoleTextAttribute(conHandle, oldAttr) else: stdout.write("\e[0m") @@ -249,7 +308,7 @@ proc setStyle*(style: set[Style]) = if styleBlink in style: a = a or int16(BACKGROUND_INTENSITY) if styleReverse in style: a = a or 0x4000'i16 # COMMON_LVB_REVERSE_VIDEO if styleUnderscore in style: a = a or 0x8000'i16 # COMMON_LVB_UNDERSCORE - discard SetConsoleTextAttribute(conHandle, a) + discard setConsoleTextAttribute(conHandle, a) else: for s in items(style): stdout.write("\e[" & $ord(s) & 'm') @@ -260,7 +319,7 @@ proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = var old = getAttributes() setStyle(style) stdout.write(txt) - discard SetConsoleTextAttribute(conHandle, old) + discard setConsoleTextAttribute(conHandle, old) else: setStyle(style) stdout.write(txt) @@ -309,7 +368,7 @@ proc setForegroundColor*(fg: ForegroundColor, bright=false) = (FOREGROUND_RED or FOREGROUND_BLUE), (FOREGROUND_BLUE or FOREGROUND_GREEN), (FOREGROUND_BLUE or FOREGROUND_GREEN or FOREGROUND_RED)] - discard SetConsoleTextAttribute(conHandle, toU16(old or lookup[fg])) + discard setConsoleTextAttribute(conHandle, toU16(old or lookup[fg])) else: gFG = ord(fg) if bright: inc(gFG, 60) @@ -330,7 +389,7 @@ proc setBackgroundColor*(bg: BackgroundColor, bright=false) = (BACKGROUND_RED or BACKGROUND_BLUE), (BACKGROUND_BLUE or BACKGROUND_GREEN), (BACKGROUND_BLUE or BACKGROUND_GREEN or BACKGROUND_RED)] - discard SetConsoleTextAttribute(conHandle, toU16(old or lookup[bg])) + discard setConsoleTextAttribute(conHandle, toU16(old or lookup[bg])) else: gBG = ord(bg) if bright: inc(gBG, 60) From 6c256ddcaef0c92b94d1a3e61f4e56cf7d017b50 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 10 Jun 2015 11:50:08 +0200 Subject: [PATCH 20/20] cause I have no life --- bootstrap.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bootstrap.sh diff --git a/bootstrap.sh b/bootstrap.sh old mode 100644 new mode 100755