CRC -> Hash

This commit is contained in:
Flaviu Tamas
2015-06-29 18:23:23 -04:00
parent 39f549d06a
commit 62b55592e8
7 changed files with 69 additions and 69 deletions

View File

@@ -429,7 +429,7 @@ proc resetCompilationLists* =
initLinkedList(toCompile)
## XXX: we must associate these with their originating module
# when the module is loaded/unloaded it adds/removes its items
# That's because we still need to CRC check the external files
# That's because we still need to hash check the external files
# Maybe we can do that in checkDep on the other hand?
initLinkedList(externalToCompile)
initLinkedList(toLink)
@@ -585,18 +585,18 @@ proc externalFileChanged(filename: string): bool =
if gCmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM}:
return false
var crcFile = toGeneratedFile(filename.withPackageName, "sha1")
var currentCrc = footprint(filename)
var hashFile = toGeneratedFile(filename.withPackageName, "sha1")
var currentHash = footprint(filename)
var f: File
if open(f, crcFile, fmRead):
let oldCrc = parseSecureHash(f.readLine())
if open(f, hashFile, fmRead):
let oldHash = parseSecureHash(f.readLine())
close(f)
result = oldCrc != currentCrc
result = oldHash != currentHash
else:
result = true
if result:
if open(f, crcFile, fmWrite):
f.writeLine($currentCrc)
if open(f, hashFile, fmWrite):
f.writeLine($currentHash)
close(f)
proc addExternalFileToCompile*(filename: string) =

View File

@@ -70,7 +70,7 @@ proc commandCompileToC =
lastCaasCmd = curCaasCmd
resetCgenModules()
for i in 0 .. <gMemCacheData.len:
gMemCacheData[i].crcStatus = crcCached
gMemCacheData[i].hashStatus = hashCached
gMemCacheData[i].needsRecompile = Maybe
# XXX: clean these global vars

View File

@@ -15,14 +15,14 @@ import
type
TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled
TCrcStatus* = enum crcNotTaken, crcCached, crcHasChanged, crcNotChanged
THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged
TModuleInMemory* = object
compiledAt*: float
crc*: SecureHash
hash*: SecureHash
deps*: seq[int32] ## XXX: slurped files are currently not tracked
needsRecompile*: TNeedRecompile
crcStatus*: TCrcStatus
hashStatus*: THashStatus
var
gCompiledModules: seq[PSym] = @[]
@@ -34,36 +34,36 @@ proc getModule(fileIdx: int32): PSym =
if fileIdx >= 0 and fileIdx < gCompiledModules.len:
result = gCompiledModules[fileIdx]
template crc(x: PSym): expr =
gMemCacheData[x.position].crc
template hash(x: PSym): expr =
gMemCacheData[x.position].hash
proc crcChanged(fileIdx: int32): bool =
proc hashChanged(fileIdx: int32): bool =
internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
template updateStatus =
gMemCacheData[fileIdx].crcStatus = if result: crcHasChanged
else: crcNotChanged
# echo "TESTING CRC: ", fileIdx.toFilename, " ", result
gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged
else: hashNotChanged
# echo "TESTING Hash: ", fileIdx.toFilename, " ", result
case gMemCacheData[fileIdx].crcStatus:
of crcHasChanged:
case gMemCacheData[fileIdx].hashStatus:
of hashHasChanged:
result = true
of crcNotChanged:
of hashNotChanged:
result = false
of crcCached:
let newCrc = secureHashFile(fileIdx.toFullPath)
result = newCrc != gMemCacheData[fileIdx].crc
gMemCacheData[fileIdx].crc = newCrc
of hashCached:
let newHash = secureHashFile(fileIdx.toFullPath)
result = newHash != gMemCacheData[fileIdx].hash
gMemCacheData[fileIdx].hash = newHash
updateStatus()
of crcNotTaken:
gMemCacheData[fileIdx].crc = secureHashFile(fileIdx.toFullPath)
of hashNotTaken:
gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
result = true
updateStatus()
proc doCRC(fileIdx: int32) =
if gMemCacheData[fileIdx].crcStatus == crcNotTaken:
# echo "FIRST CRC: ", fileIdx.ToFilename
gMemCacheData[fileIdx].crc = secureHashFile(fileIdx.toFullPath)
proc doHash(fileIdx: int32) =
if gMemCacheData[fileIdx].hashStatus == hashNotTaken:
# echo "FIRST Hash: ", fileIdx.ToFilename
gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
proc addDep(x: PSym, dep: int32) =
growCache gMemCacheData, dep
@@ -94,7 +94,7 @@ proc checkDepMem(fileIdx: int32): TNeedRecompile =
return gMemCacheData[fileIdx].needsRecompile
if optForceFullMake in gGlobalOptions or
crcChanged(fileIdx):
hashChanged(fileIdx):
markDirty
if gMemCacheData[fileIdx].deps != nil:
@@ -155,7 +155,7 @@ proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym =
if optCaasEnabled in gGlobalOptions:
gMemCacheData[fileIdx].compiledAt = gLastCmdTime
gMemCacheData[fileIdx].needsRecompile = Recompiled
doCRC fileIdx
doHash fileIdx
else:
if checkDepMem(fileIdx) == Yes:
result = compileModule(fileIdx, flags)
@@ -174,7 +174,7 @@ proc includeModule*(s: PSym, fileIdx: int32): PNode {.procvar.} =
if optCaasEnabled in gGlobalOptions:
growCache gMemCacheData, fileIdx
addDep(s, fileIdx)
doCRC(fileIdx)
doHash(fileIdx)
proc `==^`(a, b: string): bool =
try:

View File

@@ -19,8 +19,8 @@
# NIM:$fileversion\n
# - the module's id (even if the module changed, its ID will not!):
# ID:Ax3\n
# - CRC value of this module:
# CRC:CRC-val\n
# - HASH value of this module:
# HASH:HASH-val\n
# - a section containing the compiler options and defines this
# module has been compiled with:
# OPTIONS:options\n
@@ -33,7 +33,7 @@
# )
# - an include file dependency section:
# INCLUDES(
# <fileidx> <CRC of myfile.inc>\n # fileidx is the LINE in the file section!
# <fileidx> <Hash of myfile.inc>\n # fileidx is the LINE in the file section!
# )
# - a module dependency section:
# DEPS: <fileidx> <fileidx>\n
@@ -98,7 +98,7 @@ type
rrNone, # no need to recompile
rrRodDoesNotExist, # rod file does not exist
rrRodInvalid, # rod file is invalid
rrCrcChange, # file has been edited since last recompilation
rrHashChange, # file has been edited since last recompilation
rrDefines, # defines have changed
rrOptions, # options have changed
rrInclDeps, # an include has changed
@@ -538,20 +538,20 @@ proc cmdChangeTriggersRecompilation(old, new: TCommands): bool =
# else: trigger recompilation:
result = true
proc processRodFile(r: PRodReader, crc: SecureHash) =
proc processRodFile(r: PRodReader, hash: SecureHash) =
var
w: string
d: int
var inclCrc: SecureHash
var inclHash: SecureHash
while r.s[r.pos] != '\0':
var section = rdWord(r)
if r.reason != rrNone:
break # no need to process this file further
case section
of "CRC":
of "HASH":
inc(r.pos) # skip ':'
if crc != parseSecureHash(decodeStr(r.s, r.pos)):
r.reason = rrCrcChange
if hash != parseSecureHash(decodeStr(r.s, r.pos)):
r.reason = rrHashChange
of "ID":
inc(r.pos) # skip ':'
r.moduleID = decodeVInt(r.s, r.pos)
@@ -598,9 +598,9 @@ proc processRodFile(r: PRodReader, crc: SecureHash) =
while r.s[r.pos] != ')':
w = r.files[decodeVInt(r.s, r.pos)].toFullPath
inc(r.pos) # skip ' '
inclCrc = parseSecureHash(decodeStr(r.s, r.pos))
inclHash = parseSecureHash(decodeStr(r.s, r.pos))
if r.reason == rrNone:
if not existsFile(w) or (inclCrc != secureHashFile(w)):
if not existsFile(w) or (inclHash != secureHashFile(w)):
r.reason = rrInclDeps
if r.s[r.pos] == '\x0A':
inc(r.pos)
@@ -651,7 +651,7 @@ proc startsWith(buf: cstring, token: string, pos = 0): bool =
while s < token.len and buf[pos+s] == token[s]: inc s
result = s == token.len
proc newRodReader(modfilename: string, crc: SecureHash,
proc newRodReader(modfilename: string, hash: SecureHash,
readerIndex: int): PRodReader =
new(result)
try:
@@ -703,8 +703,8 @@ type
filename*: string
reason*: TReasonForRecompile
rd*: PRodReader
crc*: SecureHash
crcDone*: bool
hash*: SecureHash
hashDone*: bool
TFileModuleMap = seq[TFileModuleRec]
@@ -796,14 +796,14 @@ proc loadMethods(r: PRodReader) =
r.methods.add(rrGetSym(r, d, unknownLineInfo()))
if r.s[r.pos] == ' ': inc(r.pos)
proc getCRC*(fileIdx: int32): SecureHash =
proc getHash*(fileIdx: int32): SecureHash =
internalAssert fileIdx >= 0 and fileIdx < gMods.len
if gMods[fileIdx].crcDone:
return gMods[fileIdx].crc
if gMods[fileIdx].hashDone:
return gMods[fileIdx].hash
result = secureHashFile(fileIdx.toFilename)
gMods[fileIdx].crc = result
gMods[fileIdx].hash = result
template growCache*(cache, pos) =
if cache.len <= pos: cache.setLen(pos+1)
@@ -815,16 +815,16 @@ proc checkDep(fileIdx: int32): TReasonForRecompile =
# reason has already been computed for this module:
return gMods[fileIdx].reason
let filename = fileIdx.toFilename
var crc = getCRC(fileIdx)
var hash = getHash(fileIdx)
gMods[fileIdx].reason = rrNone # we need to set it here to avoid cycles
result = rrNone
var r: PRodReader = nil
var rodfile = toGeneratedFile(filename.withPackageName, RodExt)
r = newRodReader(rodfile, crc, fileIdx)
r = newRodReader(rodfile, hash, fileIdx)
if r == nil:
result = (if existsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist)
else:
processRodFile(r, crc)
processRodFile(r, hash)
result = r.reason
if result == rrNone:
# check modules it depends on
@@ -1029,9 +1029,9 @@ proc viewFile(rodfile: string) =
while r.s[r.pos] != '\0':
let section = rdWord(r)
case section
of "CRC":
of "HASH":
inc(r.pos) # skip ':'
outf.writeLine("CRC:", $decodeVInt(r.s, r.pos))
outf.writeLine("HASH:", $decodeVInt(r.s, r.pos))
of "ID":
inc(r.pos) # skip ':'
r.moduleID = decodeVInt(r.s, r.pos)
@@ -1084,11 +1084,11 @@ proc viewFile(rodfile: string) =
while r.s[r.pos] != ')':
let w = r.files[decodeVInt(r.s, r.pos)]
inc(r.pos) # skip ' '
let inclCrc = decodeVInt(r.s, r.pos)
let inclHash = decodeVInt(r.s, r.pos)
if r.s[r.pos] == '\x0A':
inc(r.pos)
inc(r.line)
outf.write(w, " ", inclCrc, "\n")
outf.write(w, " ", inclHash, "\n")
if r.s[r.pos] == ')': inc(r.pos)
outf.write(")\n")
of "DEPS":

View File

@@ -21,7 +21,7 @@ import
type
TRodWriter = object of TPassContext
module: PSym
crc: SecureHash
hash: SecureHash
options: TOptions
defines: string
inclDeps: string
@@ -39,7 +39,7 @@ type
PRodWriter = ref TRodWriter
proc newRodWriter(crc: SecureHash, module: PSym): PRodWriter
proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter
proc addModDep(w: PRodWriter, dep: string)
proc addInclDep(w: PRodWriter, dep: string)
proc addInterfaceSym(w: PRodWriter, s: PSym)
@@ -63,7 +63,7 @@ proc fileIdx(w: PRodWriter, filename: string): int =
template filename*(w: PRodWriter): string =
w.module.filename
proc newRodWriter(crc: SecureHash, module: PSym): PRodWriter =
proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter =
new(result)
result.sstack = @[]
result.tstack = @[]
@@ -71,7 +71,7 @@ proc newRodWriter(crc: SecureHash, module: PSym): PRodWriter =
initIiTable(result.imports.tab)
result.index.r = ""
result.imports.r = ""
result.crc = crc
result.hash = hash
result.module = module
result.defines = getDefines()
result.options = options.gOptions
@@ -440,9 +440,9 @@ proc writeRod(w: PRodWriter) =
f.write(orig)
f.write(rodNL)
var crc = "CRC:"
encodeStr($w.crc, crc)
f.write(crc)
var hash = "HASH:"
encodeStr($w.hash, hash)
f.write(hash)
f.write(rodNL)
var options = "OPTIONS:"
@@ -575,7 +575,7 @@ proc process(c: PPassContext, n: PNode): PNode =
proc myOpen(module: PSym): PPassContext =
if module.id < 0: internalError("rodwrite: module ID not set")
var w = newRodWriter(module.fileIdx.getCRC, module)
var w = newRodWriter(module.fileIdx.getHash, module)
rawAddInterfaceSym(w, module)
result = w

View File

@@ -18,7 +18,7 @@ when useCaas:
# We cache modules and the dependency graph. However, we don't check for
# file changes but expect the client to tell us about them, otherwise the
# repeated CRC calculations may turn out to be too slow.
# repeated hash calculations may turn out to be too slow.
var
curCaasCmd* = ""

View File

@@ -318,7 +318,7 @@ with the project:
.. code-block:: Nim
{.compile: "myfile.cpp".}
**Note**: Nim computes a CRC checksum and only recompiles the file if it
**Note**: Nim computes a SHA1 checksum and only recompiles the file if it
has changed. You can use the ``-f`` command line option to force recompilation
of the file.