C codegen: first version of signature hashing for better incremental builds

This commit is contained in:
Araq
2016-11-03 12:02:09 +01:00
parent c66580911e
commit 3b43cff0f8
9 changed files with 226 additions and 56 deletions

View File

@@ -1953,7 +1953,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
of skMethod:
if {sfDispatcher, sfForward} * sym.flags != {}:
# we cannot produce code for the dispatcher yet:
fillProcLoc(sym)
fillProcLoc(p.module, sym)
genProcPrototype(p.module, sym)
else:
genProc(p.module, sym)

View File

@@ -970,7 +970,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
if r == nil:
# if no name has already been given,
# it doesn't matter much:
r = mangleName(sym)
r = mangleName(p.module, sym)
sym.loc.r = r # but be consequent!
res.add($r)
else: internalError(t.sons[i].info, "genAsmOrEmitStmt()")

View File

@@ -11,7 +11,7 @@
# ------------------------- Name Mangling --------------------------------
import debuginfo
import debuginfo, sighashes
proc isKeyword(w: PIdent): bool =
# Nim and C++ share some keywords
@@ -29,14 +29,27 @@ proc mangleField(name: PIdent): string =
# Mangling makes everything lowercase,
# but some identifiers are C keywords
proc hashOwner(s: PSym): FilenameHash =
proc hashOwner(s: PSym): SigHash =
var m = s
while m.kind != skModule: m = m.owner
let p = m.owner
assert p.kind == skPackage
result = gDebugInfo.register(p.name.s, m.name.s)
proc mangleName(s: PSym): Rope =
proc idOrSig(m: BModule; s: PSym): BiggestInt =
if s.kind in routineKinds and s.typ != nil and sfExported in s.flags:
# signatures for exported routines are reliable enough to
# produce a unique name and this means produced C++ is more stable wrt
# Nim changes:
let h = hashType(s.typ, {considerParamNames})
if m.hashConflicts.containsOrIncl(cast[int](h)):
result = s.id
else:
result = BiggestInt(h)
else:
result = s.id
proc mangleName(m: BModule; s: PSym): Rope =
result = s.loc.r
if result == nil:
let keepOrigName = s.kind in skLocalVars - {skForVar} and
@@ -86,7 +99,7 @@ proc mangleName(s: PSym): Rope =
result.add "0"
else:
add(result, ~"_")
add(result, rope(s.id))
add(result, rope(m.idOrSig(s)))
add(result, ~"_")
add(result, rope(hashOwner(s).BiggestInt))
s.loc.r = result
@@ -95,12 +108,22 @@ proc typeName(typ: PType): Rope =
result = if typ.sym != nil: typ.sym.name.s.mangle.rope
else: ~"TY"
proc getTypeName(typ: PType): Rope =
proc getTypeName(m: BModule; typ: PType): Rope =
if typ.sym != nil and {sfImportc, sfExportc} * typ.sym.flags != {}:
result = typ.sym.loc.r
else:
if typ.loc.r == nil:
typ.loc.r = typ.typeName & typ.id.rope
when false:
# doesn't work yet and would require bigger rewritings
let h = hashType(typ, {considerParamNames})
let sig =
if m.hashConflicts.containsOrIncl(cast[int](h)):
BiggestInt typ.id
else:
BiggestInt h
else:
let sig = BiggestInt typ.id
typ.loc.r = typ.typeName & sig.rope
result = typ.loc.r
if result == nil: internalError("getTypeName: " & $typ.kind)
@@ -235,9 +258,9 @@ proc fillResult(param: PSym) =
incl(param.loc.flags, lfIndirect)
param.loc.s = OnUnknown
proc typeNameOrLiteral(t: PType, literal: string): Rope =
proc typeNameOrLiteral(m: BModule; t: PType, literal: string): Rope =
if t.sym != nil and sfImportc in t.sym.flags and t.sym.magic == mNone:
result = getTypeName(t)
result = getTypeName(m, t)
else:
result = rope(literal)
@@ -249,16 +272,16 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
"NU", "NU8", "NU16", "NU32", "NU64"]
case typ.kind
of tyPointer:
result = typeNameOrLiteral(typ, "void*")
result = typeNameOrLiteral(m, typ, "void*")
of tyString:
discard cgsym(m, "NimStringDesc")
result = typeNameOrLiteral(typ, "NimStringDesc*")
of tyCString: result = typeNameOrLiteral(typ, "NCSTRING")
of tyBool: result = typeNameOrLiteral(typ, "NIM_BOOL")
of tyChar: result = typeNameOrLiteral(typ, "NIM_CHAR")
of tyNil: result = typeNameOrLiteral(typ, "0")
result = typeNameOrLiteral(m, typ, "NimStringDesc*")
of tyCString: result = typeNameOrLiteral(m, typ, "NCSTRING")
of tyBool: result = typeNameOrLiteral(m, typ, "NIM_BOOL")
of tyChar: result = typeNameOrLiteral(m, typ, "NIM_CHAR")
of tyNil: result = typeNameOrLiteral(m, typ, "0")
of tyInt..tyUInt64:
result = typeNameOrLiteral(typ, NumericalTypeToStr[typ.kind])
result = typeNameOrLiteral(m, typ, NumericalTypeToStr[typ.kind])
of tyDistinct, tyRange, tyOrdinal: result = getSimpleTypeDesc(m, typ.sons[0])
of tyStatic:
if typ.n != nil: result = getSimpleTypeDesc(m, lastSon typ)
@@ -290,7 +313,7 @@ proc getTypeForward(m: BModule, typ: PType): Rope =
if result != nil: return
case typ.kind
of tySequence, tyTuple, tyObject:
result = getTypeName(typ)
result = getTypeName(m, typ)
if not isImportedType(typ):
addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
[structOrUnion(typ), result])
@@ -336,7 +359,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
var param = t.n.sons[i].sym
if isCompileTimeOnly(param.typ): continue
if params != nil: add(params, ~", ")
fillLoc(param.loc, locParam, param.typ, mangleName(param),
fillLoc(param.loc, locParam, param.typ, mangleName(m, param),
param.paramStorageLoc)
if ccgIntroducedPtr(param):
add(params, getTypeDescWeak(m, param.typ, check))
@@ -583,7 +606,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope =
let t = if t.kind == tyRange: t.lastSon else: t
result = cacheGetType(m.typeCache, t)
if result == nil:
result = getTypeName(t)
result = getTypeName(m, t)
if not (isImportedCppType(t) or
(sfImportc in t.sym.flags and t.sym.magic == mNone)):
idTablePut(m.typeCache, t, result)
@@ -609,7 +632,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope =
gDebugInfo.registerEnum(EnumDesc(size: size, owner: owner, id: t.sym.id,
name: t.sym.name.s, values: vals))
of tyProc:
result = getTypeName(t)
result = getTypeName(m, t)
idTablePut(m.typeCache, t, result)
var rettype, desc: Rope
genProcParams(m, t, rettype, desc, check, true, true)
@@ -627,7 +650,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope =
# with the name of the struct, not with the pointer to the struct:
result = cacheGetType(m.forwTypeCache, t)
if result == nil:
result = getTypeName(t)
result = getTypeName(m, t)
if not isImportedType(t):
addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
[structOrUnion(t), result])
@@ -650,7 +673,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope =
of tyArrayConstr, tyArray:
var n: BiggestInt = lengthOrd(t)
if n <= 0: n = 1 # make an array of at least one element
result = getTypeName(t)
result = getTypeName(m, t)
idTablePut(m.typeCache, t, result)
if not isImportedType(t):
let foo = getTypeDescAux(m, t.sons[1], check)
@@ -660,7 +683,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope =
if isImportedCppType(t) and typ.kind == tyGenericInst:
# for instantiated templates we do not go through the type cache as the
# the type cache is not aware of 'tyGenericInst'.
let cppName = getTypeName(t)
let cppName = getTypeName(m, t)
var i = 0
var chunkStart = 0
while i < cppName.data.len:
@@ -693,7 +716,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope =
else:
result = cacheGetType(m.forwTypeCache, t)
if result == nil:
result = getTypeName(t)
result = getTypeName(m, t)
if not isImportedType(t):
addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
[structOrUnion(t), result])
@@ -703,7 +726,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope =
else: getTupleDesc(m, t, result, check)
if not isImportedType(t): add(m.s[cfsTypes], recdesc)
of tySet:
result = getTypeName(t.lastSon) & "Set"
result = getTypeName(m, t.lastSon) & "Set"
idTablePut(m.typeCache, t, result)
if not isImportedType(t):
let s = int(getSize(t))
@@ -764,7 +787,7 @@ proc genProcHeader(m: BModule, prc: PSym): Rope =
elif prc.typ.callConv == ccInline:
result.add "static "
var check = initIntSet()
fillLoc(prc.loc, locProc, prc.typ, mangleName(prc), OnUnknown)
fillLoc(prc.loc, locProc, prc.typ, mangleName(m, prc), OnUnknown)
genProcParams(m, prc.typ, rettype, params, check)
# careful here! don't access ``prc.ast`` as that could reload large parts of
# the object graph!
@@ -810,7 +833,7 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) =
proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope) =
var base: Rope
if (sonsLen(typ) > 0) and (typ.sons[0] != nil):
if sonsLen(typ) > 0 and typ.sons[0] != nil:
var x = typ.sons[0]
if typ.kind == tyObject: x = x.skipTypes(skipPtrs)
base = genTypeInfo(m, x)
@@ -1005,7 +1028,17 @@ proc genDeepCopyProc(m: BModule; s: PSym; result: Rope) =
proc genTypeInfo(m: BModule, t: PType): Rope =
let origType = t
var t = getUniqueType(t)
result = "NTI$1" % [rope(t.id)]
when false:
let h = hashType(t, {considerParamNames})
let tid = if m.hashConflicts.containsOrIncl(cast[int](h)):
BiggestInt t.id
else:
BiggestInt h
else:
let tid = t.id
result = "NTI$1" % [rope(tid)]
if containsOrIncl(m.typeInfoMarker, t.id):
return "(&".rope & result & ")".rope

View File

@@ -359,7 +359,7 @@ proc localDebugInfo(p: BProc, s: PSym) =
proc localVarDecl(p: BProc; s: PSym): Rope =
if s.loc.k == locNone:
fillLoc(s.loc, locLocalVar, s.typ, mangleName(s), OnStack)
fillLoc(s.loc, locLocalVar, s.typ, mangleName(p.module, s), OnStack)
if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy)
result = getTypeDesc(p.module, s.loc.t)
if s.constraint.isNil:
@@ -387,7 +387,7 @@ proc mangleDynLibProc(sym: PSym): Rope
proc assignGlobalVar(p: BProc, s: PSym) =
if s.loc.k == locNone:
fillLoc(s.loc, locGlobalVar, s.typ, mangleName(s), OnHeap)
fillLoc(s.loc, locGlobalVar, s.typ, mangleName(p.module, s), OnHeap)
if lfDynamicLib in s.loc.flags:
var q = findPendingModule(p.module, s)
@@ -426,9 +426,9 @@ proc assignParam(p: BProc, s: PSym) =
assert(s.loc.r != nil)
localDebugInfo(p, s)
proc fillProcLoc(sym: PSym) =
proc fillProcLoc(m: BModule; sym: PSym) =
if sym.loc.k == locNone:
fillLoc(sym.loc, locProc, sym.typ, mangleName(sym), OnStack)
fillLoc(sym.loc, locProc, sym.typ, mangleName(m, sym), OnStack)
proc getLabel(p: BProc): TLabel =
inc(p.labels)
@@ -729,7 +729,7 @@ proc genProcPrototype(m: BModule, sym: PSym) =
add(m.s[cfsProcHeaders], rfmt(nil, "$1;$n", header))
proc genProcNoForward(m: BModule, prc: PSym) =
fillProcLoc(prc)
fillProcLoc(m, prc)
useHeader(m, prc)
if lfImportCompilerProc in prc.loc.flags:
# dependency to a compilerproc:
@@ -757,7 +757,7 @@ proc requestConstImpl(p: BProc, sym: PSym) =
var m = p.module
useHeader(m, sym)
if sym.loc.k == locNone:
fillLoc(sym.loc, locData, sym.typ, mangleName(sym), OnStatic)
fillLoc(sym.loc, locData, sym.typ, mangleName(p.module, sym), OnStatic)
if lfNoDecl in sym.loc.flags: return
# declare implementation:
var q = findPendingModule(m, sym)
@@ -778,7 +778,7 @@ proc isActivated(prc: PSym): bool = prc.typ != nil
proc genProc(m: BModule, prc: PSym) =
if sfBorrow in prc.flags or not isActivated(prc): return
fillProcLoc(prc)
fillProcLoc(m, prc)
if sfForward in prc.flags: addForwardedProc(m, prc)
else:
genProcNoForward(m, prc)
@@ -792,7 +792,7 @@ proc genProc(m: BModule, prc: PSym) =
proc genVarPrototypeAux(m: BModule, sym: PSym) =
#assert(sfGlobal in sym.flags)
useHeader(m, sym)
fillLoc(sym.loc, locGlobalVar, sym.typ, mangleName(sym), OnHeap)
fillLoc(sym.loc, locGlobalVar, sym.typ, mangleName(m, sym), OnHeap)
if (lfNoDecl in sym.loc.flags) or containsOrIncl(m.declaredThings, sym.id):
return
if sym.owner.id != m.module.id:
@@ -1107,6 +1107,7 @@ proc rawNewModule(module: PSym, filename: string): BModule =
result.forwardedProcs = @[]
result.typeNodesName = getTempName(result)
result.nimTypesName = getTempName(result)
result.hashConflicts = initIntSet()
# no line tracing for the init sections of the system module so that we
# don't generate a TFrame which can confuse the stack botton initialization:
if sfSystemModule in module.flags:
@@ -1140,6 +1141,7 @@ proc resetModule*(m: BModule) =
nullify m.s
m.typeNodes = 0
m.nimTypes = 0
m.hashConflicts = initIntSet()
nullify m.extensionLoaders
# indicate that this is now cached module

View File

@@ -127,6 +127,7 @@ type
extensionLoaders*: array['0'..'9', Rope] # special procs for the
# OpenGL wrapper
injectStmt*: Rope
hashConflicts*: IntSet
var
mainModProcs*, mainModInit*, otherModsInit*, mainDatInit*: Rope

View File

@@ -11,14 +11,15 @@
## to Nim. The data structure has been designed to produce something useful with Nim's marshal
## module.
import sighashes
type
FilenameHash* = uint32
FilenameMapping* = object
package*, file*: string
mangled*: FilenameHash
mangled*: SigHash
EnumDesc* = object
size*: int
owner*: FilenameHash
owner*: SigHash
id*: int
name*: string
values*: seq[(string, int)]
@@ -28,11 +29,7 @@ type
enums*: seq[EnumDesc]
conflicts*: bool
proc sdbmHash(hash: FilenameHash, c: char): FilenameHash {.inline.} =
return FilenameHash(c) + (hash shl 6) + (hash shl 16) - hash
proc sdbmHash(package, file: string): FilenameHash =
template `&=`(x, c) = x = sdbmHash(x, c)
proc sdbmHash(package, file: string): SigHash =
result = 0
for i in 0..<package.len:
result &= package[i]
@@ -40,7 +37,7 @@ proc sdbmHash(package, file: string): FilenameHash =
for i in 0..<file.len:
result &= file[i]
proc register*(self: var DebugInfo; package, file: string): FilenameHash =
proc register*(self: var DebugInfo; package, file: string): SigHash =
result = sdbmHash(package, file)
for f in self.files:
if f.mangled == result:
@@ -49,7 +46,7 @@ proc register*(self: var DebugInfo; package, file: string): FilenameHash =
break
self.files.add(FilenameMapping(package: package, file: file, mangled: result))
proc hasEnum*(self: DebugInfo; ename: string; id: int; owner: FilenameHash): bool =
proc hasEnum*(self: DebugInfo; ename: string; id: int; owner: SigHash): bool =
for en in self.enums:
if en.owner == owner and en.name == ename and en.id == id: return true

View File

@@ -385,9 +385,9 @@ proc symStack(w: PRodWriter): int =
inc result
elif iiTableGet(w.index.tab, s.id) == InvalidKey:
var m = getModule(s)
if m == nil and s.kind != skPackage:
if m == nil and s.kind != skPackage and sfGenSym notin s.flags:
internalError("symStack: module nil: " & s.name.s)
if s.kind == skPackage or m.id == w.module.id or sfFromGeneric in s.flags:
if s.kind == skPackage or {sfFromGeneric, sfGenSym} * s.flags != {} or m.id == w.module.id:
# put definition in here
var L = w.data.len
addToIndex(w.index, s.id, L)

137
compiler/sighashes.nim Normal file
View File

@@ -0,0 +1,137 @@
#
#
# The Nim Compiler
# (c) Copyright 2016 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Computes hash values for routine (proc, method etc) signatures.
import ast
type
SigHash* = uint32 ## a hash good enough for a filename or a proc signature
proc sdbmHash(hash: SigHash, c: char): SigHash {.inline.} =
return SigHash(c) + (hash shl 6) + (hash shl 16) - hash
template `&=`*(x: var SigHash, c: char) = x = sdbmHash(x, c)
template `&=`*(x: var SigHash, s: string) =
for c in s: x = sdbmHash(x, c)
proc hashSym(c: var SigHash, s: PSym) =
if sfAnon in s.flags or s.kind == skGenericParam:
c &= ":anon"
else:
var it = s
while it != nil:
c &= it.name.s
c &= "."
it = it.owner
proc hashTree(c: var SigHash, n: PNode) =
template lowlevel(v) =
for i in 0..<sizeof(v): c = sdbmHash(c, cast[cstring](unsafeAddr(v))[i])
if n == nil:
c &= "\255"
return
let k = n.kind
c &= char(k)
# we really must not hash line information. 'n.typ' is debatable but
# shouldn't be necessary for now and avoids potential infinite recursions.
case n.kind
of nkEmpty, nkNilLit, nkType: discard
of nkIdent:
c &= n.ident.s
of nkSym:
hashSym(c, n.sym)
of nkCharLit..nkUInt64Lit:
let v = n.intVal
lowlevel v
of nkFloatLit..nkFloat64Lit:
let v = n.floatVal
lowlevel v
of nkStrLit..nkTripleStrLit:
c &= n.strVal
else:
for i in 0.. <n.len: hashTree(c, n.sons[i])
type
ConsiderFlag* = enum
considerParamNames
proc hashType(c: var SigHash, t: PType; flags: set[ConsiderFlag]) =
# modelled after 'typeToString'
if t == nil:
c &= "\254"
return
c &= char(t.kind)
# Every cyclic type in Nim need to be constructed via some 't.sym', so this
# is actually safe without an infinite recursion check:
if t.sym != nil and sfAnon notin t.sym.flags:
# t.n for literals, but not for e.g. objects!
if t.kind in {tyFloat, tyInt}: c.hashTree(t.n)
c.hashSym(t.sym)
return
case t.kind
of tyGenericBody, tyGenericInst, tyGenericInvocation:
for i in countup(0, sonsLen(t) - 1 - ord(t.kind != tyGenericInvocation)):
c.hashType t.sons[i], flags
of tyUserTypeClass:
if t.sym != nil and t.sym.owner != nil:
c &= t.sym.owner.name.s
else:
c &= "unknown typeclass"
of tyUserTypeClassInst:
let body = t.sons[0]
c.hashSym body.sym
for i in countup(1, sonsLen(t) - 2):
c.hashType t.sons[i], flags
of tyFromExpr, tyFieldAccessor:
c.hashTree(t.n)
of tyArrayConstr:
c.hashTree(t.sons[0].n)
c.hashType(t.sons[1], flags)
of tyTuple:
if t.n != nil:
assert(sonsLen(t.n) == sonsLen(t))
for i in countup(0, sonsLen(t.n) - 1):
assert(t.n.sons[i].kind == nkSym)
c &= t.n.sons[i].sym.name.s
c &= ':'
c.hashType(t.sons[i], flags)
c &= ','
else:
for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i], flags
of tyRange:
c.hashTree(t.n)
c.hashType(t.sons[0], flags)
of tyProc:
c &= (if tfIterator in t.flags: "iterator " else: "proc ")
if considerParamNames in flags and t.n != nil:
let params = t.n
for i in 1..<params.len:
let param = params[i].sym
c &= param.name.s
c &= ':'
c.hashType(param.typ, flags)
c &= ','
c.hashType(t.sons[0], flags)
else:
for i in 0.. <t.len: c.hashType(t.sons[i], flags)
c &= char(t.callConv)
if tfNoSideEffect in t.flags: c &= ".noSideEffect"
if tfThread in t.flags: c &= ".thread"
else:
for i in 0.. <t.len: c.hashType(t.sons[i], flags)
if tfNotNil in t.flags: c &= "not nil"
proc hashType*(t: PType; flags: set[ConsiderFlag]): SigHash =
result = 0
hashType result, t, flags

View File

@@ -20,8 +20,11 @@ type
preferName, preferDesc, preferExported, preferModuleInfo, preferGenericArg
proc typeToString*(typ: PType; prefer: TPreferedDesc = preferName): string
proc base*(t: PType): PType
# ------------------- type iterator: ----------------------------------------
proc base*(t: PType): PType =
result = t.sons[0]
# ------------------- type iterator: ----------------------------------------
type
TTypeIter* = proc (t: PType, closure: RootRef): bool {.nimcall.} # true if iteration should stop
TTypeMutator* = proc (t: PType, closure: RootRef): PType {.nimcall.} # copy t and mutate it
@@ -444,8 +447,8 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
add(result, typeToString(t.sons[i], preferGenericArg))
add(result, ']')
of tyTypeDesc:
if t.base.kind == tyNone: result = "typedesc"
else: result = "typedesc[" & typeToString(t.base) & "]"
if t.sons[0].kind == tyNone: result = "typedesc"
else: result = "typedesc[" & typeToString(t.sons[0]) & "]"
of tyStatic:
internalAssert t.len > 0
if prefer == preferGenericArg and t.n != nil:
@@ -572,9 +575,6 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
result = typeToStr[t.kind]
result.addTypeFlags(t)
proc base(t: PType): PType =
result = t.sons[0]
proc firstOrd(t: PType): BiggestInt =
case t.kind
of tyBool, tyChar, tySequence, tyOpenArray, tyString, tyVarargs, tyProxy: