mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2025-12-28 17:04:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

baby steps for incremental compilation

Browse Source
This commit is contained in:
Andreas Rumpf
2018-05-30 23:50:34 +02:00
parent a36c779f39
commit 61fb83ecbb
23 changed files with 70 additions and 1982 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
compiler/ast.nim
View File

@@ -1724,3 +1724,5 @@ template incompleteType*(t: PType): bool =
template typeCompleted*(s: PSym) =
incl s.flags, sfNoForward
template getBody*(s: PSym): PNode = s.ast[bodyPos]

17
compiler/cgen.nim
View File

@@ -12,7 +12,7 @@
import
ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets,
nversion, nimsets, msgs, std / sha1, bitsets, idents, types,
ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth,
ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth,
condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
lowerings, semparallel, tables, sets, ndi, lineinfos
@@ -1306,7 +1306,9 @@ proc rawNewModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
proc newModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
# we should create only one cgen module for each module sym
result = rawNewModule(g, module, conf)
growCache g.modules, module.position
if module.position >= g.modules.len:
setLen(g.modules, module.position + 1)
#growCache g.modules, module.position
g.modules[module.position] = result
template injectG() {.dirty.} =
@@ -1417,7 +1419,7 @@ proc writeModule(m: BModule, pending: bool) =
# generate code for the init statements of the module:
let cfile = getCFile(m)
if m.rd == nil or optForceFullMake in m.config.globalOptions:
if true or optForceFullMake in m.config.globalOptions:
genInitCode(m)
finishTypeDescriptions(m)
if sfMainModule in m.module.flags:
@@ -1476,7 +1478,6 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
# if the module is cached, we don't regenerate the main proc
# nor the dispatchers? But if the dispatchers changed?
# XXX emit the dispatchers into its own .c file?
if b.rd != nil: return
if n != nil:
m.initProc.options = initProcOptions(m)
genStmts(m.initProc, n)
@@ -1499,13 +1500,9 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
if g.generatedHeader != nil: finishModule(g.generatedHeader)
while g.forwardedProcsCounter > 0:
for m in cgenModules(g):
if m.rd == nil:
finishModule(m)
finishModule(m)
for m in cgenModules(g):
if m.rd != nil:
m.updateCachedModule
else:
m.writeModule(pending=true)
m.writeModule(pending=true)
writeMapping(config, g.mapping)
if g.generatedHeader != nil: writeHeader(g.generatedHeader)

4
compiler/commands.nim
View File

@@ -626,9 +626,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
of "help", "h":
expectNoArg(conf, switch, arg, pass, info)
helpOnError(conf, pass)
of "symbolfiles":
of "symbolfiles", "incremental":
case arg.normalize
of "on": conf.symbolFiles = enabledSf
of "on": conf.symbolFiles = v2Sf
of "off": conf.symbolFiles = disabledSf
of "writeonly": conf.symbolFiles = writeOnlySf
of "readonly": conf.symbolFiles = readOnlySf

2
compiler/destroyer.nim
View File

@@ -116,7 +116,7 @@ Remarks: Rule 1.2 is not yet implemented because ``sink`` is currently
import
intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
strutils, options, dfa, lowerings, rodread, tables, modulegraphs,
strutils, options, dfa, lowerings, tables, modulegraphs,
lineinfos
const

2
compiler/evaltempl.nim
View File

@@ -11,7 +11,7 @@
import
strutils, options, ast, astalgo, msgs, os, idents, wordrecg, renderer,
rodread, lineinfos
lineinfos
type
TemplCtx = object

5
compiler/importer.nim
View File

@@ -10,7 +10,7 @@
# This module implements the symbol importing mechanism.
import
intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups,
intsets, strutils, os, ast, astalgo, msgs, options, idents, lookups,
semdata, passes, renderer, modulepaths, sigmatch, lineinfos
proc evalImport*(c: PContext, n: PNode): PNode
@@ -74,7 +74,8 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
if s == nil:
errorUndeclaredIdentifier(c, n.info, ident.s)
else:
if s.kind == skStub: loadStub(s)
when false:
if s.kind == skStub: loadStub(s)
if s.kind notin ExportableSymKinds:
internalError(c.config, n.info, "importSymbol: 2")
# for an enumeration we have to add all identifiers

4
compiler/jsgen.nim
View File

@@ -31,8 +31,8 @@ implements the required case distinction.
import
ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options,
nversion, nimsets, msgs, std / sha1, bitsets, idents, types, os, tables,
times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils,
intsets, cgmeth, lowerings, sighashes, lineinfos
times, ropes, math, passes, ccgutils, wordrecg, renderer,
intsets, cgmeth, lowerings, sighashes, lineinfos, rodutils
from modulegraphs import ModuleGraph

2
compiler/lambdalifting.nim
View File

@@ -11,7 +11,7 @@
import
intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
idents, renderer, types, magicsys, rodread, lowerings, tables,
idents, renderer, types, magicsys, lowerings, tables,
modulegraphs, lineinfos
discard """

14
compiler/lookups.nim
View File

@@ -10,7 +10,7 @@
# This module implements lookup helpers.
import
intsets, ast, astalgo, idents, semdata, types, msgs, options, rodread,
intsets, ast, astalgo, idents, semdata, types, msgs, options,
renderer, wordrecg, idgen, nimfix.prettybase, lineinfos, strutils
proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
@@ -288,7 +288,8 @@ proc lookUp*(c: PContext, n: PNode): PSym =
return
if contains(c.ambiguousSymbols, result.id):
errorUseQualifier(c, n.info, result)
if result.kind == skStub: loadStub(result)
when false:
if result.kind == skStub: loadStub(result)
type
TLookupFlag* = enum
@@ -343,7 +344,8 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
result = errorSym(c, n.sons[1])
else:
result = nil
if result != nil and result.kind == skStub: loadStub(result)
when false:
if result != nil and result.kind == skStub: loadStub(result)
proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
case n.kind
@@ -392,7 +394,8 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
o.inSymChoice = initIntSet()
incl(o.inSymChoice, result.id)
else: discard
if result != nil and result.kind == skStub: loadStub(result)
when false:
if result != nil and result.kind == skStub: loadStub(result)
proc lastOverloadScope*(o: TOverloadIter): int =
case o.mode
@@ -443,7 +446,8 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
result = firstIdentExcluding(o.it, o.scope.symbols,
n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
if result != nil and result.kind == skStub: loadStub(result)
when false:
if result != nil and result.kind == skStub: loadStub(result)
proc pickSym*(c: PContext, n: PNode; kinds: set[TSymKind];
flags: TSymFlags = {}): PSym =

11
compiler/magicsys.nim
View File

@@ -10,7 +10,7 @@
# Built-in types and compilerprocs are registered here.
import
ast, astalgo, hashes, msgs, platform, nversion, times, idents, rodread,
ast, astalgo, hashes, msgs, platform, nversion, times, idents,
modulegraphs, lineinfos
export createMagic
@@ -31,7 +31,6 @@ proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym =
localError(g.config, info, "system module needs: " & name)
result = newSym(skError, getIdent(g.cache, name), g.systemModule, g.systemModule.info, {})
result.typ = newType(tyError, g.systemModule)
if result.kind == skStub: loadStub(result)
if result.kind == skAlias: result = result.owner
proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSym =
@@ -39,7 +38,6 @@ proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSy
let id = getIdent(g.cache, name)
var r = initIdentIter(ti, g.systemModule.tab, id)
while r != nil:
if r.kind == skStub: loadStub(r)
if r.magic == m:
# prefer the tyInt variant:
if r.typ.sons[0] != nil and r.typ.sons[0].kind == tyInt: return r
@@ -159,13 +157,6 @@ proc setIntLitType*(g: ModuleGraph; result: PNode) =
proc getCompilerProc*(g: ModuleGraph; name: string): PSym =
let ident = getIdent(g.cache, name)
result = strTableGet(g.compilerprocs, ident)
when false:
if result == nil:
result = strTableGet(g.rodCompilerprocs, ident)
if result != nil:
strTableAdd(g.compilerprocs, result)
if result.kind == skStub: loadStub(result)
if result.kind == skAlias: result = result.owner
proc registerCompilerProc*(g: ModuleGraph; s: PSym) =
strTableAdd(g.compilerprocs, s)

11
compiler/main.nim
View File

@@ -14,7 +14,7 @@ when not defined(nimcore):
import
llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs,
os, condsyms, rodread, rodwrite, times,
os, condsyms, times,
wordrecg, sem, semdata, idents, passes, docgen, extccomp,
cgen, jsgen, json, nversion,
platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
@@ -23,10 +23,6 @@ import
from magicsys import resetSysTypes
proc rodPass(g: ModuleGraph) =
if g.config.symbolFiles in {enabledSf, writeOnlySf}:
registerPass(g, rodwritePass)
proc codegenPass(g: ModuleGraph) =
registerPass g, cgenPass
@@ -47,7 +43,6 @@ proc writeDepsFile(g: ModuleGraph; project: string) =
proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) =
semanticPasses(graph)
registerPass(graph, gendependPass)
#registerPass(cleanupPass)
compileProject(graph, cache)
let project = graph.config.projectFull
writeDepsFile(graph, project)
@@ -59,7 +54,6 @@ proc commandCheck(graph: ModuleGraph; cache: IdentCache) =
graph.config.errorMax = high(int) # do not stop after first error
defineSymbol(graph.config.symbols, "nimcheck")
semanticPasses(graph) # use an empty backend for semantic checking only
rodPass(graph)
compileProject(graph, cache)
proc commandDoc2(graph: ModuleGraph; cache: IdentCache; json: bool) =
@@ -67,7 +61,6 @@ proc commandDoc2(graph: ModuleGraph; cache: IdentCache; json: bool) =
semanticPasses(graph)
if json: registerPass(graph, docgen2JsonPass)
else: registerPass(graph, docgen2Pass)
#registerPass(cleanupPass())
compileProject(graph, cache)
finishDoc2Pass(graph.config.projectName)
@@ -76,8 +69,6 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) =
extccomp.initVars(conf)
semanticPasses(graph)
registerPass(graph, cgenPass)
rodPass(graph)
#registerPass(cleanupPass())
compileProject(graph, cache)
cgenWriteModules(graph.backend, conf)

6
compiler/modules.nim
View File

@@ -10,7 +10,7 @@
## Implements the module handling, including the caching of modules.
import
ast, astalgo, magicsys, std / sha1, rodread, msgs, cgendata, sigmatch, options,
ast, astalgo, magicsys, std / sha1, msgs, cgendata, sigmatch, options,
idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod,
lineinfos
@@ -42,7 +42,9 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
result.owner = packSym
result.position = int fileIdx
growCache graph.modules, int fileIdx
if int(fileIdx) >= graph.modules.len:
setLen(graph.modules, int(fileIdx) + 1)
#growCache graph.modules, int fileIdx
graph.modules[result.position] = result
incl(result.flags, sfUsed)

4
compiler/options.nim
View File

@@ -121,7 +121,7 @@ type
notnil
SymbolFilesOption* = enum
disabledSf, enabledSf, writeOnlySf, readOnlySf, v2Sf
disabledSf, writeOnlySf, readOnlySf, v2Sf
TSystemCC* = enum
ccNone, ccGcc, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc,
@@ -362,7 +362,7 @@ proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.cmd in {cmdDoc,
proc usesNativeGC*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gcRefc
template compilationCachePresent*(conf: ConfigRef): untyped =
conf.symbolFiles in {enabledSf, writeOnlySf}
conf.symbolFiles in {v2Sf, writeOnlySf}
# {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {}
template optPreserveOrigSource*(conf: ConfigRef): untyped =

15
compiler/passes.nim
View File

@@ -13,13 +13,13 @@
import
strutils, options, ast, astalgo, llstream, msgs, platform, os,
condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder, rod,
nimsets, syntaxes, times, idgen, modulegraphs, reorder, rod,
lineinfos
type
PRodReader* = ref object
TPassContext* = object of RootObj # the pass's context
rd*: PRodReader # != nil if created by "openCached"
PPassContext* = ref TPassContext
@@ -107,8 +107,6 @@ proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym,
for i in countup(0, gPassesLen - 1):
if not isNil(gPasses[i].openCached):
a[i] = gPasses[i].openCached(g, module, rd)
if a[i] != nil:
a[i].rd = rd
else:
a[i] = nil
@@ -198,7 +196,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
if not isNil(gPasses[i].close) and not gPasses[i].isFrontend:
m = gPasses[i].close(graph, a[i], m)
a[i] = nil
elif rd == nil:
else:
openPasses(graph, a, module, cache)
if stream == nil:
let filename = toFullPathConsiderDirty(graph.config, fileIdx)
@@ -241,11 +239,4 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
closePasses(graph, a)
# id synchronization point for more consistent code generation:
idSynchronizationPoint(1000)
else:
openPassesCached(graph, a, module, rd)
var n = loadInitSection(rd)
for i in countup(0, sonsLen(n) - 1):
if graph.stopCompile(): break
processTopLevelStmtCached(n.sons[i], a)
closePassesCached(graph, a)
result = true

2
compiler/plugins/itersgen.nim
View File

@@ -11,7 +11,7 @@
import ".." / [ast, astalgo,
magicsys, lookups, semdata,
lambdalifting, rodread, msgs]
lambdalifting, msgs]
proc iterToProcImpl*(c: PContext, n: PNode): PNode =
result = newNodeI(nkStmtList, n.info)

5
compiler/pragmas.nim
View File

@@ -12,7 +12,7 @@
import
os, platform, condsyms, ast, astalgo, idents, semdata, msgs, renderer,
wordrecg, ropes, options, strutils, extccomp, math, magicsys, trees,
rodread, types, lookups, lineinfos
types, lookups, lineinfos
const
FirstCallConv* = wNimcall
@@ -482,7 +482,8 @@ proc semAsmOrEmit*(con: PContext, n: PNode, marker: char): PNode =
if sub != "":
var e = searchInScopes(con, getIdent(con.cache, sub))
if e != nil:
if e.kind == skStub: loadStub(e)
when false:
if e.kind == skStub: loadStub(e)
incl(e.flags, sfUsed)
addSon(result, newSymNode(e))
else:

31
compiler/rodimpl.nim
View File

@@ -43,7 +43,7 @@ proc needsRecompile(g: ModuleGraph; fileIdx: FileIndex; fullpath: string;
return false
proc getModuleId*(g: ModuleGraph; fileIdx: FileIndex; fullpath: string): int =
if g.config.symbolFiles != v2Sf: return getID()
if g.config.symbolFiles in {disabledSf, writeOnlySf}: return getID()
let module = g.incr.db.getRow(
sql"select id, fullHash from modules where fullpath = ?", fullpath)
let currentFullhash = hashFileCached(g.config, fileIdx, fullpath)
@@ -57,6 +57,7 @@ proc getModuleId*(g: ModuleGraph; fileIdx: FileIndex; fullpath: string): int =
doAssert(result != 0)
var cycleCheck = initIntSet()
if not needsRecompile(g, fileIdx, fullpath, cycleCheck):
echo "cached successfully! ", fullpath
return -result
db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0])
db.exec(sql"delete from deps where module = ?", module[0])
@@ -342,17 +343,21 @@ proc storeSym(g: ModuleGraph; s: PSym) =
encodeSym(g, s, buf)
# XXX only store the name for exported symbols in order to speed up lookup
# times once we enable the skStub logic.
let m = getModule(s)
let mid = if m == nil: 0 else: abs(m.id)
db.exec(sql"insert into syms(nimid, module, name, data, exported) values (?, ?, ?, ?, ?)",
s.id, abs(getModule(s).id), s.name.s, buf, ord(sfExported in s.flags))
s.id, mid, s.name.s, buf, ord(sfExported in s.flags))
proc storeType(g: ModuleGraph; t: PType) =
var buf = newStringOfCap(160)
encodeType(g, t, buf)
let m = if t.owner != nil: getModule(t.owner) else: nil
let mid = if m == nil: 0 else: abs(m.id)
db.exec(sql"insert into types(nimid, module, data) values (?, ?, ?)",
t.id, abs(getModule(t.owner).id), buf)
t.id, mid, buf)
proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) =
if g.config.symbolFiles != v2Sf: return
if g.config.symbolFiles == disabledSf: return
var buf = newStringOfCap(160)
encodeNode(g, module.info, n, buf)
db.exec(sql"insert into toplevelstmts(module, position, data) values (?, ?, ?)",
@@ -377,11 +382,14 @@ proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) =
inc i
proc storeRemaining*(g: ModuleGraph; module: PSym) =
if g.config.symbolFiles != v2Sf: return
if g.config.symbolFiles == disabledSf: return
var stillForwarded: seq[PSym] = @[]
for s in w.forwardedSyms:
assert sfForward notin s.flags
storeSym(g, s)
w.forwardedSyms.setLen 0
if sfForward notin s.flags:
storeSym(g, s)
else:
stillForwarded.add s
swap w.forwardedSyms, stillForwarded
# ---------------- decoder -----------------------------------
@@ -529,6 +537,8 @@ proc loadBlob(g; query: SqlQuery; id: int): BlobReader =
internalError(g.config, "symbolfiles: cannot find ID " & $ id)
result = BlobReader(pos: 0)
shallowCopy(result.s, blob)
# ensure we can read without index checks:
result.s.add '\0'
proc loadType(g; id: int; info: TLineInfo): PType =
result = g.incr.r.types.getOrDefault(id)
@@ -742,6 +752,8 @@ proc loadModuleSymTab(g; module: PSym) =
if s == nil:
var b = BlobReader(pos: 0)
shallowCopy(b.s, row[1])
# ensure we can read without index checks:
b.s.add '\0'
s = loadSymFromBlob(g, b, module.info)
assert s != nil
strTableAdd(module.tab, s)
@@ -749,7 +761,6 @@ proc loadModuleSymTab(g; module: PSym) =
g.systemModule = module
proc loadNode*(g: ModuleGraph; module: PSym; index: int): PNode =
assert g.config.symbolFiles == v2Sf
if index == 0:
loadModuleSymTab(g, module)
#index = parseInt db.getValue(
@@ -763,7 +774,7 @@ proc loadNode*(g: ModuleGraph; module: PSym; index: int): PNode =
result = decodeNode(g, b, module.info)
proc setupModuleCache*(g: ModuleGraph) =
if g.config.symbolFiles != v2Sf: return
if g.config.symbolFiles == disabledSf: return
let dbfile = getNimcacheDir(g.config) / "rodfiles.db"
if not fileExists(dbfile):
db = open(connection=dbfile, user="nim", password="",

1245
compiler/rodread.nim
View File

File diff suppressed because it is too large Load Diff

653
compiler/rodwrite.nim
View File

@@ -1,653 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2012 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
# This module is responsible for writing of rod files. Note that writing of
# rod files is a pass, reading of rod files is not! This is why reading and
# writing of rod files is split into two different modules.
import
intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform,
condsyms, ropes, idents, std / sha1, rodread, passes, idgen,
rodutils, modulepaths, lineinfos
from modulegraphs import ModuleGraph
type
TRodWriter = object of TPassContext
module: PSym
hash: SecureHash
options: TOptions
defines: string
inclDeps: string
modDeps: string
interf: string
compilerProcs: string
index, imports: TIndex
converters, methods: string
init: string
data: string
sstack: TSymSeq # a stack of symbols to process
tstack: TTypeSeq # a stack of types to process
files: TStringSeq
origFile: string
cache: IdentCache
config: ConfigRef
PRodWriter = ref TRodWriter
proc getDefines(conf: ConfigRef): string =
result = ""
for d in definedSymbolNames(conf.symbols):
if result.len != 0: add(result, " ")
add(result, d)
proc fileIdx(w: PRodWriter, filename: string): int =
for i in countup(0, high(w.files)):
if w.files[i] == filename:
return i
result = len(w.files)
setLen(w.files, result + 1)
w.files[result] = filename
template filename*(w: PRodWriter): string =
toFilename(w.config, FileIndex w.module.position)
proc newRodWriter(hash: SecureHash, module: PSym; cache: IdentCache;
config: ConfigRef): PRodWriter =
new(result)
result.config = config
result.sstack = @[]
result.tstack = @[]
initIiTable(result.index.tab)
initIiTable(result.imports.tab)
result.index.r = ""
result.imports.r = ""
result.hash = hash
result.module = module
result.defines = getDefines(config)
result.options = config.options
result.files = @[]
result.inclDeps = ""
result.modDeps = ""
result.interf = newStringOfCap(2_000)
result.compilerProcs = ""
result.converters = ""
result.methods = ""
result.init = ""
result.origFile = toFullPath(config, module.info)
result.data = newStringOfCap(12_000)
result.cache = cache
proc addModDep(w: PRodWriter, dep: string; info: TLineInfo) =
if w.modDeps.len != 0: add(w.modDeps, ' ')
let resolved = findModule(w.config, dep, toFullPath(w.config, info))
encodeVInt(fileIdx(w, resolved), w.modDeps)
const
rodNL = "\x0A"
proc addInclDep(w: PRodWriter, dep: string; info: TLineInfo) =
let resolved = findModule(w.config, dep, toFullPath(w.config, info))
encodeVInt(fileIdx(w, resolved), w.inclDeps)
add(w.inclDeps, " ")
encodeStr($secureHashFile(resolved), w.inclDeps)
add(w.inclDeps, rodNL)
proc pushType(w: PRodWriter, t: PType) =
# check so that the stack does not grow too large:
if iiTableGet(w.index.tab, t.id) == InvalidKey:
w.tstack.add(t)
proc pushSym(w: PRodWriter, s: PSym) =
# check so that the stack does not grow too large:
if iiTableGet(w.index.tab, s.id) == InvalidKey:
when false:
if s.kind == skMethod:
echo "encoding ", s.id, " ", s.name.s
writeStackTrace()
w.sstack.add(s)
proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
result: var string) =
if n == nil:
# nil nodes have to be stored too:
result.add("()")
return
result.add('(')
encodeVInt(ord(n.kind), result)
# we do not write comments for now
# Line information takes easily 20% or more of the filesize! Therefore we
# omit line information if it is the same as the father's line information:
if fInfo.fileIndex != n.info.fileIndex:
result.add('?')
encodeVInt(n.info.col, result)
result.add(',')
encodeVInt(int n.info.line, result)
result.add(',')
encodeVInt(fileIdx(w, toFullPath(w.config, n.info)), result)
elif fInfo.line != n.info.line:
result.add('?')
encodeVInt(n.info.col, result)
result.add(',')
encodeVInt(int n.info.line, result)
elif fInfo.col != n.info.col:
result.add('?')
encodeVInt(n.info.col, result)
# No need to output the file index, as this is the serialization of one
# file.
var f = n.flags * PersistentNodeFlags
if f != {}:
result.add('$')
encodeVInt(cast[int32](f), result)
if n.typ != nil:
result.add('^')
encodeVInt(n.typ.id, result)
pushType(w, n.typ)
case n.kind
of nkCharLit..nkUInt64Lit:
if n.intVal != 0:
result.add('!')
encodeVBiggestInt(n.intVal, result)
of nkFloatLit..nkFloat64Lit:
if n.floatVal != 0.0:
result.add('!')
encodeStr($n.floatVal, result)
of nkStrLit..nkTripleStrLit:
if n.strVal != "":
result.add('!')
encodeStr(n.strVal, result)
of nkIdent:
result.add('!')
encodeStr(n.ident.s, result)
of nkSym:
result.add('!')
encodeVInt(n.sym.id, result)
pushSym(w, n.sym)
else:
for i in countup(0, sonsLen(n) - 1):
encodeNode(w, n.info, n.sons[i], result)
add(result, ')')
proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
var oldLen = result.len
result.add('<')
if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
if loc.storage != low(loc.storage):
add(result, '*')
encodeVInt(ord(loc.storage), result)
if loc.flags != {}:
add(result, '$')
encodeVInt(cast[int32](loc.flags), result)
if loc.lode != nil:
add(result, '^')
encodeNode(w, unknownLineInfo(), loc.lode, result)
#encodeVInt(cast[int32](loc.t.id), result)
#pushType(w, loc.t)
if loc.r != nil:
add(result, '!')
encodeStr($loc.r, result)
if oldLen + 1 == result.len:
# no data was necessary, so remove the '<' again:
setLen(result, oldLen)
else:
add(result, '>')
proc encodeType(w: PRodWriter, t: PType, result: var string) =
if t == nil:
# nil nodes have to be stored too:
result.add("[]")
return
# we need no surrounding [] here because the type is in a line of its own
if t.kind == tyForward: internalError(w.config, "encodeType: tyForward")
# for the new rodfile viewer we use a preceding [ so that the data section
# can easily be disambiguated:
add(result, '[')
encodeVInt(ord(t.kind), result)
add(result, '+')
encodeVInt(t.id, result)
if t.n != nil:
encodeNode(w, unknownLineInfo(), t.n, result)
if t.flags != {}:
add(result, '$')
encodeVInt(cast[int32](t.flags), result)
if t.callConv != low(t.callConv):
add(result, '?')
encodeVInt(ord(t.callConv), result)
if t.owner != nil:
add(result, '*')
encodeVInt(t.owner.id, result)
pushSym(w, t.owner)
if t.sym != nil:
add(result, '&')
encodeVInt(t.sym.id, result)
pushSym(w, t.sym)
if t.size != - 1:
add(result, '/')
encodeVBiggestInt(t.size, result)
if t.align != 2:
add(result, '=')
encodeVInt(t.align, result)
if t.lockLevel.ord != UnspecifiedLockLevel.ord:
add(result, '\14')
encodeVInt(t.lockLevel.int16, result)
if t.destructor != nil and t.destructor.id != 0:
add(result, '\15')
encodeVInt(t.destructor.id, result)
pushSym(w, t.destructor)
if t.deepCopy != nil:
add(result, '\16')
encodeVInt(t.deepcopy.id, result)
pushSym(w, t.deepcopy)
if t.assignment != nil:
add(result, '\17')
encodeVInt(t.assignment.id, result)
pushSym(w, t.assignment)
if t.sink != nil:
add(result, '\18')
encodeVInt(t.sink.id, result)
pushSym(w, t.sink)
for i, s in items(t.methods):
add(result, '\19')
encodeVInt(i, result)
add(result, '\20')
encodeVInt(s.id, result)
pushSym(w, s)
encodeLoc(w, t.loc, result)
for i in countup(0, sonsLen(t) - 1):
if t.sons[i] == nil:
add(result, "^()")
else:
add(result, '^')
encodeVInt(t.sons[i].id, result)
pushType(w, t.sons[i])
proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
add(result, '|')
encodeVInt(ord(lib.kind), result)
add(result, '|')
encodeStr($lib.name, result)
add(result, '|')
encodeNode(w, info, lib.path, result)
proc encodeInstantiations(w: PRodWriter; s: seq[PInstantiation];
result: var string) =
for t in s:
result.add('\15')
encodeVInt(t.sym.id, result)
pushSym(w, t.sym)
for tt in t.concreteTypes:
result.add('\17')
encodeVInt(tt.id, result)
pushType(w, tt)
result.add('\20')
encodeVInt(t.compilesId, result)
proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
if s == nil:
# nil nodes have to be stored too:
result.add("{}")
return
# we need no surrounding {} here because the symbol is in a line of its own
encodeVInt(ord(s.kind), result)
result.add('+')
encodeVInt(s.id, result)
result.add('&')
encodeStr(s.name.s, result)
if s.typ != nil:
result.add('^')
encodeVInt(s.typ.id, result)
pushType(w, s.typ)
result.add('?')
if s.info.col != -1'i16: encodeVInt(s.info.col, result)
result.add(',')
if s.info.line != 0'u16: encodeVInt(int s.info.line, result)
result.add(',')
encodeVInt(fileIdx(w, toFullPath(w.config, s.info)), result)
if s.owner != nil:
result.add('*')
encodeVInt(s.owner.id, result)
pushSym(w, s.owner)
if s.flags != {}:
result.add('$')
encodeVInt(cast[int32](s.flags), result)
if s.magic != mNone:
result.add('@')
encodeVInt(ord(s.magic), result)
if s.options != w.options:
result.add('!')
encodeVInt(cast[int32](s.options), result)
if s.position != 0:
result.add('%')
encodeVInt(s.position, result)
if s.offset != - 1:
result.add('`')
encodeVInt(s.offset, result)
encodeLoc(w, s.loc, result)
if s.annex != nil: encodeLib(w, s.annex, s.info, result)
if s.constraint != nil:
add(result, '#')
encodeNode(w, unknownLineInfo(), s.constraint, result)
case s.kind
of skType, skGenericParam:
for t in s.typeInstCache:
result.add('\14')
encodeVInt(t.id, result)
pushType(w, t)
of routineKinds:
encodeInstantiations(w, s.procInstCache, result)
if s.gcUnsafetyReason != nil:
result.add('\16')
encodeVInt(s.gcUnsafetyReason.id, result)
pushSym(w, s.gcUnsafetyReason)
of skModule, skPackage:
encodeInstantiations(w, s.usedGenerics, result)
# we don't serialize:
#tab*: TStrTable # interface table for modules
of skLet, skVar, skField, skForVar:
if s.guard != nil:
result.add('\18')
encodeVInt(s.guard.id, result)
pushSym(w, s.guard)
if s.bitsize != 0:
result.add('\19')
encodeVInt(s.bitsize, result)
else: discard
# lazy loading will soon reload the ast lazily, so the ast needs to be
# the last entry of a symbol:
if s.ast != nil:
# we used to attempt to save space here by only storing a dummy AST if
# it is not necessary, but Nim's heavy compile-time evaluation features
# make that unfeasible nowadays:
encodeNode(w, s.info, s.ast, result)
proc addToIndex(w: var TIndex, key, val: int) =
if key - w.lastIdxKey == 1:
# we do not store a key-diff of 1 to safe space
encodeVInt(val - w.lastIdxVal, w.r)
else:
encodeVInt(key - w.lastIdxKey, w.r)
add(w.r, ' ')
encodeVInt(val - w.lastIdxVal, w.r)
add(w.r, rodNL)
w.lastIdxKey = key
w.lastIdxVal = val
iiTablePut(w.tab, key, val)
const debugWrittenIds = false
when debugWrittenIds:
var debugWritten = initIntSet()
proc symStack(w: PRodWriter): int =
var i = 0
while i < len(w.sstack):
var s = w.sstack[i]
if sfForward in s.flags:
w.sstack[result] = s
inc result
elif iiTableGet(w.index.tab, s.id) == InvalidKey:
var m = getModule(s)
#if m == nil and s.kind != skPackage and sfGenSym notin s.flags:
# internalError("symStack: module nil: " & s.name.s & " " & $s.kind & " ID " & $s.id)
if m == nil or s.kind == skPackage or {sfFromGeneric, sfGenSym} * s.flags != {} or m.id == w.module.id:
# put definition in here
var L = w.data.len
addToIndex(w.index, s.id, L)
when debugWrittenIds: incl(debugWritten, s.id)
encodeSym(w, s, w.data)
add(w.data, rodNL)
# put into interface section if appropriate:
if {sfExported, sfFromGeneric} * s.flags == {sfExported} and
s.kind in ExportableSymKinds:
encodeStr(s.name.s, w.interf)
add(w.interf, ' ')
encodeVInt(s.id, w.interf)
add(w.interf, rodNL)
if sfCompilerProc in s.flags:
encodeStr(s.name.s, w.compilerProcs)
add(w.compilerProcs, ' ')
encodeVInt(s.id, w.compilerProcs)
add(w.compilerProcs, rodNL)
if s.kind == skConverter or (s.ast != nil and hasPattern(s)):
if w.converters.len != 0: add(w.converters, ' ')
encodeVInt(s.id, w.converters)
if s.kind == skMethod and sfDispatcher notin s.flags:
if w.methods.len != 0: add(w.methods, ' ')
encodeVInt(s.id, w.methods)
elif iiTableGet(w.imports.tab, s.id) == InvalidKey:
addToIndex(w.imports, s.id, m.id)
when debugWrittenIds:
if not contains(debugWritten, s.id):
echo(w.filename)
debug(s)
debug(s.owner)
debug(m)
internalError("Symbol referred to but never written")
inc(i)
setLen(w.sstack, result)
proc typeStack(w: PRodWriter): int =
var i = 0
while i < len(w.tstack):
var t = w.tstack[i]
if t.kind == tyForward:
w.tstack[result] = t
inc result
elif iiTableGet(w.index.tab, t.id) == InvalidKey:
var L = w.data.len
addToIndex(w.index, t.id, L)
encodeType(w, t, w.data)
add(w.data, rodNL)
inc(i)
setLen(w.tstack, result)
proc processStacks(w: PRodWriter, finalPass: bool) =
var oldS = 0
var oldT = 0
while true:
var slen = symStack(w)
var tlen = typeStack(w)
if slen == oldS and tlen == oldT: break
oldS = slen
oldT = tlen
if finalPass and (oldS != 0 or oldT != 0):
internalError(w.config, "could not serialize some forwarded symbols/types")
proc rawAddInterfaceSym(w: PRodWriter, s: PSym) =
pushSym(w, s)
processStacks(w, false)
proc addInterfaceSym(w: PRodWriter, s: PSym) =
if w == nil: return
if s.kind in ExportableSymKinds and
{sfExported, sfCompilerProc} * s.flags != {}:
rawAddInterfaceSym(w, s)
proc addStmt(w: PRodWriter, n: PNode) =
encodeVInt(w.data.len, w.init)
add(w.init, rodNL)
encodeNode(w, unknownLineInfo(), n, w.data)
add(w.data, rodNL)
processStacks(w, false)
proc writeRod(w: PRodWriter) =
processStacks(w, true)
var f: File
if not open(f, completeGeneratedFilePath(w.config, changeFileExt(
withPackageName(w.config, w.filename), RodExt)),
fmWrite):
#echo "couldn't write rod file for: ", w.filename
return
# write header:
f.write("NIM:")
f.write(RodFileVersion)
f.write(rodNL)
var id = "ID:"
encodeVInt(w.module.id, id)
f.write(id)
f.write(rodNL)
var orig = "ORIGFILE:"
encodeStr(w.origFile, orig)
f.write(orig)
f.write(rodNL)
var hash = "HASH:"
encodeStr($w.hash, hash)
f.write(hash)
f.write(rodNL)
var options = "OPTIONS:"
encodeVInt(cast[int32](w.options), options)
f.write(options)
f.write(rodNL)
var goptions = "GOPTIONS:"
encodeVInt(cast[int32](w.config.globalOptions), goptions)
f.write(goptions)
f.write(rodNL)
var cmd = "CMD:"
encodeVInt(cast[int32](w.config.cmd), cmd)
f.write(cmd)
f.write(rodNL)
f.write("DEFINES:")
f.write(w.defines)
f.write(rodNL)
var files = "FILES(" & rodNL
for i in countup(0, high(w.files)):
encodeStr(w.files[i], files)
files.add(rodNL)
f.write(files)
f.write(')' & rodNL)
f.write("INCLUDES(" & rodNL)
f.write(w.inclDeps)
f.write(')' & rodNL)
f.write("DEPS:")
f.write(w.modDeps)
f.write(rodNL)
f.write("INTERF(" & rodNL)
f.write(w.interf)
f.write(')' & rodNL)
f.write("COMPILERPROCS(" & rodNL)
f.write(w.compilerProcs)
f.write(')' & rodNL)
f.write("INDEX(" & rodNL)
f.write(w.index.r)
f.write(')' & rodNL)
f.write("IMPORTS(" & rodNL)
f.write(w.imports.r)
f.write(')' & rodNL)
f.write("CONVERTERS:")
f.write(w.converters)
f.write(rodNL)
f.write("METHODS:")
f.write(w.methods)
f.write(rodNL)
f.write("INIT(" & rodNL)
f.write(w.init)
f.write(')' & rodNL)
f.write("DATA(" & rodNL)
f.write(w.data)
f.write(')' & rodNL)
# write trailing zero which is necessary because we use memory mapped files
# for reading:
f.write("\0")
f.close()
#echo "interf: ", w.interf.len
#echo "index: ", w.index.r.len
#echo "init: ", w.init.len
#echo "data: ", w.data.len
proc process(c: PPassContext, n: PNode): PNode =
result = n
if c == nil: return
var w = PRodWriter(c)
case n.kind
of nkStmtList:
for i in countup(0, sonsLen(n) - 1): discard process(c, n.sons[i])
#var s = n.sons[namePos].sym
#addInterfaceSym(w, s)
of nkProcDef, nkFuncDef, nkIteratorDef, nkConverterDef,
nkTemplateDef, nkMacroDef:
let s = n.sons[namePos].sym
if s == nil: internalError(w.config, n.info, "rodwrite.process")
if n.sons[bodyPos] == nil:
internalError(w.config, n.info, "rodwrite.process: body is nil")
if n.sons[bodyPos].kind != nkEmpty or s.magic != mNone or
sfForward notin s.flags:
addInterfaceSym(w, s)
of nkMethodDef:
let s = n.sons[namePos].sym
if s == nil: internalError(w.config, n.info, "rodwrite.process")
if n.sons[bodyPos] == nil:
internalError(w.config, n.info, "rodwrite.process: body is nil")
if n.sons[bodyPos].kind != nkEmpty or s.magic != mNone or
sfForward notin s.flags:
pushSym(w, s)
processStacks(w, false)
of nkVarSection, nkLetSection, nkConstSection:
for i in countup(0, sonsLen(n) - 1):
var a = n.sons[i]
if a.kind == nkCommentStmt: continue
addInterfaceSym(w, a.sons[0].sym)
of nkTypeSection:
for i in countup(0, sonsLen(n) - 1):
var a = n.sons[i]
if a.kind == nkCommentStmt: continue
if a.sons[0].kind != nkSym: internalError(w.config, a.info, "rodwrite.process")
var s = a.sons[0].sym
addInterfaceSym(w, s)
# this takes care of enum fields too
# Note: The check for ``s.typ.kind = tyEnum`` is wrong for enum
# type aliasing! Otherwise the same enum symbol would be included
# several times!
of nkImportStmt:
for i in countup(0, sonsLen(n) - 1):
addModDep(w, getModuleName(w.config, n.sons[i]), n.info)
addStmt(w, n)
of nkFromStmt, nkImportExceptStmt:
addModDep(w, getModuleName(w.config, n.sons[0]), n.info)
addStmt(w, n)
of nkIncludeStmt:
for i in countup(0, sonsLen(n) - 1):
addInclDep(w, getModuleName(w.config, n.sons[i]), n.info)
of nkPragma:
addStmt(w, n)
else:
discard
proc myOpen(g: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
if module.id < 0: internalError(g.config, "rodwrite: module ID not set")
var w = newRodWriter(rodread.getHash(g.config, FileIndex module.position), module, cache, g.config)
rawAddInterfaceSym(w, module)
result = w
proc myClose(graph: ModuleGraph; c: PPassContext, n: PNode): PNode =
result = process(c, n)
var w = PRodWriter(c)
writeRod(w)
idgen.saveMaxIds(graph.config, graph.config.projectPath / graph.config.projectName)
const rodwritePass* = makePass(open = myOpen, close = myClose, process = process)

9
compiler/sem.nim
View File

@@ -13,7 +13,7 @@ import
ast, strutils, hashes, options, lexer, astalgo, trees, treetab,
wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
magicsys, parser, nversion, nimsets, semfold, modulepaths, importer,
procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch,
intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity,
semparallel, lowerings, pluginsupport, plugins.active, rod, lineinfos
@@ -513,10 +513,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
result = c
proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext =
result = myOpen(graph, module, rd.cache)
proc replayMethodDefs(graph: ModuleGraph; rd: PRodReader) =
for m in items(rd.methods): methodDef(graph, m, true)
result = myOpen(graph, module, graph.cache)
proc isImportSystemStmt(g: ModuleGraph; n: PNode): bool =
if g.systemModule == nil: return false
@@ -623,8 +620,6 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
addCodeForGenerics(c, result)
if c.module.ast != nil:
result.add(c.module.ast)
if c.rd != nil:
replayMethodDefs(graph, c.rd)
popOwner(c)
popProcCon(c)
storeRemaining(c.graph, c.module)

2
compiler/semdata.nim
View File

@@ -13,7 +13,7 @@ import
strutils, intsets, options, lexer, ast, astalgo, trees, treetab,
wordrecg,
ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math,
magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef,
magicsys, nversion, nimsets, parser, times, passes, vmdef,
modulegraphs, lineinfos
type

4
compiler/transf.nim
View File

@@ -20,7 +20,7 @@
import
intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread,
idents, renderer, types, passes, semfold, magicsys, cgmeth,
lambdalifting, sempass2, lowerings, lookups, destroyer, liftlocals,
modulegraphs, lineinfos
@@ -916,7 +916,7 @@ proc processTransf(c: PTransf, n: PNode, owner: PSym): PNode =
# Note: For interactive mode we cannot call 'passes.skipCodegen' and skip
# this step! We have to rely that the semantic pass transforms too errornous
# nodes into an empty node.
if c.rd != nil or nfTransf in n.flags: return n
if nfTransf in n.flags: return n
pushTransCon(c, newTransCon(owner))
result = PNode(transform(c, n))
popTransCon(c)

2
compiler/vmgen.nim
View File

@@ -29,7 +29,7 @@
import
strutils, ast, astalgo, types, msgs, renderer, vmdef,
trees, intsets, rodread, magicsys, options, lowerings, lineinfos
trees, intsets, magicsys, options, lowerings, lineinfos
import platform
from os import splitFile