diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim index ce64221010..49669e4e2c 100644 --- a/compiler/ic/dce.nim +++ b/compiler/ic/dce.nim @@ -112,8 +112,8 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N followLater(c, g, c.thisModule, n.operand) of nkModuleRef: let (n1, n2) = sons2(tree, n) - assert n1.kind == nkInt32Lit - assert n2.kind == nkInt32Lit + assert n1.kind == nkNone + assert n2.kind == nkNone let m = n1.litId let item = n2.operand let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 4d02822b2e..1b52c82b0e 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -16,6 +16,8 @@ from std/os import removeFile, isAbsolute import ../../dist/checksums/src/checksums/sha1 +import ".." / nir / nirlineinfos + when defined(nimPreviewSlimSystem): import std/[syncio, assertions, formatfloat] @@ -60,6 +62,7 @@ type strings*: BiTable[string] # we could share these between modules. numbers*: BiTable[BiggestInt] # we also store floats in here so # that we can assure that every bit is kept + man*: LineInfoManager cfg: PackedConfig @@ -75,37 +78,36 @@ type symMarker*: IntSet #Table[ItemId, SymId] # ItemId.item -> SymId config*: ConfigRef -proc toString*(tree: PackedTree; n: NodePos; m: PackedModule; nesting: int; +proc toString*(tree: PackedTree; pos: NodePos; m: PackedModule; nesting: int; result: var string) = - let pos = n.int if result.len > 0 and result[^1] notin {' ', '\n'}: result.add ' ' result.add $tree[pos].kind - case tree.nodes[pos].kind - of nkNone, nkEmpty, nkNilLit, nkType: discard + case tree[pos].kind + of nkEmpty, nkNilLit, nkType: discard of nkIdent, nkStrLit..nkTripleStrLit: result.add " " - result.add m.strings[LitId tree.nodes[pos].operand] + result.add m.strings[LitId tree[pos].operand] of nkSym: result.add " " - result.add m.strings[m.syms[tree.nodes[pos].operand].name] + result.add m.strings[m.syms[tree[pos].operand].name] of directIntLit: result.add " " - result.addInt tree.nodes[pos].operand + result.addInt tree[pos].operand of externSIntLit: result.add " " - result.addInt m.numbers[LitId tree.nodes[pos].operand] + result.addInt m.numbers[LitId tree[pos].operand] of externUIntLit: result.add " " - result.addInt cast[uint64](m.numbers[LitId tree.nodes[pos].operand]) + result.addInt cast[uint64](m.numbers[LitId tree[pos].operand]) of nkFloatLit..nkFloat128Lit: result.add " " - result.addFloat cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand]) + result.addFloat cast[BiggestFloat](m.numbers[LitId tree[pos].operand]) else: result.add "(\n" for i in 1..(nesting+1)*2: result.add ' ' - for child in sonsReadonly(tree, n): + for child in sonsReadonly(tree, pos): toString(tree, child, m, nesting + 1, result) result.add "\n" for i in 1..nesting*2: result.add ' ' @@ -284,7 +286,8 @@ proc toLitId(x: BiggestInt; m: var PackedModule): LitId = result = getOrIncl(m.numbers, x) proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo = - PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m)) + pack(m.man, toLitId(x.fileIndex, c, m), x.line.int32, x.col.int32) + #PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m)) proc safeItemId(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId {.inline.} = ## given a symbol, produce an ItemId with the correct properties @@ -421,53 +424,49 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = ## add a remote symbol reference to the tree let info = n.info.toPackedInfo(c, m) - ir.nodes.add PackedNode(kind: nkModuleRef, operand: 3.int32, # spans 3 nodes in total - typeId: storeTypeLater(n.typ, c, m), info: info) - ir.nodes.add PackedNode(kind: nkInt32Lit, info: info, - operand: toLitId(n.sym.itemId.module.FileIndex, c, m).int32) - ir.nodes.add PackedNode(kind: nkInt32Lit, info: info, - operand: n.sym.itemId.item) + ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total + typeId = storeTypeLater(n.typ, c, m), info = info) + ir.addNode(kind = nkNone, info = info, + operand = toLitId(n.sym.itemId.module.FileIndex, c, m).int32) + ir.addNode(kind = nkNone, info = info, + operand = n.sym.itemId.item) proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = ## serialize a node into the tree if n == nil: - ir.nodes.add PackedNode(kind: nkNilRodNode, flags: {}, operand: 1) + ir.addNode(kind = nkNilRodNode, operand = 1, info = NoLineInfo) return let info = toPackedInfo(n.info, c, m) case n.kind of nkNone, nkEmpty, nkNilLit, nkType: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, operand: 0, - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, operand = 0, + typeId = storeTypeLater(n.typ, c, m), info = info) of nkIdent: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.strings, n.ident.s), - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.strings, n.ident.s), + typeId = storeTypeLater(n.typ, c, m), info = info) of nkSym: if n.sym.itemId.module == c.thisModule: # it is a symbol that belongs to the module we're currently # packing: let id = n.sym.storeSymLater(c, m).item - ir.nodes.add PackedNode(kind: nkSym, flags: n.flags, operand: id, - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = nkSym, flags = n.flags, operand = id, + typeId = storeTypeLater(n.typ, c, m), info = info) else: # store it as an external module reference: addModuleRef(n, ir, c, m) - of directIntLit: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32(n.intVal), - typeId: storeTypeLater(n.typ, c, m), info: info) of externIntLit: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.numbers, n.intVal), - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.numbers, n.intVal), + typeId = storeTypeLater(n.typ, c, m), info = info) of nkStrLit..nkTripleStrLit: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.strings, n.strVal), - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.strings, n.strVal), + typeId = storeTypeLater(n.typ, c, m), info = info) of nkFloatLit..nkFloat128Lit: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)), - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)), + typeId = storeTypeLater(n.typ, c, m), info = info) else: let patchPos = ir.prepare(n.kind, n.flags, storeTypeLater(n.typ, c, m), info) @@ -492,8 +491,8 @@ proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var # do not serialize the body of the proc, it's unnecessary since # n[0].sym.ast has the sem'checked variant of it which is what # everybody should use instead. - ir.nodes.add PackedNode(kind: nkEmpty, flags: {}, operand: 0, - typeId: nilItemId, info: info) + ir.addNode(kind = nkEmpty, flags = {}, operand = 0, + typeId = nilItemId, info = info) ir.patch patchPos proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModule) = @@ -609,9 +608,9 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef loadSeqSection methodsSection, m.methods loadSeqSection pureEnumsSection, m.pureEnums - loadSeqSection toReplaySection, m.toReplay.nodes - loadSeqSection topLevelSection, m.topLevel.nodes - loadSeqSection bodiesSection, m.bodies.nodes + loadTabSection toReplaySection, m.toReplay + loadTabSection topLevelSection, m.topLevel + loadTabSection bodiesSection, m.bodies loadSeqSection symsSection, m.syms loadSeqSection typesSection, m.types @@ -625,6 +624,9 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef f.loadSection backendFlagsSection f.loadPrim m.backendFlags + f.loadSection sideChannelSection + f.load m.man + close(f) result = f.err @@ -672,10 +674,10 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac storeSeqSection methodsSection, m.methods storeSeqSection pureEnumsSection, m.pureEnums - storeSeqSection toReplaySection, m.toReplay.nodes - storeSeqSection topLevelSection, m.topLevel.nodes + storeTabSection toReplaySection, m.toReplay + storeTabSection topLevelSection, m.topLevel - storeSeqSection bodiesSection, m.bodies.nodes + storeTabSection bodiesSection, m.bodies storeSeqSection symsSection, m.syms storeSeqSection typesSection, m.types @@ -690,6 +692,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac f.storeSection backendFlagsSection f.storePrim m.backendFlags + f.storeSection sideChannelSection + f.store m.man + close(f) encoder.disable() if f.err != ok: @@ -748,8 +753,9 @@ proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule: proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; x: PackedLineInfo): TLineInfo = assert g[thisModule].status in {loaded, storing, stored} - result = TLineInfo(line: x.line, col: x.col, - fileIndex: toFileIndexCached(c, g, thisModule, x.file)) + let (fileId, line, col) = unpack(g[thisModule].fromDisk.man, x) + result = TLineInfo(line: line.uint16, col: col.int16, + fileIndex: toFileIndexCached(c, g, thisModule, fileId)) proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; tree: PackedTree; n: NodePos): PNode = @@ -768,9 +774,9 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; of nkIdent: result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId]) of nkSym: - result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand)) + result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree[n].operand)) of directIntLit: - result.intVal = tree.nodes[n.int].operand + result.intVal = tree[n].operand of externIntLit: result.intVal = g[thisModule].fromDisk.numbers[n.litId] of nkStrLit..nkTripleStrLit: @@ -779,10 +785,10 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.numbers[n.litId]) of nkModuleRef: let (n1, n2) = sons2(tree, n) - assert n1.kind == nkInt32Lit - assert n2.kind == nkInt32Lit + assert n1.kind == nkNone + assert n2.kind == nkNone transitionNoneToSym(result) - result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)) + result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree[n2].operand)) else: for n0 in sonsReadonly(tree, n): result.addAllowNil loadNodes(c, g, thisModule, tree, n0) @@ -1239,5 +1245,5 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = echo " local ID: ", i, " kind ", m.syms[i].kind echo "symbols: ", m.syms.len, " types: ", m.types.len, - " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len, + " top level nodes: ", m.topLevel.len, " other nodes: ", m.bodies.len, " strings: ", m.strings.len, " numbers: ", m.numbers.len diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim index ed87ae59db..00014f15a3 100644 --- a/compiler/ic/integrity.nim +++ b/compiler/ic/integrity.nim @@ -72,15 +72,15 @@ proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) = c.thisModule = oldThisModule proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) = - if tree[n.int].typeId != nilItemId: - checkType(c, tree[n.int].typeId) + if tree[n].typeId != nilItemId: + checkType(c, tree[n].typeId) case n.kind of nkEmpty, nkNilLit, nkType, nkNilRodNode: discard of nkIdent: assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId of nkSym: - checkLocalSym(c, tree.nodes[n.int].operand) + checkLocalSym(c, tree[n].operand) of directIntLit: discard of externIntLit, nkFloatLit..nkFloat128Lit: @@ -89,9 +89,9 @@ proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) = assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId of nkModuleRef: let (n1, n2) = sons2(tree, n) - assert n1.kind == nkInt32Lit - assert n2.kind == nkInt32Lit - checkForeignSym(c, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)) + assert n1.kind == nkNone + assert n2.kind == nkNone + checkForeignSym(c, PackedItemId(module: n1.litId, item: tree[n2].operand)) else: for n0 in sonsReadonly(tree, n): checkNode(c, tree, n0) diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim index aea5e12e7f..c2d7ee4ade 100644 --- a/compiler/ic/navigator.nim +++ b/compiler/ic/navigator.nim @@ -20,20 +20,25 @@ when defined(nimPreviewSlimSystem): import std/assertions import ".." / [ast, modulegraphs, msgs, options] +import ".." / nir / nirlineinfos import packed_ast, bitabs, ic type + UnpackedLineInfo = object + file: LitId + line, col: int NavContext = object g: ModuleGraph thisModule: int32 - trackPos: PackedLineInfo + trackPos: UnpackedLineInfo alreadyEmitted: HashSet[string] outputSep: char # for easier testing, use short filenames and spaces instead of tabs. -proc isTracked(current, trackPos: PackedLineInfo, tokenLen: int): bool = - if current.file == trackPos.file and current.line == trackPos.line: +proc isTracked(man: LineInfoManager; current: PackedLineInfo, trackPos: UnpackedLineInfo, tokenLen: int): bool = + let (currentFile, currentLine, currentCol) = man.unpack(current) + if currentFile == trackPos.file and currentLine == trackPos.line: let col = trackPos.col - if col >= current.col and col < current.col+tokenLen: + if col >= currentCol and col < currentCol+tokenLen: result = true else: result = false @@ -42,32 +47,34 @@ proc isTracked(current, trackPos: PackedLineInfo, tokenLen: int): bool = proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool = result = s.name != LitId(0) and - isTracked(info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len) + isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len) proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool = let name = c.g.packed[s.module].fromDisk.syms[s.item].name result = name != LitId(0) and - isTracked(info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len) + isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len) const EmptyItemId = ItemId(module: -1'i32, item: -1'i32) proc search(c: var NavContext; tree: PackedTree): ItemId = # We use the linear representation here directly: - for i in 0..high(tree.nodes): - case tree.nodes[i].kind + for i in 0..= 0 and tree[n.int].kind in declarativeNodes + result = n.int >= 0 and tree[n].kind in declarativeNodes proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) = + let (fileId, line, col) = unpack(c.g.packed[c.thisModule].fromDisk.man, info) var m = "" - var file = c.g.packed[c.thisModule].fromDisk.strings[info.file] + var file = c.g.packed[c.thisModule].fromDisk.strings[fileId] if c.outputSep == ' ': file = os.extractFilename file - toLocation(m, file, info.line.int, info.col.int + ColOffset) + toLocation(m, file, line, col + ColOffset) if not c.alreadyEmitted.containsOrIncl(m): msgWriteln c.g.config, (if isDecl: "def" else: "usage") & c.outputSep & m proc list(c: var NavContext; tree: PackedTree; sym: ItemId) = - for i in 0..high(tree.nodes): - case tree.nodes[i].kind + for i in 0.. nkNilLit: @@ -357,16 +320,17 @@ const nkIntLit, nkInt8Lit, nkInt16Lit, + nkInt32Lit, nkInt64Lit, nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, - nkUInt64Lit} # nkInt32Lit is missing by design! + nkUInt64Lit} - externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt64Lit} + externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt32Lit, nkInt64Lit} externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit} - directIntLit* = nkInt32Lit + directIntLit* = nkNone when false: proc identIdImpl(tree: PackedTree; n: NodePos): LitId = @@ -420,3 +384,9 @@ iterator allNodes*(tree: PackedTree): NodePos = proc toPackedItemId*(item: int32): PackedItemId {.inline.} = PackedItemId(module: LitId(0), item: item) + +proc load*(f: var RodFile; t: var PackedTree) = + loadSeq f, t.nodes + +proc store*(f: var RodFile; t: PackedTree) = + storeSeq f, t.nodes diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index be5095fbb7..968bf255f4 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -112,7 +112,7 @@ type # better than exceptions. const - RodVersion = 1 + RodVersion = 2 defaultCookie = [byte(0), byte('R'), byte('O'), byte('D'), byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)] diff --git a/compiler/nir/nirlineinfos.nim b/compiler/nir/nirlineinfos.nim index 5f5d550868..f11ef7c427 100644 --- a/compiler/nir/nirlineinfos.nim +++ b/compiler/nir/nirlineinfos.nim @@ -42,6 +42,9 @@ type LineInfoManager* = object aside: seq[(LitId, int32, int32)] +const + NoLineInfo* = PackedLineInfo(0'u32) + proc pack*(m: var LineInfoManager; file: LitId; line, col: int32): PackedLineInfo = if file.uint32 <= FileMax.uint32 and line <= LineMax and col <= ColMax: let col = if col < 0'i32: 0'u32 else: col.uint32