From cf4ac376439c5abc117e1fa1d735e1ba8024ad85 Mon Sep 17 00:00:00 2001 From: araq Date: Sun, 9 Nov 2025 09:50:11 +0100 Subject: [PATCH] progress --- compiler/ast2nif.nim | 107 +++++++++++----- compiler/icnif/enum2nif.nim | 16 +-- tools/enumgen.nim | 247 ++++++++++++++++++++++++++++++++++++ 3 files changed, 331 insertions(+), 39 deletions(-) create mode 100644 tools/enumgen.nim diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 1fe4863742..8e16f5da08 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -141,18 +141,30 @@ type ParsedSymName* = object name*: string module*: string + count*: int proc parseSymName*(s: string): ParsedSymName = var i = s.len - 2 while i > 0: if s[i] == '.': if s[i+1] in {'0'..'9'}: - return ParsedSymName(name: substr(s, 0, i-1), module: "") + var count = ord(s[i+1]) - ord('0') + var j = i+2 + while j < s.len and s[j] in {'0'..'9'}: + count = count * 10 + ord(s[j]) - ord('0') + inc j + return ParsedSymName(name: substr(s, 0, i-1), module: "", count: count) else: let mend = s.high var b = i-1 while b > 0 and s[b] != '.': dec b - return ParsedSymName(name: substr(s, 0, b-1), module: substr(s, i+1, mend)) + var j = b+1 + var count = 0 + while j < s.len and s[j] in {'0'..'9'}: + count = count * 10 + ord(s[j]) - ord('0') + inc j + + return ParsedSymName(name: substr(s, 0, b-1), module: substr(s, i+1, mend), count: count) dec i return ParsedSymName(name: s, module: "") @@ -460,7 +472,7 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = proc nodeKind(n: Cursor): TNodeKind {.inline.} = assert n.kind == ParLe - pool.tags[n.tagId].parseNodeKind() + parse(TNodeKind, pool.tags[n.tagId]) proc expect(n: Cursor; k: set[NifKind]) = if n.kind notin k: @@ -501,6 +513,14 @@ proc incExpectTag(n: var Cursor; tagId: TagId) = inc n expectTag(n, tagId) +proc parseBool(n: var Cursor): bool = + if n.kind == ParLe: + result = pool.tags[n.tagId] == "true" + inc n + skipParRi n + else: + raiseAssert "(true)/(false) expected" + type DecodeContext* = object infos: LineInfoWriter @@ -508,6 +528,7 @@ type types: Table[ItemId, (PType, NifIndexEntry)] syms: Table[ItemId, (PSym, NifIndexEntry)] indexes: seq[NifIndex] + symCounters: Table[int32, int32] # Module ID -> counter cache: IdentCache proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = @@ -545,6 +566,8 @@ proc fromNifNodeFlags(n: var Cursor): set[TNodeFlag] = else: raiseAssert "expected Node flag (`ident`) but got " & $n.kind +proc loadNode(c: var DecodeContext; n: var Cursor): PNode + proc loadTypeStub(c: var DecodeContext; t: SymId): PType = let name = pool.syms[t] assert name.startsWith("`t.") @@ -578,13 +601,18 @@ proc loadTypeStub(c: var DecodeContext; n: var Cursor): PType = raiseAssert "type expected but got " & $n.kind proc loadSymStub(c: var DecodeContext; t: SymId): PSym = - let name = parseSymName(pool.syms[t]) - let id = ItemId(module: moduleId(c, name.module), item: itemId) - result = c.types.getOrDefault(id)[0] + let symAsStr = pool.syms[t] + let sn = parseSymName(symAsStr) + let module = moduleId(c, sn.module) + let val = addr c.symCounters.mgetOrPut(module, 0) + inc val[] + + let id = ItemId(module: module, item: val[]) + result = c.syms.getOrDefault(id)[0] if result == nil: - let offs = c.getOffset(id.module, name) - result = PType(itemId: id, uniqueId: id, kind: tyStub) - c.types[id] = (result, offs) + let offs = c.getOffset(module, symAsStr) + result = PSym(itemId: id, kind: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32) + c.syms[id] = (result, offs) proc loadSymStub(c: var DecodeContext; n: var Cursor): PSym = if n.kind == DotToken: @@ -601,14 +629,15 @@ proc loadSymStub(c: var DecodeContext; n: var Cursor): PSym = else: raiseAssert "sym expected but got " & $n.kind -proc isStub*(t: PType): bool = t.kind == tyStub +proc isStub*(t: PType): bool {.inline.} = t.kind == tyStub +proc isStub*(s: PSym): bool {.inline.} = s.kind == skStub proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = expect n, Ident - loc.k = pool.strings[n.litId].parseLocKind() + loc.k = parse(TLocKind, pool.strings[n.litId]) inc n expect n, Ident - loc.storage = pool.strings[n.litId].parseStorageLoc() + loc.storage = parse(TStorageLoc, pool.strings[n.litId]) inc n expect n, Ident loc.flags = pool.strings[n.litId].parseLocFlags() @@ -617,10 +646,9 @@ proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = loc.snippet = pool.strings[n.litId] inc n -proc loadTypeBody(c: var DecodeContext; t: PType) = +proc loadType*(c: var DecodeContext; t: PType) = if t.kind != tyStub: return - assert t.size < 0, "type has no offset" - var n = getCursorAt() + var n = default(Cursor) # getCursorAt() expect n, ParLe if n.tagId != tdefTag: @@ -630,42 +658,59 @@ proc loadTypeBody(c: var DecodeContext; t: PType) = # ignore the type's name, we have already used it to create this PType's itemId! inc n expect n, Ident - t.kind = parseTypeKind(pool.strings[n.litId]) + t.kind = parse(TTypeKind, pool.strings[n.litId]) inc n expect n, Ident t.flags = parseTypeFlags(pool.strings[n.litId]) inc n expect n, Ident - t.callConv = parseCallingConvention(pool.strings[n.litId]) + t.callConv = parse(TCallingConvention, pool.strings[n.litId]) inc n expect n, IntLit - typ.size = pool.integers[n.intId] + t.size = pool.integers[n.intId] inc n expect n, IntLit - typ.align = pool.integers[n.intId] + t.align = pool.integers[n.intId].int16 inc n expect n, IntLit - typ.paddingAtEnd = pool.integers[n.intId] + t.paddingAtEnd = pool.integers[n.intId].int16 inc n expect n, IntLit - typ.itemId.item = pool.integers[n.intId] + t.itemId.item = pool.integers[n.intId].int32 inc n - loadTypeStub c, n, typ.typeInst - loadNode c, n, typ.n - loadSymStub c, n, typ.owner - loadSymStub c, n, typ.sym - loadLoc c, n, typ.loc + t.typeInst = loadTypeStub(c, n) + t.n = loadNode(c, n) + t.setOwner loadSymStub(c, n) + t.sym = loadSymStub(c, n) + loadLoc c, n, t.loc + var kids: seq[PType] = @[] while n.kind != ParRi: - t.typ.kids.add loadTypeStub(c, n) + kids.add loadTypeStub(c, n) + + t.setSons kids skipParRi n +proc loadSym*(c: var DecodeContext; s: PSym) = + if s.kind != skStub: return + var n = default(Cursor) # getCursorAt() + + expect n, ParLe + if n.tagId != sdefTag: + raiseAssert "(sd) expected" + inc n + expect n, SymbolDef + # ignore the symbol's name, we have already used it to create this PSym instance! + inc n + + + template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = let info = c.infos.oldLineInfo(n.info) let flags = fromNifNodeFlags n @@ -675,7 +720,7 @@ template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNod body skipParRi n -proc fromNif(c: var DecodeContext; n: var Cursor): PNode = +proc loadNode(c: var DecodeContext; n: var Cursor): PNode = result = nil case n.kind: of DotToken: @@ -752,7 +797,7 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode = else: c.withNode n, result, kind: while n.kind != ParRi: - result.addAllowNil c.fromNif n + result.addAllowNil c.loadNode n else: raiseAssert "Not yet implemented " & $n.kind @@ -768,6 +813,6 @@ proc loadNifModule*(config: ConfigRef; f: FileIndex): PNode = when isMainModule: import std / syncio let obj = parseSymName("a.123.sys") - echo obj.name, " ", obj.module + echo obj.name, " ", obj.module, " ", obj.count let objb = parseSymName("abcdef.0121") - echo objb.name, " ", objb.module + echo objb.name, " ", objb.module, " ", objb.count diff --git a/compiler/icnif/enum2nif.nim b/compiler/icnif/enum2nif.nim index 20e6217855..d11dd04b6a 100644 --- a/compiler/icnif/enum2nif.nim +++ b/compiler/icnif/enum2nif.nim @@ -1,4 +1,4 @@ -# Generated by gear2/generator/enumgen.nim in Nimony repo. DO NOT EDIT! +# Generated by tools/enumgen.nim. DO NOT EDIT! import ".." / [ast, options] @@ -172,7 +172,7 @@ proc toNifTag*(s: TNodeKind): string = of nkOpenSym: "opensym" -proc parseNodeKind*(s: string): TNodeKind = +proc parse*(t: typedesc[TNodeKind]; s: string): TNodeKind = case s of "none": nkNone of "empty": nkEmpty @@ -372,7 +372,7 @@ proc toNifTag*(s: TSymKind): string = of skPackage: "package" -proc parseSymKind*(s: string): TSymKind = +proc parse*(t: typedesc[TSymKind]; s: string): TSymKind = case s of "unknown": skUnknown of "conditional": skConditional @@ -472,7 +472,7 @@ proc toNifTag*(s: TTypeKind): string = of tyStub: "stub" -proc parseTypeKind*(s: string): TTypeKind = +proc parse*(t: typedesc[TTypeKind]; s: string): TTypeKind = case s of "none": tyNone of "bool": tyBool @@ -558,7 +558,7 @@ proc toNifTag*(s: TLocKind): string = of locOther: "other" -proc parseLocKind*(s: string): TLocKind = +proc parse*(t: typedesc[TLocKind]; s: string): TLocKind = case s of "none": locNone of "temp": locTemp @@ -590,7 +590,7 @@ proc toNifTag*(s: TCallingConvention): string = of ccMember: "member" -proc parseCallingConvention*(s: string): TCallingConvention = +proc parse*(t: typedesc[TCallingConvention]; s: string): TCallingConvention = case s of "nimcall": ccNimCall of "stdcall": ccStdCall @@ -889,7 +889,7 @@ proc toNifTag*(s: TMagic): string = of mZeroDefault: "zerodefault" -proc parseMagic*(s: string): TMagic = +proc parse*(t: typedesc[TMagic]; s: string): TMagic = case s of "nonem": mNone of "defined": mDefined @@ -1180,7 +1180,7 @@ proc toNifTag*(s: TStorageLoc): string = of OnHeap: "heap" -proc parseStorageLoc*(s: string): TStorageLoc = +proc parse*(t: typedesc[TStorageLoc]; s: string): TStorageLoc = case s of "unknown": OnUnknown of "static": OnStatic diff --git a/tools/enumgen.nim b/tools/enumgen.nim new file mode 100644 index 0000000000..37493a99cc --- /dev/null +++ b/tools/enumgen.nim @@ -0,0 +1,247 @@ +## Generate effective NIF representation for `Enum` + +import ".." / compiler / [ast, options] + +import std / [syncio, assertions, strutils, tables] + +# We need to duplicate this type here as ast.nim's version of it does not work +# as it sets the string values explicitly breaking our logic... +type + TCallingConventionMirror = enum + ccNimCall + ccStdCall + ccCDecl + ccSafeCall + ccSysCall + ccInline + ccNoInline + ccFastCall + ccThisCall + ccClosure + ccNoConvention + ccMember + +const + SpecialCases = [ + ("nkCommand", "cmd"), + ("nkIfStmt", "if"), + ("nkError", "err"), + ("nkType", "onlytype"), + ("nkTypeSection", "type"), + ("tySequence", "seq"), + ("tyVar", "mut"), + ("tyProc", "proctype"), + ("tyUncheckedArray", "uarray"), + ("nkExprEqExpr", "vv"), + ("nkExprColonExpr", "kv"), + ("nkDerefExpr", "deref"), + ("nkReturnStmt", "ret"), + ("nkBreakStmt", "brk"), + ("nkStmtListExpr", "expr"), + ("nkEnumFieldDef", "efld"), + ("nkNilLit", "nil"), + ("ccNoConvention", "noconv"), + ("mExpr", "exprm"), + ("mStmt", "stmtm"), + ("mEqNimrodNode", "eqnimnode"), + ("mPNimrodNode", "nimnode"), + ("mNone", "nonem"), + ("mAsgn", "asgnm"), + ("mOf", "ofm"), + ("mAddr", "addrm"), + ("mType", "typem"), + ("mStatic", "staticm"), + ("mRange", "rangem"), + ("mVar", "varm"), + ("mInSet", "contains"), + ("mNil", "nilm"), + ("tyBuiltInTypeClass", "bconcept"), + ("tyUserTypeClass", "uconcept"), + ("tyUserTypeClassInst", "uconceptinst"), + ("tyCompositeTypeClass", "cconcept"), + ("tyGenericInvocation", "ginvoke"), + ("tyGenericBody", "gbody"), + ("tyGenericInst", "ginst"), + ("tyGenericParam", "gparam"), + ("nkStmtList", "stmts"), + ("nkDotExpr", "dot"), + ("nkBracketExpr", "at") + ] + SuffixesToReplace = [ + ("Section", ""), ("Branch", ""), ("Stmt", ""), ("I", ""), + ("Expr", "x"), ("Def", "") + ] + PrefixesToReplace = [ + ("Length", "len"), + ("SetLength", "setlen"), + ("Append", "add") + ] + AdditionalNodes = [ + "nf", # "node flag" + "tf", # "type flag" + "sf", # "sym flag" + "htype", # annotated with a hidden type + "missing" + ] + +proc genEnum[E](f: var File; enumName: string; known: var OrderedTable[string, bool]; prefixLen = 2) = + var mappingA = initOrderedTable[string, E]() + var cases = "" + for e in low(E)..high(E): + var es = $e + if es.startsWith("nkHidden"): + es = es.replace("nkHidden", "nkh") # prefix will be removed + else: + for (suffix, repl) in items SuffixesToReplace: + if es.len - prefixLen > suffix.len and es.endsWith(suffix): + es.setLen es.len - len(suffix) + es.add repl + break + for (suffix, repl) in items PrefixesToReplace: + if es.len - prefixLen > suffix.len and es.substr(prefixLen).startsWith(suffix): + es = es.substr(0, prefixLen-1) & repl & es.substr(prefixLen+suffix.len) + break + + let s = es.substr(prefixLen) + var done = false + for enu, key in items SpecialCases: + if $e == enu: + assert(not mappingA.hasKey(key)) + if known.hasKey(key): echo "conflict: ", key + known[key] = true + assert key.len > 0 + mappingA[key] = e + cases.add " of " & $e & ": " & escape(key) & "\n" + done = true + break + if not done: + let key = s.toLowerAscii + if not mappingA.hasKey(key): + assert key.len > 0, $e + if known.hasKey(key): echo "conflict: ", key + known[key] = true + mappingA[key] = e + cases.add " of " & $e & ": " & escape(key) & "\n" + done = true + if not done: + var d = 0 + while d < 10: + let key = s.toLowerAscii & $d + if not mappingA.hasKey(key): + assert key.len > 0 + mappingA[key] = e + cases.add " of " & $e & ": " & escape(key) & "\n" + done = true + break + inc d + if not done: + echo "Could not map: " & s + #echo mapping + var code = "" + code.add "proc toNifTag*(s: " & enumName & "): string =\n" + code.add " case s\n" + code.add cases + code.add "\n\n" + let procname = "parse" # & enumName.substr(1) + code.add "proc " & procname & "*(t: typedesc[" & enumName & "]; s: string): " & enumName & " =\n" + code.add " case s\n" + for (k, v) in pairs mappingA: + code.add " of " & escape(k) & ": " & $v & "\n" + code.add " else: " & $low(E) & "\n\n\n" + f.write code + +proc genEnum[E](f: var File; enumName: string; prefixLen = 2) = + var known = initOrderedTable[string, bool]() + genEnum[E](f, enumName, known, prefixLen) + + +proc genFlags[E](f: var File; enumName: string; prefixLen = 2) = + var mappingA = initOrderedTable[string, E]() + var mappingB = initOrderedTable[string, E]() + var cases = "" + for e in low(E)..high(E): + let s = ($e).substr(prefixLen) + var done = false + for c in s: + if c in {'A'..'Z'}: + let key = $c.toLowerAscii + if not mappingA.hasKey(key): + mappingA[key] = e + cases.add " of " & $e & ": dest.add " & escape(key) & "\n" + done = true + break + if not done: + var d = 0 + while d < 10: + let key = $s[0].toLowerAscii & $d + if not mappingB.hasKey(key): + mappingB[key] = e + cases.add " of " & $e & ": dest.add " & escape(key) & "\n" + done = true + break + inc d + if not done: + quit "Could not map: " & s + #echo mapping + var code = "" + code.add "proc genFlags*(s: set[" & enumName & "]; dest: var string) =\n" + code.add " for e in s:\n" + code.add " case e\n" + code.add cases + code.add "\n\n" + let procname = "parse" & enumName.substr(1) & "s" + code.add "proc " & procname & "*(s: string): set[" & enumName & "] =\n" + code.add " result = {}\n" + code.add " var i = 0\n" + code.add " while i < s.len:\n" + code.add " case s[i]\n" + for c in 'a'..'z': + var letterFound = false + var digitsFound = 0 + for d in '0'..'9': + if mappingB.hasKey($c & $d): + if not letterFound: + letterFound = true + code.add " of '" & c & "':\n" + if digitsFound == 0: + code.add " if" + else: + code.add " elif" + inc digitsFound + code.add " i+1 < s.len and s[i+1] == '" & d & "':\n" + code.add " result.incl " & $mappingB[$c & $d] & "\n" + code.add " inc i\n" + + if mappingA.hasKey($c): + if digitsFound == 0: + code.add " of '" & c & "': " + else: + code.add " else: " + code.add "result.incl " & $mappingA[$c] & "\n" + + code.add " else: discard\n" + code.add " inc i\n\n" + f.write code + +var f = open("compiler/icnif/enum2nif.nim", fmWrite) +f.write "# Generated by tools/enumgen.nim. DO NOT EDIT!\n\n" +f.write "import \"..\" / [ast, options]\n\n" +# use the same mapping for TNodeKind and TMagic so that we can detect conflicts! +var nodeTags = initOrderedTable[string, bool]() +for a in AdditionalNodes: + nodeTags[a] = true + +genEnum[TNodeKind](f, "TNodeKind", nodeTags) +genEnum[TSymKind](f, "TSymKind") +genEnum[TTypeKind](f, "TTypeKind") +genEnum[TLocKind](f, "TLocKind", 3) +genEnum[TCallingConventionMirror](f, "TCallingConvention", 2) +genEnum[TMagic](f, "TMagic", nodeTags, 1) +genEnum[TStorageLoc](f, "TStorageLoc") +genFlags[TSymFlag](f, "TSymFlag") +genFlags[TNodeFlag](f, "TNodeFlag") +genFlags[TTypeFlag](f, "TTypeFlag") +genFlags[TLocFlag](f, "TLocFlag") +genFlags[TOption](f, "TOption", 3) + +f.close()