retain postfix node in type section typed AST, with docgen fix (#23101)

Continued from #23096 which was reverted due to breaking a package and
failing docgen tests. Docgen should now work, but ~~a PR is still
pending for the package: https://github.com/SciNim/Unchained/pull/45~~
has been merged

compiler/docgen.nim

@@ -1031,7 +1031,7 @@ proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol =
     if genNode != nil:
       var literal = ""
       var r: TSrcGen = initTokRender(genNode, {renderNoBody, renderNoComments,
-        renderNoPragmas, renderNoProcDefs, renderExpandUsing})
+        renderNoPragmas, renderNoProcDefs, renderExpandUsing, renderNoPostfix})
       var kind = tkEof
       while true:
         getNextTok(r, kind, literal)
@@ -1059,7 +1059,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonEx
 
   # Obtain the plain rendered string for hyperlink titles.
   var r: TSrcGen = initTokRender(n, {renderNoBody, renderNoComments, renderDocComments,
-    renderNoPragmas, renderNoProcDefs, renderExpandUsing})
+    renderNoPragmas, renderNoProcDefs, renderExpandUsing, renderNoPostfix})
   while true:
     getNextTok(r, kind, literal)
     if kind == tkEof:
@@ -1086,6 +1086,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonEx
     symbolOrIdEnc = encodeUrl(symbolOrId, usePlus = false)
     deprecationMsg = genDeprecationMsg(d, pragmaNode)
     rstLangSymbol = toLangSymbol(k, n, cleanPlainSymbol)
+    symNameNode =
+      if nameNode.kind == nkPostfix: nameNode[1]
+      else: nameNode
 
   # we generate anchors automatically for subsequent use in doc comments
   let lineinfo = rstast.TLineInfo(
@@ -1096,10 +1099,10 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonEx
                priority = symbolPriority(k), info = lineinfo,
                module = addRstFileIndex(d, FileIndex d.module.position))
 
-  let renderFlags =
-    if nonExports: {renderNoBody, renderNoComments, renderDocComments, renderSyms,
-      renderExpandUsing, renderNonExportedFields}
-    else: {renderNoBody, renderNoComments, renderDocComments, renderSyms, renderExpandUsing}
+  var renderFlags = {renderNoBody, renderNoComments, renderDocComments,
+    renderSyms, renderExpandUsing, renderNoPostfix}
+  if nonExports:
+    renderFlags.incl renderNonExportedFields
   nodeToHighlightedHtml(d, n, result, renderFlags, symbolOrIdEnc)
 
   let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
@@ -1122,18 +1125,19 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonEx
   let external = d.destFile.AbsoluteFile.relativeTo(d.conf.outDir, '/').changeFileExt(HtmlExt).string
 
   var attype = ""
-  if k in routineKinds and nameNode.kind == nkSym:
+  if k in routineKinds and symNameNode.kind == nkSym:
     let att = attachToType(d, nameNode.sym)
     if att != nil:
       attype = esc(d.target, att.name.s)
-  elif k == skType and nameNode.kind == nkSym and nameNode.sym.typ.kind in {tyEnum, tyBool}:
-    let etyp = nameNode.sym.typ
+  elif k == skType and symNameNode.kind == nkSym and
+      symNameNode.sym.typ.kind in {tyEnum, tyBool}:
+    let etyp = symNameNode.sym.typ
     for e in etyp.n:
       if e.sym.kind != skEnumField: continue
       let plain = renderPlainSymbolName(e)
       let symbolOrId = d.newUniquePlainSymbol(plain)
       setIndexTerm(d[], ieNim, htmlFile = external, id = symbolOrId,
-                   term = plain, linkTitle = nameNode.sym.name.s & '.' & plain,
+                   term = plain, linkTitle = symNameNode.sym.name.s & '.' & plain,
                    linkDesc = xmltree.escape(getPlainDocstring(e).docstringSummary),
                    line = n.info.line.int)
 
@@ -1154,8 +1158,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonEx
                linkTitle = detailedName,
                linkDesc = xmltree.escape(plainDocstring.docstringSummary),
                line = n.info.line.int)
-  if k == skType and nameNode.kind == nkSym:
-    d.types.strTableAdd nameNode.sym
+  if k == skType and symNameNode.kind == nkSym:
+    d.types.strTableAdd symNameNode.sym
 
 proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind, nonExports = false): JsonItem =
   if not isVisible(d, nameNode): return
@@ -1163,7 +1167,8 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind, nonExports = false):
     name = getNameEsc(d, nameNode)
     comm = genRecComment(d, n)
     r: TSrcGen
-    renderFlags = {renderNoBody, renderNoComments, renderDocComments, renderExpandUsing}
+    renderFlags = {renderNoBody, renderNoComments, renderDocComments,
+      renderExpandUsing, renderNoPostfix}
   if nonExports:
     renderFlags.incl renderNonExportedFields
   r = initTokRender(n, renderFlags)

compiler/renderer.nim

@@ -25,7 +25,7 @@ type
   TRenderFlag* = enum
     renderNone, renderNoBody, renderNoComments, renderDocComments,
     renderNoPragmas, renderIds, renderNoProcDefs, renderSyms, renderRunnableExamples,
-    renderIr, renderNonExportedFields, renderExpandUsing
+    renderIr, renderNonExportedFields, renderExpandUsing, renderNoPostfix
 
   TRenderFlags* = set[TRenderFlag]
   TRenderTok* = object
@@ -546,7 +546,11 @@ proc lsub(g: TSrcGen; n: PNode): int =
   of nkInfix: result = lsons(g, n) + 2
   of nkPrefix:
     result = lsons(g, n)+1+(if n.len > 0 and n[1].kind == nkInfix: 2 else: 0)
-  of nkPostfix: result = lsons(g, n)
+  of nkPostfix:
+    if renderNoPostfix notin g.flags:
+      result = lsons(g, n)
+    else:
+      result = lsub(g, n[1])
   of nkCallStrLit: result = lsons(g, n)
   of nkPragmaExpr: result = lsub(g, n[0]) + lcomma(g, n, 1)
   of nkRange: result = lsons(g, n) + 2
@@ -1330,14 +1334,20 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
     put(g, tkColon, ":")
     gsub(g, n, bodyPos)
   of nkIdentDefs:
-    # Skip if this is a property in a type and its not exported
-    # (While also not allowing rendering of non exported fields)
-    if ObjectDef in g.inside and (not n[0].isExported() and renderNonExportedFields notin g.flags):
-      return
+    var exclFlags: TRenderFlags = {}
+    if ObjectDef in g.inside:
+      if not n[0].isExported() and renderNonExportedFields notin g.flags:
+        # Skip if this is a property in a type and its not exported
+        # (While also not allowing rendering of non exported fields)
+        return
+      # render postfix for object fields:
+      exclFlags = g.flags * {renderNoPostfix}
     # We render the identDef without being inside the section incase we render something like
     # y: proc (x: string) # (We wouldn't want to check if x is exported)
     g.outside(ObjectDef):
+      g.flags.excl(exclFlags)
       gcomma(g, n, 0, -3)
+      g.flags.incl(exclFlags)
       if n.len >= 2 and n[^2].kind != nkEmpty:
         putWithSpace(g, tkColon, ":")
         gsub(g, n[^2], c)
@@ -1416,7 +1426,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
       postStatements(g, n, i, fromStmtList)
   of nkPostfix:
     gsub(g, n, 1)
-    gsub(g, n, 0)
+    if renderNoPostfix notin g.flags:
+      gsub(g, n, 0)
   of nkRange:
     gsub(g, n, 0)
     put(g, tkDotDot, "..")

compiler/semstmts.nim

@@ -1259,6 +1259,9 @@ proc typeSectionTypeName(c: PContext; n: PNode): PNode =
     result = n[0]
   else:
     result = n
+  if result.kind == nkPostfix:
+    if result.len != 2: illFormedAst(n, c.config)
+    result = result[1]
   if result.kind != nkSym: illFormedAst(n, c.config)
 
 proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) =
@@ -1326,9 +1329,15 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) =
     elif s.owner == nil: s.owner = getCurrOwner(c)
 
   if name.kind == nkPragmaExpr:
-    typeDef[0][0] = newSymNode(s)
+    if name[0].kind == nkPostfix:
+      typeDef[0][0][1] = newSymNode(s)
+    else:
+      typeDef[0][0] = newSymNode(s)
   else:
-    typeDef[0] = newSymNode(s)
+    if name.kind == nkPostfix:
+      typeDef[0][1] = newSymNode(s)
+    else:
+      typeDef[0] = newSymNode(s)
 
 proc typeSectionLeftSidePass(c: PContext, n: PNode) =
   # process the symbols on the left side for the whole type section, before
@@ -1538,8 +1547,15 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
       of nkSym: obj.ast[0] = symNode
       of nkPragmaExpr:
         obj.ast[0] = a[0].shallowCopy
-        obj.ast[0][0] = symNode
+        if a[0][0].kind == nkPostfix:
+          obj.ast[0][0] = a[0][0].shallowCopy
+          obj.ast[0][0][1] = symNode
+        else:
+          obj.ast[0][0] = symNode
         obj.ast[0][1] = a[0][1]
+      of nkPostfix:
+        obj.ast[0] = a[0].shallowCopy
+        obj.ast[0][1] = symNode
       else: assert(false)
       obj.ast[1] = a[1]
       obj.ast[2] = a[2][0]

testament/important_packages.nim

@@ -97,7 +97,8 @@ pkg "memo"
 pkg "msgpack4nim", "nim c -r tests/test_spec.nim"
 pkg "nake", "nim c nakefile.nim"
 pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim", url = "https://github.com/nim-lang/neo"
-pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true
+pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true, allowFailure = true
+  # inactive, tests not adapted to #23096
 pkg "netty"
 pkg "nico", allowFailure = true
 pkg "nicy", "nim c -r src/nicy.nim"

tests/macros/tastrepr.nim

@@ -11,6 +11,8 @@ for i, (x, y) in pairs(data):
 var
   a = 1
   b = 2
+type
+  A* = object
 
 var data = @[(1, "one"), (2, "two")]
 for (i, d) in pairs(data):
@@ -20,6 +22,8 @@ for i, d in pairs(data):
 for i, (x, y) in pairs(data):
   discard
 var (a, b) = (1, 2)
+type
+  A* = object
 '''
 """
 
@@ -44,3 +48,4 @@ echoTypedAndUntypedRepr:
   for i, (x,y) in pairs(data):
     discard
   var (a,b) = (1,2)
+  type A* = object # issue #22933

tests/macros/tgetimpl.nim

@@ -75,7 +75,9 @@ assert: check_gen_proc(len(a)) == (false, true)
 macro check(x: type): untyped =
   let z = getType(x)
   let y = getImpl(z[1])  
-  let sym = if y[0].kind == nnkSym: y[0] else: y[0][0]
+  var sym = y[0]
+  if sym.kind == nnkPragmaExpr: sym = sym[0]
+  if sym.kind == nnkPostfix: sym = sym[1]
   expectKind(z[1], nnkSym)
   expectKind(sym, nnkSym)
   expectKind(y[2], nnkObjectTy)