Merge branch 'devel' into pr_string_v3

compiler/astalgo.nim

@@ -370,7 +370,7 @@ proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
       if conf != nil:
         result.addf(",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
       case n.kind
-      of nkCharLit..nkInt64Lit:
+      of nkCharLit..nkUInt64Lit:
         result.addf(",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
       of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
         result.addf(",$N$1\"floatVal\": $2",

compiler/lexer.nim

@@ -122,7 +122,6 @@ type
                               # this is needed because scanning comments
                               # needs so much look-ahead
     currLineIndent*: int
-    strongSpaces*, allowTabs*: bool
     errorHandler*: ErrorHandler
     cache*: IdentCache
     when defined(nimsuggest):
@@ -176,32 +175,6 @@ proc printTok*(conf: ConfigRef; tok: Token) =
   # xxx factor with toLocation
   msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" & $tok.tokType & " " & $tok)
 
-proc initToken*(L: var Token) =
-  L.tokType = tkInvalid
-  L.iNumber = 0
-  L.indent = 0
-  L.spacing = {}
-  L.literal = ""
-  L.fNumber = 0.0
-  L.base = base10
-  L.ident = nil
-  when defined(nimpretty):
-    L.commentOffsetA = 0
-    L.commentOffsetB = 0
-
-proc fillToken(L: var Token) =
-  L.tokType = tkInvalid
-  L.iNumber = 0
-  L.indent = 0
-  L.spacing = {}
-  setLen(L.literal, 0)
-  L.fNumber = 0.0
-  L.base = base10
-  L.ident = nil
-  when defined(nimpretty):
-    L.commentOffsetA = 0
-    L.commentOffsetB = 0
-
 proc openLexer*(lex: var Lexer, fileIdx: FileIndex, inputstream: PLLStream;
                  cache: IdentCache; config: ConfigRef) =
   openBaseLexer(lex, inputstream)
@@ -798,7 +771,7 @@ proc getString(L: var Lexer, tok: var Token, mode: StringMode) =
     if mode != normal: tok.tokType = tkRStrLit
     else: tok.tokType = tkStrLit
     while true:
-      var c = L.buf[pos]
+      let c = L.buf[pos]
       if c == '\"':
         if mode != normal and L.buf[pos+1] == '\"':
           inc(pos, 2)
@@ -824,7 +797,7 @@ proc getCharacter(L: var Lexer; tok: var Token) =
   tokenBegin(tok, L.bufpos)
   let startPos = L.bufpos
   inc(L.bufpos)               # skip '
-  var c = L.buf[L.bufpos]
+  let c = L.buf[L.bufpos]
   case c
   of '\0'..pred(' '), '\'':
     lexMessage(L, errGenerated, "invalid character literal")
@@ -942,7 +915,7 @@ proc getOperator(L: var Lexer, tok: var Token) =
   tokenBegin(tok, pos)
   var h: Hash = 0
   while true:
-    var c = L.buf[pos]
+    let c = L.buf[pos]
     if c in OpChars:
       h = h !& ord(c)
       inc(pos)
@@ -1010,23 +983,6 @@ proc getPrecedence*(tok: Token): int =
   of tkOr, tkXor, tkPtr, tkRef: result = 3
   else: return -10
 
-proc newlineFollows*(L: Lexer): bool =
-  result = false
-  var pos = L.bufpos
-  while true:
-    case L.buf[pos]
-    of ' ', '\t':
-      inc(pos)
-    of CR, LF:
-      result = true
-      break
-    of '#':
-      inc(pos)
-      if L.buf[pos] == '#': inc(pos)
-      if L.buf[pos] != '[': return true
-    else:
-      break
-
 proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
                           isDoc: bool) =
   var pos = start
@@ -1118,9 +1074,7 @@ proc scanComment(L: var Lexer, tok: var Token) =
         toStrip = 0
       else:  # found first non-whitespace character
         stripInit = true
-    var lastBackslash = -1
     while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
-      if L.buf[pos] == '\\': lastBackslash = pos+1
       tok.literal.add(L.buf[pos])
       inc(pos)
     tokenEndIgnore(tok, pos)
@@ -1163,7 +1117,7 @@ proc skip(L: var Lexer, tok: var Token) =
       inc(pos)
       tok.spacing.incl(tsLeading)
     of '\t':
-      if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabs are not allowed, use spaces instead")
+      lexMessagePos(L, errGenerated, pos, "tabs are not allowed, use spaces instead")
       inc(pos)
     of CR, LF:
       tokenEndPrevious(tok, pos)
@@ -1231,7 +1185,7 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
         L.previousToken.line = tok.line.uint16
         L.previousToken.col = tok.col.int16
 
-  fillToken(tok)
+  reset(tok)
   if L.indentAhead >= 0:
     tok.indent = L.indentAhead
     L.currLineIndent = L.indentAhead
@@ -1243,7 +1197,7 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
     if tok.tokType == tkComment:
       L.indentAhead = L.currLineIndent
       return
-  var c = L.buf[L.bufpos]
+  let c = L.buf[L.bufpos]
   tok.line = L.lineNumber
   tok.col = getColNumber(L, L.bufpos)
   if c in SymStartChars - {'r', 'R'} - UnicodeOperatorStartChars:
@@ -1402,7 +1356,6 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
   result = 0
   var lex: Lexer = default(Lexer)
   var tok: Token = default(Token)
-  initToken(tok)
   openLexer(lex, fileIdx, inputstream, cache, config)
   var prevToken = tkEof
   while tok.tokType != tkEof:
@@ -1415,11 +1368,11 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
 
 proc getPrecedence*(ident: PIdent): int =
   ## assumes ident is binary operator already
-  var tok: Token
-  initToken(tok)
-  tok.ident = ident
-  tok.tokType =
-    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol)..ord(tokKeywordHigh) - ord(tkSymbol):
-      TokType(tok.ident.id + ord(tkSymbol))
-    else: tkOpr
+  let
+    tokType =
+      if ident.id in ord(tokKeywordLow) - ord(tkSymbol)..ord(tokKeywordHigh) - ord(tkSymbol):
+        TokType(ident.id + ord(tkSymbol))
+      else: tkOpr
+    tok = Token(ident: ident, tokType: tokType)
+
   getPrecedence(tok)

compiler/main.nim

@@ -224,7 +224,6 @@ proc commandScan(cache: IdentCache, config: ConfigRef) =
     var
       L: Lexer
       tok: Token = default(Token)
-    initToken(tok)
     openLexer(L, f, stream, cache, config)
     while true:
       rawGetTok(L, tok)

compiler/nimconf.nim

@@ -222,7 +222,6 @@ proc readConfigFile*(filename: AbsoluteFile; cache: IdentCache;
     stream: PLLStream
   stream = llStreamOpen(filename, fmRead)
   if stream != nil:
-    initToken(tok)
     openLexer(L, filename, stream, cache, config)
     tok.tokType = tkEof       # to avoid a pointless warning
     var condStack: seq[bool] = @[]

compiler/parser.nim

@@ -83,10 +83,6 @@ type
   PrimaryMode = enum
     pmNormal, pmTypeDesc, pmTypeDef, pmTrySimple
 
-proc parseAll*(p: var Parser): PNode
-proc closeParser*(p: var Parser)
-proc parseTopLevelStmt*(p: var Parser): PNode
-
 # helpers for the other parsers
 proc isOperator*(tok: Token): bool
 proc getTok*(p: var Parser)
@@ -144,7 +140,7 @@ proc openParser*(p: var Parser, fileIdx: FileIndex, inputStream: PLLStream,
                  cache: IdentCache; config: ConfigRef) =
   ## Open a parser, using the given arguments to set up its internal state.
   ##
-  initToken(p.tok)
+  reset(p.tok)
   openLexer(p.lex, fileIdx, inputStream, cache, config)
   when defined(nimpretty):
     openEmitter(p.em, cache, config, fileIdx)
@@ -156,7 +152,7 @@ proc openParser*(p: var Parser, filename: AbsoluteFile, inputStream: PLLStream,
                  cache: IdentCache; config: ConfigRef) =
   openParser(p, fileInfoIdx(config, filename), inputStream, cache, config)
 
-proc closeParser(p: var Parser) =
+proc closeParser*(p: var Parser) =
   ## Close a parser, freeing up its resources.
   closeLexer(p.lex)
   when defined(nimpretty):
@@ -2520,7 +2516,7 @@ proc parseStmt(p: var Parser): PNode =
           if err and p.tok.tokType == tkEof: break
   setEndInfo()
 
-proc parseAll(p: var Parser): PNode =
+proc parseAll*(p: var Parser): PNode =
   ## Parses the rest of the input stream held by the parser into a PNode.
   result = newNodeP(nkStmtList, p)
   while p.tok.tokType != tkEof:
@@ -2540,7 +2536,7 @@ proc checkFirstLineIndentation*(p: var Parser) =
   if p.tok.indent != 0 and tsLeading in p.tok.spacing:
     parMessage(p, errInvalidIndentation)
 
-proc parseTopLevelStmt(p: var Parser): PNode =
+proc parseTopLevelStmt*(p: var Parser): PNode =
   ## Implements an iterator which, when called repeatedly, returns the next
   ## top-level statement or emptyNode if end of stream.
   result = p.emptyNode

compiler/pragmas.nim

@@ -14,6 +14,8 @@ import
   wordrecg, ropes, options, extccomp, magicsys, trees,
   types, lookups, lineinfos, pathutils, linter, modulepaths
 
+from sigmatch import trySuggestPragmas
+
 import std/[os, math, strutils]
 
 when defined(nimPreviewSlimSystem):
@@ -119,6 +121,7 @@ const
 
 proc invalidPragma*(c: PContext; n: PNode) =
   localError(c.config, n.info, "invalid pragma: " & renderTree(n, {renderNoComments}))
+
 proc illegalCustomPragma*(c: PContext, n: PNode, s: PSym) =
   var msg = "cannot attach a custom pragma to '" & s.name.s & "'"
   if s != nil:
@@ -685,9 +688,12 @@ proc pragmaLine(c: PContext, n: PNode) =
 proc processPragma(c: PContext, n: PNode, i: int) =
   ## Create and add a new custom pragma `{.pragma: name.}` node to the module's context.
   let it = n[i]
-  if it.kind notin nkPragmaCallKinds and it.safeLen == 2: invalidPragma(c, n)
+  if it.kind notin nkPragmaCallKinds and it.safeLen == 2:
+    invalidPragma(c, n)
+    return
   elif it.safeLen != 2 or it[0].kind != nkIdent or it[1].kind != nkIdent:
     invalidPragma(c, n)
+    return
 
   var userPragma = newSym(skTemplate, it[1].ident, c.idgen, c.module, it.info, c.config.options)
   styleCheckDef(c, userPragma)
@@ -787,6 +793,8 @@ proc semCustomPragma(c: PContext, n: PNode, sym: PSym): PNode =
     invalidPragma(c, n)
     return n
 
+  trySuggestPragmas(c, callNode[0])
+
   let r = c.semOverloadedCall(c, callNode, n, {skTemplate}, {efNoUndeclared})
   if r.isNil or sfCustomPragma notin r[0].sym.flags:
     invalidPragma(c, n)
@@ -1313,7 +1321,7 @@ proc implicitPragmas*(c: PContext, sym: PSym, info: TLineInfo,
   if sym != nil and sym.kind != skModule:
     for it in c.optionStack:
       let o = it.otherPragmas
-      if not o.isNil:
+      if not o.isNil and sfFromGeneric notin sym.flags: # bug #23019
         pushInfoContext(c.config, info)
         var i = 0
         while i < o.len:

compiler/renderer.nim

@@ -125,6 +125,8 @@ template outside(g: var TSrcGen, section: Section, body: untyped) =
 const
   IndentWidth = 2
   longIndentWid = IndentWidth * 2
+  MaxLineLen = 80
+  LineCommentColumn = 30
 
 when defined(nimpretty):
   proc minmaxLine(n: PNode): (int, int) =
@@ -143,10 +145,6 @@ when defined(nimpretty):
   proc lineDiff(a, b: PNode): int =
     result = minmaxLine(b)[0] - minmaxLine(a)[1]
 
-const
-  MaxLineLen = 80
-  LineCommentColumn = 30
-
 proc initSrcGen(renderFlags: TRenderFlags; config: ConfigRef): TSrcGen =
   result = TSrcGen(comStack: @[], tokens: @[], indent: 0,
                    lineLen: 0, pos: 0, idx: 0, buf: "",

compiler/semstmts.nim

@@ -525,6 +525,8 @@ proc semVarMacroPragma(c: PContext, a: PNode, n: PNode): PNode =
       let it = pragmas[i]
       let key = if it.kind in nkPragmaCallKinds and it.len >= 1: it[0] else: it
 
+      trySuggestPragmas(c, key)
+
       if isPossibleMacroPragma(c, it, key):
         # we transform ``var p {.m, rest.}`` into ``m(do: var p {.rest.})`` and
         # let the semantic checker deal with it:
@@ -1741,6 +1743,8 @@ proc semProcAnnotation(c: PContext, prc: PNode;
     let it = n[i]
     let key = if it.kind in nkPragmaCallKinds and it.len >= 1: it[0] else: it
 
+    trySuggestPragmas(c, key)
+
     if isPossibleMacroPragma(c, it, key):
       # we transform ``proc p {.m, rest.}`` into ``m(do: proc p {.rest.})`` and
       # let the semantic checker deal with it:

compiler/semtypes.nim

@@ -1741,10 +1741,10 @@ proc applyTypeSectionPragmas(c: PContext; pragmas, operand: PNode): PNode =
   result = nil
   for p in pragmas:
     let key = if p.kind in nkPragmaCallKinds and p.len >= 1: p[0] else: p
-
     if p.kind == nkEmpty or whichPragma(p) != wInvalid:
       discard "builtin pragma"
     else:
+      trySuggestPragmas(c, key)
       let ident = considerQuotedIdent(c, key)
       if strTableGet(c.userPragmas, ident) != nil:
         discard "User-defined pragma"

compiler/suggest.nim

@@ -126,7 +126,7 @@ proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info
                   inTypeContext: bool; scope: int;
                   useSuppliedInfo = false,
                   endLine: uint16 = 0,
-                  endCol = 0): Suggest =
+                  endCol = 0, extractDocs = true): Suggest =
   new(result)
   result.section = section
   result.quality = quality
@@ -165,7 +165,8 @@ proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info
     else:
       result.forth = ""
     when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler):
-      result.doc = extractDocComment(g, s)
+      if extractDocs:
+        result.doc = extractDocComment(g, s)
   if s.kind == skModule and s.ast.len != 0 and section != ideHighlight:
     result.filePath = toFullPath(g.config, s.ast[0].info)
     result.line = 1
@@ -746,6 +747,38 @@ proc suggestEnum*(c: PContext; n: PNode; t: PType) =
   produceOutput(outputs, c.config)
   if outputs.len > 0: suggestQuit()
 
+proc suggestPragmas*(c: PContext, n: PNode) =
+  ## Suggests anything that might be a pragma
+  ## - template that has {.pragma.}
+  ## - macros
+  ## - user pragmas
+  let info = n.info
+  var outputs: Suggestions = @[]
+  # First filter for template/macros
+  wholeSymTab(filterSym(it, n, pm) and
+    (sfCustomPragma in it.flags or it.kind == skMacro),
+    ideSug)
+
+  # Now show suggestions for user pragmas
+  for pragma in c.userPragmas:
+      var pm = default(PrefixMatch)
+      if filterSym(pragma, n, pm):
+        outputs &= symToSuggest(c.graph, pragma, isLocal=true, ideSug, info,
+                                 pragma.getQuality, pm, c.inTypeContext > 0, 0,
+                                 extractDocs=false)
+
+  produceOutput(outputs, c.config)
+  if outputs.len > 0:
+    suggestQuit()
+
+template trySuggestPragmas*(c: PContext, n: PNode) =
+  ## Runs [suggestPragmas] when compiling nimsuggest and
+  ## we are querying the node
+  when defined(nimsuggest):
+    let tmp = n
+    if c.config.ideCmd == ideSug and exactEquals(c.config.m.trackPos, tmp.info):
+      suggestPragmas(c, tmp)
+
 proc suggestSentinel*(c: PContext) =
   if c.config.ideCmd != ideSug or c.module.position != c.config.m.trackPos.fileIndex.int32: return
   if c.compilesContextId > 0: return

lib/js/asyncjs.nim

@@ -90,6 +90,8 @@ proc replaceReturn(node: var NimNode) =
       node[z] = nnkReturnStmt.newTree(value)
     elif son.kind == nnkAsgn and son[0].kind == nnkIdent and $son[0] == "result":
       node[z] = nnkAsgn.newTree(son[0], nnkCall.newTree(jsResolve, son[1]))
+    elif son.kind in RoutineNodes:
+      discard
     else:
       replaceReturn(son)
     inc z

nimsuggest/tests/tsug_pragmas.nim

@@ -0,0 +1,40 @@
+template fooBar1() {.pragma.}
+proc fooBar2() = discard
+macro fooBar3(x: untyped) = discard
+{.pragma: fooBar4 fooBar3.}
+
+proc test1() {.fooBar#[!]#.} = discard
+
+var test2 {.fooBar#[!]#.} = 9
+
+type
+  Person {.fooBar#[!]#.} = object
+    hello {.fooBar#[!]#.}: string
+  Callback = proc () {.fooBar#[!]#.}
+
+# Check only macros/templates/pragmas are suggested
+discard """
+$nimsuggest --tester $file
+>sug $1
+sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix
+sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix
+sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe.};;$file;;3;;6;;"";;50;;Prefix
+>sug $2
+sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix
+sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix
+sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe.};;$file;;3;;6;;"";;50;;Prefix
+>sug $3
+sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix
+sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix
+sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe.};;$file;;3;;6;;"";;50;;Prefix
+>sug $4
+sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix
+sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix
+sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe.};;$file;;3;;6;;"";;50;;Prefix
+>sug $5
+sug;;skTemplate;;fooBar4;;;;$file;;4;;8;;"";;100;;Prefix
+sug;;skTemplate;;tsug_pragmas.fooBar1;;template ();;$file;;1;;9;;"";;100;;Prefix
+sug;;skMacro;;tsug_pragmas.fooBar3;;macro (x: untyped){.noSideEffect, gcsafe.};;$file;;3;;6;;"";;50;;Prefix
+"""
+
+

tests/js/tasyncjs.nim

@@ -99,4 +99,9 @@ block asyncPragmaInType:
   proc foo() {.async.} = discard
   var x: Handler = foo
 
+block: # 13341
+  proc f {.async.} =
+    proc g: int =
+      result = 123
+
 discard main()

tests/pragmas/tinvalid_user_pragma.nim

@@ -0,0 +1,9 @@
+discard """
+cmd: "nim check $file"
+"""
+
+{.pragma test: foo.} #[tt.Error
+^ invalid pragma:  {.pragma, test: foo.} ]#
+
+{.pragma: 1.} #[tt.Error
+^ invalid pragma:  {.pragma: 1.} ]#

tests/pragmas/tpush.nim

@@ -77,3 +77,25 @@ block: # bug #22913
     {.pop.}
 
     discard foo2()
+
+block: # bug #23019
+  proc f(x: bool)
+
+  proc a(x: int) =
+    if false: f(true)
+
+  proc f(x: bool) =
+    if false: a(0)
+
+  proc k(r: int|int) {.inline.} =  # seems to require being generic and inline
+    if false: a(0)
+
+
+  # {.push tags: [].}
+  {.push raises: [].}
+
+  {.push warning[ObservableStores]:off.}  # can be any warning, off or on
+  let w = 0
+  k(w)
+  {.pop.}
+  {.pop.}

tools/grammar_nanny.nim

@@ -22,7 +22,6 @@ proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) =
     var
       L: Lexer
       tok: Token
-    initToken(tok)
     openLexer(L, f, stream, cache, config)
     # load the first token:
     rawGetTok(L, tok)