nimsuggest supports prefix matching (first version)

compiler/sem.nim

@@ -519,12 +519,17 @@ proc myProcess(context: PPassContext, n: PNode): PNode =
       recoverContext(c)
       c.inGenericInst = oldInGenericInst
       msgs.setInfoContextLen(oldContextLen)
-      if getCurrentException() of ESuggestDone: result = nil
+      if getCurrentException() of ESuggestDone:
-      else: result = ast.emptyNode
+        c.suggestionsMade = true
+        result = nil
+      else:
+        result = ast.emptyNode
       #if gCmd == cmdIdeTools: findSuggest(c, n)
 
 proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   var c = PContext(context)
+  if gCmd == cmdIdeTools and not c.suggestionsMade:
+    suggestSentinel(c)
   closeScope(c)         # close module's scope
   rawCloseScope(c)      # imported symbols; don't check for unused ones!
   result = newNode(nkStmtList)

compiler/semdata.nim

@@ -111,6 +111,7 @@ type
     graph*: ModuleGraph
     signatures*: TStrTable
     recursiveDep*: string
+    suggestionsMade*: bool
 
 proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair =
   result.genericSym = s

compiler/suggest.nim

@@ -8,6 +8,20 @@
 #
 
 ## This file implements features required for IDE support.
+##
+## Due to Nim's natures and the fact that ``system.nim`` is always imported,
+## there are lots of potential symbols. Furthermore thanks to templates and
+## macros even context based analysis does not help much: In a context like
+## ``let x: |`` where a type has to follow, that type might be constructed from
+## a template like ``extractField(MyObject, fieldName)``. We deal with this
+## problem by smart sorting so that the likely symbols come first. This sorting
+## is done this way:
+##
+## - If there is a prefix (foo|), symbols starting with this prefix come first.
+## - If the prefix is part of the name (but the name doesn't start with it),
+##   these symbols come second.
+## - Otherwise consider the context. We currently distinguish between type
+##   and non-type contexts.
 
 # included from sigmatch.nim
 
@@ -133,11 +147,20 @@ proc suggestResult(s: Suggest) =
   else:
     suggestWriteln($s)
 
-proc filterSym(s: PSym): bool {.inline.} =
+proc filterSym(s: PSym; prefix: PNode): bool {.inline.} =
-  result = s.kind != skModule
+  proc prefixMatch(s: PSym; n: PNode): bool =
+    case n.kind
+    of nkIdent: result = s.name.s.startsWith(n.ident.s)
+    of nkSym: result = s.name.s.startsWith(n.sym.name.s)
+    of nkOpenSymChoice, nkClosedSymChoice, nkAccQuoted:
+      if n.len > 0:
+        result = prefixMatch(s, n[0])
+    else: discard
+  if s.kind != skModule:
+    result = prefix.isNil or prefixMatch(s, prefix)
 
-proc filterSymNoOpr(s: PSym): bool {.inline.} =
+proc filterSymNoOpr(s: PSym; prefix: PNode): bool {.inline.} =
-  result = s.kind != skModule and s.name.s[0] in lexer.SymChars and
+  result = filterSym(s, prefix) and s.name.s[0] in lexer.SymChars and
      not isKeyword(s.name)
 
 proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} =
@@ -148,8 +171,8 @@ proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} =
       result = true
       break
 
-proc suggestField(c: PContext, s: PSym, outputs: var int) =
+proc suggestField(c: PContext, s: PSym; f: PNode; outputs: var int) =
-  if filterSym(s) and fieldVisible(c, s):
+  if filterSym(s, f) and fieldVisible(c, s):
     suggestResult(symToSuggest(s, isLocal=true, $ideSug, 100))
     inc outputs
 
@@ -166,22 +189,22 @@ template wholeSymTab(cond, section: untyped) =
         suggestResult(symToSuggest(it, isLocal = isLocal, section, 100))
         inc outputs
 
-proc suggestSymList(c: PContext, list: PNode, outputs: var int) =
+proc suggestSymList(c: PContext, list, f: PNode, outputs: var int) =
   for i in countup(0, sonsLen(list) - 1):
     if list.sons[i].kind == nkSym:
-      suggestField(c, list.sons[i].sym, outputs)
+      suggestField(c, list.sons[i].sym, f, outputs)
     #else: InternalError(list.info, "getSymFromList")
 
-proc suggestObject(c: PContext, n: PNode, outputs: var int) =
+proc suggestObject(c: PContext, n, f: PNode, outputs: var int) =
   case n.kind
   of nkRecList:
-    for i in countup(0, sonsLen(n)-1): suggestObject(c, n.sons[i], outputs)
+    for i in countup(0, sonsLen(n)-1): suggestObject(c, n.sons[i], f, outputs)
   of nkRecCase:
     var L = sonsLen(n)
     if L > 0:
-      suggestObject(c, n.sons[0], outputs)
+      suggestObject(c, n.sons[0], f, outputs)
-      for i in countup(1, L-1): suggestObject(c, lastSon(n.sons[i]), outputs)
+      for i in countup(1, L-1): suggestObject(c, lastSon(n.sons[i]), f, outputs)
-  of nkSym: suggestField(c, n.sym, outputs)
+  of nkSym: suggestField(c, n.sym, f, outputs)
   else: discard
 
 proc nameFits(c: PContext, s: PSym, n: PNode): bool =
@@ -205,7 +228,7 @@ proc argsFit(c: PContext, candidate: PSym, n, nOrig: PNode): bool =
     result = false
 
 proc suggestCall(c: PContext, n, nOrig: PNode, outputs: var int) =
-  wholeSymTab(filterSym(it) and nameFits(c, it, n) and argsFit(c, it, n, nOrig),
+  wholeSymTab(filterSym(it, nil) and nameFits(c, it, n) and argsFit(c, it, n, nOrig),
               $ideCon)
 
 proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} =
@@ -221,22 +244,22 @@ proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} =
       if exp.kind in {tyExpr, tyStmt, tyGenericParam, tyAnything}: return
     result = sigmatch.argtypeMatches(c, s.typ.sons[1], firstArg)
 
-proc suggestOperations(c: PContext, n: PNode, typ: PType, outputs: var int) =
+proc suggestOperations(c: PContext, n, f: PNode, typ: PType, outputs: var int) =
   assert typ != nil
-  wholeSymTab(filterSymNoOpr(it) and typeFits(c, it, typ), $ideSug)
+  wholeSymTab(filterSymNoOpr(it, f) and typeFits(c, it, typ), $ideSug)
 
-proc suggestEverything(c: PContext, n: PNode, outputs: var int) =
+proc suggestEverything(c: PContext, n, f: PNode, outputs: var int) =
   # do not produce too many symbols:
   var isLocal = true
   for scope in walkScopes(c.currentScope):
     if scope == c.topLevelScope: isLocal = false
     for it in items(scope.symbols):
-      if filterSym(it):
+      if filterSym(it, f):
         suggestResult(symToSuggest(it, isLocal = isLocal, $ideSug, 0))
         inc outputs
-    if scope == c.topLevelScope: break
+    if scope == c.topLevelScope and f.isNil: break
 
-proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
+proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var int) =
   # special code that deals with ``myObj.``. `n` is NOT the nkDotExpr-node, but
   # ``myObj``.
   var typ = n.typ
@@ -252,7 +275,7 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
         if m == nil: typ = nil
         else:
           for it in items(n.sym.tab):
-            if filterSym(it):
+            if filterSym(it, field):
               suggestResult(symToSuggest(it, isLocal=false, $ideSug, 100))
               inc outputs
           suggestResult(symToSuggest(m, isLocal=false, $ideMod, 100))
@@ -263,38 +286,38 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
       if n.sym == c.module:
         # all symbols accessible, because we are in the current module:
         for it in items(c.topLevelScope.symbols):
-          if filterSym(it):
+          if filterSym(it, field):
             suggestResult(symToSuggest(it, isLocal=false, $ideSug, 100))
             inc outputs
       else:
         for it in items(n.sym.tab):
-          if filterSym(it):
+          if filterSym(it, field):
             suggestResult(symToSuggest(it, isLocal=false, $ideSug, 100))
             inc outputs
     else:
       # fallback:
-      suggestEverything(c, n, outputs)
+      suggestEverything(c, n, field, outputs)
   elif typ.kind == tyEnum and n.kind == nkSym and n.sym.kind == skType:
     # look up if the identifier belongs to the enum:
     var t = typ
     while t != nil:
-      suggestSymList(c, t.n, outputs)
+      suggestSymList(c, t.n, field, outputs)
       t = t.sons[0]
-    suggestOperations(c, n, typ, outputs)
+    suggestOperations(c, n, field, typ, outputs)
   else:
     let orig = skipTypes(typ, {tyGenericInst, tyAlias})
     typ = skipTypes(typ, {tyGenericInst, tyVar, tyPtr, tyRef, tyAlias})
     if typ.kind == tyObject:
       var t = typ
       while true:
-        suggestObject(c, t.n, outputs)
+        suggestObject(c, t.n, field, outputs)
         if t.sons[0] == nil: break
         t = skipTypes(t.sons[0], skipPtrs)
     elif typ.kind == tyTuple and typ.n != nil:
-      suggestSymList(c, typ.n, outputs)
+      suggestSymList(c, typ.n, field, outputs)
-    suggestOperations(c, n, orig, outputs)
+    suggestOperations(c, n, field, orig, outputs)
     if typ != orig:
-      suggestOperations(c, n, typ, outputs)
+      suggestOperations(c, n, field, typ, outputs)
 
 type
   TCheckPointResult* = enum
@@ -443,13 +466,19 @@ proc suggestExpr*(c: PContext, node: PNode) =
     if n == nil: n = node
     if n.kind == nkDotExpr:
       var obj = safeSemExpr(c, n.sons[0])
-      suggestFieldAccess(c, obj, outputs)
+      let prefix = if n.len == 2: n[1] else: nil
+      suggestFieldAccess(c, obj, prefix, outputs)
 
       #if optIdeDebug in gGlobalOptions:
       #  echo "expression ", renderTree(obj), " has type ", typeToString(obj.typ)
       #writeStackTrace()
     else:
-      suggestEverything(c, n, outputs)
+      #let m = findClosestSym(node)
+      #if m != nil:
+      #  suggestPrefix(c, m, outputs)
+      #else:
+      let prefix = if cp == cpExact: n else: nil
+      suggestEverything(c, n, prefix, outputs)
 
   elif gIdeCmd == ideCon:
     var n = findClosestCall(node)
@@ -471,3 +500,17 @@ proc suggestExpr*(c: PContext, node: PNode) =
 
 proc suggestStmt*(c: PContext, n: PNode) =
   suggestExpr(c, n)
+
+proc suggestSentinel*(c: PContext) =
+  if gIdeCmd != ideSug or c.module.position != gTrackPos.fileIndex: return
+  if c.compilesContextId > 0: return
+  inc(c.compilesContextId)
+  # suggest everything:
+  var isLocal = true
+  for scope in walkScopes(c.currentScope):
+    if scope == c.topLevelScope: isLocal = false
+    for it in items(scope.symbols):
+      if filterSymNoOpr(it, nil):
+        suggestResult(symToSuggest(it, isLocal = isLocal, $ideSug, 0))
+
+  dec(c.compilesContextId)

lib/pure/unittest.nim

@@ -380,8 +380,7 @@ macro expect*(exceptions: varargs[typed], body: untyped): untyped =
   ##  expect IOError, OSError, ValueError, AssertionError:
   ##    defectiveRobot()
   let exp = callsite()
-  template expectBody(errorTypes, lineInfoLit: expr,
+  template expectBody(errorTypes, lineInfoLit, body): NimNode {.dirty.} =
-                      body: stmt): NimNode {.dirty.} =
     try:
       body
       checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.")

tools/nimsuggest/nimsuggest.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nim Compiler
-#        (c) Copyright 2016 Andreas Rumpf
+#        (c) Copyright 2017 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -50,7 +50,7 @@ In addition, all command line options of Nim that do not affect code generation
 are supported.
 """
 type
-  Mode = enum mstdin, mtcp, mepc
+  Mode = enum mstdin, mtcp, mepc, mcmdline
 
 var
   gPort = 6000.Port
@@ -133,7 +133,8 @@ proc symFromInfo(graph: ModuleGraph; gTrackPos: TLineInfo): PSym =
 proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
              graph: ModuleGraph; cache: IdentCache) =
   if gLogging:
-    logStr("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile & "[" & $line & ":" & $col & "]")
+    logStr("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile &
+          "[" & $line & ":" & $col & "]")
   gIdeCmd = cmd
   if cmd == ideUse and suggestVersion != 2:
     graph.resetAllModules()
@@ -258,6 +259,12 @@ proc connectToNextFreePort(server: Socket, host: string): Port =
 type
   ThreadParams = tuple[port: Port; address: string]
 
+proc replStdinSingleCmd(line: string) =
+  requests.send line
+  toStdout()
+  echo ""
+  flushFile(stdout)
+
 proc replStdin(x: ThreadParams) {.thread.} =
   if gEmitEof:
     echo DummyEof
@@ -271,10 +278,11 @@ proc replStdin(x: ThreadParams) {.thread.} =
     echo Help
     var line = ""
     while readLineFromStdin("> ", line):
-      requests.send line
+      replStdinSingleCmd(line)
-      toStdout()
+
-      echo ""
+proc replCmdline(x: ThreadParams) {.thread.} =
-      flushFile(stdout)
+  replStdinSingleCmd(x.address)
+  requests.send "quit"
 
 proc replTcp(x: ThreadParams) {.thread.} =
   var server = newSocket()
@@ -313,6 +321,7 @@ proc replEpc(x: ThreadParams) {.thread.} =
   let port = connectToNextFreePort(server, "localhost")
   server.listen()
   echo port
+  stdout.flushFile()
 
   var client = newSocket()
   # Wait for connection
@@ -461,92 +470,6 @@ proc mainThread(graph: ModuleGraph; cache: IdentCache) =
 var
   inputThread: Thread[ThreadParams]
 
-proc serveStdin(graph: ModuleGraph; cache: IdentCache) {.deprecated.} =
-  if gEmitEof:
-    echo DummyEof
-    while true:
-      let line = readLine(stdin)
-      execCmd line, graph, cache
-      echo DummyEof
-      flushFile(stdout)
-  else:
-    echo Help
-    var line = ""
-    while readLineFromStdin("> ", line):
-      execCmd line, graph, cache
-      echo ""
-      flushFile(stdout)
-
-proc serveTcp(graph: ModuleGraph; cache: IdentCache) {.deprecated.} =
-  var server = newSocket()
-  server.bindAddr(gPort, gAddress)
-  var inp = "".TaintedString
-  server.listen()
-
-  while true:
-    var stdoutSocket = newSocket()
-    msgs.writelnHook = proc (line: string) =
-      stdoutSocket.send(line & "\c\L")
-
-    accept(server, stdoutSocket)
-
-    stdoutSocket.readLine(inp)
-    execCmd inp.string, graph, cache
-
-    stdoutSocket.send("\c\L")
-    stdoutSocket.close()
-
-proc serveEpc(server: Socket; graph: ModuleGraph; cache: IdentCache) {.deprecated.} =
-  var client = newSocket()
-  # Wait for connection
-  accept(server, client)
-  if gLogging:
-    for it in searchPaths:
-      logStr(it)
-    msgs.writelnHook = proc (line: string) = logStr(line)
-
-  while true:
-    var
-      sizeHex = ""
-      size = 0
-      messageBuffer = ""
-    checkSanity(client, sizeHex, size, messageBuffer)
-    let
-      message = parseSexp($messageBuffer)
-      epcAPI = message[0].getSymbol
-    case epcAPI:
-    of "call":
-      let
-        uid = message[1].getNum
-        args = message[3]
-
-      gIdeCmd = parseIdeCmd(message[2].getSymbol)
-      case gIdeCmd
-      of ideChk:
-        setVerbosity(1)
-        # Use full path because other emacs plugins depends it
-        gListFullPaths = true
-        incl(gGlobalOptions, optIdeDebug)
-        var hints_or_errors = ""
-        sendEpc(hints_or_errors, string, msgs.writelnHook)
-      of ideSug, ideCon, ideDef, ideUse, ideDus, ideOutline, ideHighlight:
-        setVerbosity(0)
-        var suggests: seq[Suggest] = @[]
-        sendEpc(suggests, Suggest, suggestionResultHook)
-      else: discard
-    of "methods":
-      returnEpc(client, message[1].getNum, listEPC())
-    of "epc-error":
-      stderr.writeline("recieved epc error: " & $messageBuffer)
-      raise newException(IOError, "epc error")
-    else:
-      let errMessage = case epcAPI
-                       of "return", "return-error":
-                         "no return expected"
-                       else:
-                         "unexpected call: " & epcAPI
-      raise newException(EUnexpectedCommand, errMessage)
-
 proc mainCommand(graph: ModuleGraph; cache: IdentCache) =
   clearPasses()
   registerPass verbosePass
@@ -556,9 +479,6 @@ proc mainCommand(graph: ModuleGraph; cache: IdentCache) =
   isServing = true
   wantMainModule()
   add(searchPaths, options.libpath)
-  #if gProjectFull.len != 0:
-    # current path is always looked first for modules
-  #  prependStr(searchPaths, gProjectPath)
 
   # do not stop after the first error:
   msgs.gErrorMax = high(int)
@@ -573,32 +493,13 @@ proc mainCommand(graph: ModuleGraph; cache: IdentCache) =
   of mstdin: createThread(inputThread, replStdin, (gPort, gAddress))
   of mtcp: createThread(inputThread, replTcp, (gPort, gAddress))
   of mepc: createThread(inputThread, replEpc, (gPort, gAddress))
+  of mcmdline: createThread(inputThread, replCmdline,
+                            (gPort, "sug \"" & options.gProjectFull & "\":" & gAddress))
   mainThread(graph, cache)
   joinThread(inputThread)
   close(requests)
   close(results)
 
-  when false:
-    case gMode
-    of mstdin:
-      compileProject(graph, cache)
-      #modules.gFuzzyGraphChecking = false
-      serveStdin(graph, cache)
-    of mtcp:
-      # until somebody accepted the connection, produce no output (logging is too
-      # slow for big projects):
-      msgs.writelnHook = proc (msg: string) = discard
-      compileProject(graph, cache)
-      #modules.gFuzzyGraphChecking = false
-      serveTcp(graph, cache)
-    of mepc:
-      var server = newSocket()
-      let port = connectToNextFreePort(server, "localhost")
-      server.listen()
-      echo port
-      compileProject(graph, cache)
-      serveEpc(server, graph, cache)
-
 proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
   var p = parseopt.initOptParser(cmd)
   while true:
@@ -614,6 +515,10 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
         gAddress = p.val
         gMode = mtcp
       of "stdin": gMode = mstdin
+      of "cmdline":
+        gMode = mcmdline
+        suggestVersion = 2
+        gAddress = p.val
       of "epc":
         gMode = mepc
         gVerbosity = 0          # Port number gotta be first.
@@ -678,12 +583,6 @@ proc handleCmdLine(cache: IdentCache; config: ConfigRef) =
     graph.suggestMode = true
     mainCommand(graph, cache)
 
-when false:
-  proc quitCalled() {.noconv.} =
-    writeStackTrace()
-
-  addQuitProc(quitCalled)
-
 condsyms.initDefines()
 defineSymbol "nimsuggest"
 handleCmdline(newIdentCache(), newConfigRef())

tools/nimsuggest/tester.nim

@@ -233,22 +233,25 @@ proc runEpcTest(filename: string): int =
   let outp = p.outputStream
   let inp = p.inputStream
   var report = ""
-  var a = newStringOfCap(120)
+  #var a = newStringOfCap(120)
   try:
     # read the port number:
-    if outp.readLine(a):
+    #discard outp.readLine(a)
-      let port = parseInt(a)
+    var i = 0
-      var socket = newSocket()
+    while not osproc.hasData(p) and i < 100:
-      socket.connect("localhost", Port(port))
+      os.sleep(50)
-      for req, resp in items(s.script):
+      inc i
-        if not runCmd(req, s.dest):
+    let a = outp.readAll().strip()
-          socket.sendEpcStr(req)
+    let port = parseInt(a)
-          let sx = parseSexp(socket.recvEpc())
+    var socket = newSocket()
-          if not req.startsWith("mod "):
+    socket.connect("localhost", Port(port))
-            let answer = sexpToAnswer(sx)
+    for req, resp in items(s.script):
-            doReport(filename, answer, resp, report)
+      if not runCmd(req, s.dest):
-    else:
+        socket.sendEpcStr(req)
-      raise newException(ValueError, "cannot read port number")
+        let sx = parseSexp(socket.recvEpc())
+        if not req.startsWith("mod "):
+          let answer = sexpToAnswer(sx)
+          doReport(filename, answer, resp, report)
   finally:
     close(p)
   if report.len > 0:
@@ -302,7 +305,9 @@ proc main() =
     for x in walkFiles(getAppDir() / "tests/t*.nim"):
       echo "Test ", x
       let xx = expandFilename x
-      failures += runTest(xx)
+      when not defined(windows):
+        # XXX Windows IO redirection seems bonkers:
+        failures += runTest(xx)
       failures += runEpcTest(xx)
   if failures > 0:
     quit 1