diff --git a/compiler/lexer.nim b/compiler/lexer.nim index db370f8b3e..afdf17baab 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -67,6 +67,10 @@ type TTokTypes* = set[TTokType] const + weakTokens = {tkComma, tkSemiColon, tkColon, + tkParRi, tkParDotRi, tkBracketRi, tkBracketDotRi, + tkCurlyRi} # \ + # tokens that should not be considered for previousToken tokKeywordLow* = succ(tkSymbol) tokKeywordHigh* = pred(tkIntLit) TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]", @@ -105,6 +109,9 @@ type # so that it is the correct default value base2, base8, base16 + CursorPosition* {.pure.} = enum ## XXX remove this again + None, InToken, BeforeToken, AfterToken + TToken* = object # a Nim token tokType*: TTokType # the type of the token indent*: int # the indentation; != -1 if the token has been @@ -128,8 +135,11 @@ type # needs so much look-ahead currLineIndent*: int strongSpaces*, allowTabs*: bool + cursor*: CursorPosition errorHandler*: TErrorHandler cache*: IdentCache + when defined(nimsuggest): + previousToken: TLineInfo var gLinesCompiled*: int # all lines that have been compiled @@ -203,6 +213,7 @@ proc openLexer*(lex: var TLexer, fileIdx: int32, inputstream: PLLStream; lex.currLineIndent = 0 inc(lex.lineNumber, inputstream.lineOffset) lex.cache = cache + lex.previousToken.fileIndex = fileIdx proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream; cache: IdentCache) = @@ -235,6 +246,41 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") = proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool = result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second) +template tokenBegin(pos) {.dirty.} = + when defined(nimsuggest): + var colA = getColNumber(L, pos) + +template tokenEnd(pos) {.dirty.} = + when defined(nimsuggest): + let colB = getColNumber(L, pos) + if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and + L.lineNumber == gTrackPos.line and gIdeCmd in {ideSug, ideCon}: + L.cursor = CursorPosition.InToken + gTrackPos.col = colA.int16 + colA = 0 + +template tokenEndIgnore(pos) = + when defined(nimsuggest): + let colB = getColNumber(L, pos) + if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and + L.lineNumber == gTrackPos.line and gIdeCmd in {ideSug, ideCon}: + gTrackPos.fileIndex = trackPosInvalidFileIdx + gTrackPos.line = -1 + colA = 0 + +template tokenEndPrevious(pos) = + when defined(nimsuggest): + # when we detect the cursor in whitespace, we attach the track position + # to the token that came before that, but only if we haven't detected + # the cursor in a string literal or comment: + let colB = getColNumber(L, pos) + if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and + L.lineNumber == gTrackPos.line and gIdeCmd in {ideSug, ideCon}: + L.cursor = CursorPosition.BeforeToken + gTrackPos = L.previousToken + gTrackPosAttached = true + colA = 0 + {.push overflowChecks: off.} # We need to parse the largest uint literal without overflow checks proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int = @@ -318,6 +364,7 @@ proc getNumber(L: var TLexer, result: var TToken) = result.literal = "" result.base = base10 startpos = L.bufpos + tokenBegin(startPos) # First stage: find out base, make verifications, build token literal string if L.buf[L.bufpos] == '0' and L.buf[L.bufpos + 1] in baseCodeChars + {'O'}: @@ -526,6 +573,7 @@ proc getNumber(L: var TLexer, result: var TToken) = lexMessageLitNum(L, errInvalidNumber, startpos) except OverflowError, RangeError: lexMessageLitNum(L, errNumberOutOfRange, startpos) + tokenEnd(postPos-1) L.bufpos = postPos proc handleHexChar(L: var TLexer, xi: var int) = @@ -642,21 +690,11 @@ proc handleCRLF(L: var TLexer, pos: int): int = result = nimlexbase.handleLF(L, pos) else: result = pos -template tokenRange(colA, pos) = - when defined(nimsuggest): - let colB = getColNumber(L, pos) - if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and - L.lineNumber == gTrackPos.line and gIdeCmd == ideSug: - gTrackPos.fileIndex = trackPosInvalidFileIdx - gTrackPos.line = -1 - colA = 0 - proc getString(L: var TLexer, tok: var TToken, rawMode: bool) = var pos = L.bufpos + 1 # skip " var buf = L.buf # put `buf` in a register var line = L.lineNumber # save linenumber for better error message - when defined(nimsuggest): - var colA = getColNumber(L, pos) + tokenBegin(pos) if buf[pos] == '\"' and buf[pos+1] == '\"': tok.tokType = tkTripleStrLit # long string literal: inc(pos, 2) # skip "" @@ -672,18 +710,18 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) = of '\"': if buf[pos+1] == '\"' and buf[pos+2] == '\"' and buf[pos+3] != '\"': - tokenRange(colA, pos+2) + tokenEndIgnore(pos+2) L.bufpos = pos + 3 # skip the three """ break add(tok.literal, '\"') inc(pos) of CR, LF: - tokenRange(colA, pos) + tokenEndIgnore(pos) pos = handleCRLF(L, pos) buf = L.buf add(tok.literal, tnl) of nimlexbase.EndOfFile: - tokenRange(colA, pos) + tokenEndIgnore(pos) var line2 = L.lineNumber L.lineNumber = line lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart) @@ -704,11 +742,11 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) = inc(pos, 2) add(tok.literal, '"') else: - tokenRange(colA, pos) + tokenEndIgnore(pos) inc(pos) # skip '"' break elif c in {CR, LF, nimlexbase.EndOfFile}: - tokenRange(colA, pos) + tokenEndIgnore(pos) lexMessage(L, errClosingQuoteExpected) break elif (c == '\\') and not rawMode: @@ -721,6 +759,7 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) = L.bufpos = pos proc getCharacter(L: var TLexer, tok: var TToken) = + tokenBegin(L.bufpos) inc(L.bufpos) # skip ' var c = L.buf[L.bufpos] case c @@ -730,12 +769,14 @@ proc getCharacter(L: var TLexer, tok: var TToken) = tok.literal = $c inc(L.bufpos) if L.buf[L.bufpos] != '\'': lexMessage(L, errMissingFinalQuote) + tokenEndIgnore(L.bufpos) inc(L.bufpos) # skip ' proc getSymbol(L: var TLexer, tok: var TToken) = var h: Hash = 0 var pos = L.bufpos var buf = L.buf + tokenBegin(pos) while true: var c = buf[pos] case c @@ -762,6 +803,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = inc(pos) else: break + tokenEnd(pos-1) h = !$h tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) L.bufpos = pos @@ -782,6 +824,7 @@ proc endOperator(L: var TLexer, tok: var TToken, pos: int, proc getOperator(L: var TLexer, tok: var TToken) = var pos = L.bufpos var buf = L.buf + tokenBegin(pos) var h: Hash = 0 while true: var c = buf[pos] @@ -789,6 +832,7 @@ proc getOperator(L: var TLexer, tok: var TToken) = h = h !& ord(c) inc(pos) endOperator(L, tok, pos, h) + tokenEnd(pos-1) # advance pos but don't store it in L.bufpos so the next token (which might # be an operator too) gets the preceding spaces: tok.strongSpaceB = 0 @@ -803,8 +847,7 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; var pos = start var buf = L.buf var toStrip = 0 - when defined(nimsuggest): - var colA = getColNumber(L, pos) + tokenBegin(pos) # detect the amount of indentation: if isDoc: toStrip = getColNumber(L, pos) @@ -831,20 +874,20 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; if isDoc: if buf[pos+1] == '#' and buf[pos+2] == '#': if nesting == 0: - tokenRange(colA, pos+2) + tokenEndIgnore(pos+2) inc(pos, 3) break dec nesting tok.literal.add ']' elif buf[pos+1] == '#': if nesting == 0: - tokenRange(colA, pos+1) + tokenEndIgnore(pos+1) inc(pos, 2) break dec nesting inc pos of CR, LF: - tokenRange(colA, pos) + tokenEndIgnore(pos) pos = handleCRLF(L, pos) buf = L.buf # strip leading whitespace: @@ -856,7 +899,7 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; inc pos dec c of nimlexbase.EndOfFile: - tokenRange(colA, pos) + tokenEndIgnore(pos) lexMessagePos(L, errGenerated, pos, "end of multiline comment expected") break else: @@ -867,8 +910,6 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; proc scanComment(L: var TLexer, tok: var TToken) = var pos = L.bufpos var buf = L.buf - when defined(nimsuggest): - var colA = getColNumber(L, pos) tok.tokType = tkComment # iNumber contains the number of '\n' in the token tok.iNumber = 0 @@ -876,6 +917,7 @@ proc scanComment(L: var TLexer, tok: var TToken) = if buf[pos+2] == '[': skipMultiLineComment(L, tok, pos+3, true) return + tokenBegin(pos) inc(pos, 2) var toStrip = 0 @@ -889,7 +931,7 @@ proc scanComment(L: var TLexer, tok: var TToken) = if buf[pos] == '\\': lastBackslash = pos+1 add(tok.literal, buf[pos]) inc(pos) - tokenRange(colA, pos) + tokenEndIgnore(pos) pos = handleCRLF(L, pos) buf = L.buf var indent = 0 @@ -908,13 +950,14 @@ proc scanComment(L: var TLexer, tok: var TToken) = else: if buf[pos] > ' ': L.indentAhead = indent - tokenRange(colA, pos) + tokenEndIgnore(pos) break L.bufpos = pos proc skip(L: var TLexer, tok: var TToken) = var pos = L.bufpos var buf = L.buf + tokenBegin(pos) tok.strongSpaceA = 0 while true: case buf[pos] @@ -925,6 +968,7 @@ proc skip(L: var TLexer, tok: var TToken) = if not L.allowTabs: lexMessagePos(L, errTabulatorsAreNotAllowed, pos) inc(pos) of CR, LF: + tokenEndPrevious(pos) pos = handleCRLF(L, pos) buf = L.buf var indent = 0 @@ -951,15 +995,24 @@ proc skip(L: var TLexer, tok: var TToken) = pos = L.bufpos buf = L.buf else: - when defined(nimsuggest): - var colA = getColNumber(L, pos) + tokenBegin(pos) while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) - tokenRange(colA, pos) + tokenEndIgnore(pos+1) else: break # EndOfFile also leaves the loop + tokenEndPrevious(pos-1) L.bufpos = pos proc rawGetTok*(L: var TLexer, tok: var TToken) = + template atTokenEnd() {.dirty.} = + when defined(nimsuggest): + # we attach the cursor to the last *strong* token + if tok.tokType notin weakTokens: + L.previousToken.line = tok.line.int16 + L.previousToken.col = tok.col.int16 + + when defined(nimsuggest): + L.cursor = CursorPosition.None fillToken(tok) if L.indentAhead >= 0: tok.indent = L.indentAhead @@ -1022,10 +1075,12 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = inc(L.bufpos) of '.': when defined(nimsuggest): - if L.fileIdx == gTrackPos.fileIndex and tok.col+1 == gTrackPos.col and + if L.fileIdx == gTrackPos.fileIndex and tok.col == gTrackPos.col and tok.line == gTrackPos.line and gIdeCmd == ideSug: tok.tokType = tkDot + L.cursor = CursorPosition.InToken inc(L.bufpos) + atTokenEnd() return if L.buf[L.bufpos+1] == ']': tok.tokType = tkBracketDotRi @@ -1092,3 +1147,4 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = tok.tokType = tkInvalid lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')') inc(L.bufpos) + atTokenEnd() diff --git a/compiler/msgs.nim b/compiler/msgs.nim index e50ed0f2a0..b89b4ee93d 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -739,6 +739,8 @@ proc `??`* (info: TLineInfo, filename: string): bool = const trackPosInvalidFileIdx* = -2 # special marker so that no suggestions # are produced within comments and string literals var gTrackPos*: TLineInfo +var gTrackPosAttached*: bool ## whether the tracking position was attached to some + ## close token. type MsgFlag* = enum ## flags altering msgWriteln behavior @@ -863,6 +865,9 @@ proc handleError(msg: TMsgKind, eh: TErrorHandling, s: string) = proc `==`*(a, b: TLineInfo): bool = result = a.line == b.line and a.fileIndex == b.fileIndex +proc exactEquals*(a, b: TLineInfo): bool = + result = a.fileIndex == b.fileIndex and a.line == b.line and a.col == b.col + proc writeContext(lastinfo: TLineInfo) = var info = lastinfo for i in countup(0, len(msgContext) - 1): diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 5630fa34f1..c780f8084e 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -405,39 +405,12 @@ proc inCheckpoint*(current: TLineInfo): TCheckPointResult = if current.line >= gTrackPos.line: return cpFuzzy -proc findClosestDot(n: PNode; inType: var bool): PNode = - if n.kind == nkDotExpr and inCheckpoint(n.info) == cpExact: - result = n - else: - for i in 0.. = current.col and col <= current.col+tokenLen-1: return true -proc findClosestSym(n: PNode): PNode = - if n.kind == nkSym and inCheckpoint(n.info) == cpExact: - result = n - elif n.kind notin {nkNone..nkNilLit}: - for i in 0.. 0: return inc(c.compilesContextId) var outputs: Suggestions = @[] if gIdeCmd == ideSug: - sugExpr(c, node, outputs, cp) + sugExpr(c, n, outputs) elif gIdeCmd == ideCon: - var n = findClosestCall(node) - if n == nil: n = node if n.kind in nkCallKinds: var a = copyNode(n) var x = safeSemExpr(c, n.sons[0]) diff --git a/koch.nim b/koch.nim index 7c01939171..20d01ae98d 100644 --- a/koch.nim +++ b/koch.nim @@ -214,9 +214,9 @@ proc buildNimble(latest: bool) = proc bundleNimsuggest(buildExe: bool) = if buildExe: - nimexec("c --noNimblePath -d:release -p:compiler tools/nimsuggest/nimsuggest.nim") - copyExe("tools/nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe) - removeFile("tools/nimsuggest/nimsuggest".exe) + nimexec("c --noNimblePath -d:release -p:compiler nimsuggest/nimsuggest.nim") + copyExe("nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe) + removeFile("nimsuggest/nimsuggest".exe) proc bundleWinTools() = nimexec("c tools/finish.nim") @@ -253,7 +253,7 @@ proc buildTool(toolname, args: string) = proc buildTools(latest: bool) = let nimsugExe = "bin/nimsuggest".exe nimexec "c --noNimblePath -p:compiler -d:release -o:" & nimsugExe & - " tools/nimsuggest/nimsuggest.nim" + " nimsuggest/nimsuggest.nim" let nimgrepExe = "bin/nimgrep".exe nimexec "c -o:" & nimgrepExe & " tools/nimgrep.nim" diff --git a/tools/nimsuggest/crashtester.nim b/nimsuggest/crashtester.nim similarity index 100% rename from tools/nimsuggest/crashtester.nim rename to nimsuggest/crashtester.nim diff --git a/tools/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim similarity index 99% rename from tools/nimsuggest/nimsuggest.nim rename to nimsuggest/nimsuggest.nim index 1798ac4e99..ee1647fbf1 100644 --- a/tools/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -174,6 +174,7 @@ proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int; else: msgs.setDirtyFile(dirtyIdx, nil) gTrackPos = newLineInfo(dirtyIdx, line, col) + gTrackPosAttached = false gErrorCounter = 0 if suggestVersion < 2: graph.usageSym = nil diff --git a/tools/nimsuggest/nimsuggest.nim.cfg b/nimsuggest/nimsuggest.nim.cfg similarity index 100% rename from tools/nimsuggest/nimsuggest.nim.cfg rename to nimsuggest/nimsuggest.nim.cfg diff --git a/tools/nimsuggest/nimsuggest.nimble b/nimsuggest/nimsuggest.nimble similarity index 100% rename from tools/nimsuggest/nimsuggest.nimble rename to nimsuggest/nimsuggest.nimble diff --git a/tools/nimsuggest/sexp.nim b/nimsuggest/sexp.nim similarity index 100% rename from tools/nimsuggest/sexp.nim rename to nimsuggest/sexp.nim diff --git a/tools/nimsuggest/tester.nim b/nimsuggest/tester.nim similarity index 100% rename from tools/nimsuggest/tester.nim rename to nimsuggest/tester.nim diff --git a/tools/nimsuggest/tests/dep_v1.nim b/nimsuggest/tests/dep_v1.nim similarity index 100% rename from tools/nimsuggest/tests/dep_v1.nim rename to nimsuggest/tests/dep_v1.nim diff --git a/tools/nimsuggest/tests/dep_v2.nim b/nimsuggest/tests/dep_v2.nim similarity index 100% rename from tools/nimsuggest/tests/dep_v2.nim rename to nimsuggest/tests/dep_v2.nim diff --git a/tools/nimsuggest/tests/tchk1.nim b/nimsuggest/tests/tchk1.nim similarity index 100% rename from tools/nimsuggest/tests/tchk1.nim rename to nimsuggest/tests/tchk1.nim diff --git a/tools/nimsuggest/tests/tcursor_at_end.nim b/nimsuggest/tests/tcursor_at_end.nim similarity index 100% rename from tools/nimsuggest/tests/tcursor_at_end.nim rename to nimsuggest/tests/tcursor_at_end.nim diff --git a/tools/nimsuggest/tests/tdef1.nim b/nimsuggest/tests/tdef1.nim similarity index 100% rename from tools/nimsuggest/tests/tdef1.nim rename to nimsuggest/tests/tdef1.nim diff --git a/tools/nimsuggest/tests/tdot1.nim b/nimsuggest/tests/tdot1.nim similarity index 100% rename from tools/nimsuggest/tests/tdot1.nim rename to nimsuggest/tests/tdot1.nim diff --git a/tools/nimsuggest/tests/tdot2.nim b/nimsuggest/tests/tdot2.nim similarity index 100% rename from tools/nimsuggest/tests/tdot2.nim rename to nimsuggest/tests/tdot2.nim diff --git a/tools/nimsuggest/tests/tdot3.nim b/nimsuggest/tests/tdot3.nim similarity index 100% rename from tools/nimsuggest/tests/tdot3.nim rename to nimsuggest/tests/tdot3.nim diff --git a/tools/nimsuggest/tests/tinclude.nim b/nimsuggest/tests/tinclude.nim similarity index 100% rename from tools/nimsuggest/tests/tinclude.nim rename to nimsuggest/tests/tinclude.nim diff --git a/tools/nimsuggest/tests/tno_deref.nim b/nimsuggest/tests/tno_deref.nim similarity index 100% rename from tools/nimsuggest/tests/tno_deref.nim rename to nimsuggest/tests/tno_deref.nim diff --git a/tools/nimsuggest/tests/tstrutils.nim b/nimsuggest/tests/tstrutils.nim similarity index 100% rename from tools/nimsuggest/tests/tstrutils.nim rename to nimsuggest/tests/tstrutils.nim diff --git a/tools/nimsuggest/tests/tsug_regression.nim b/nimsuggest/tests/tsug_regression.nim similarity index 100% rename from tools/nimsuggest/tests/tsug_regression.nim rename to nimsuggest/tests/tsug_regression.nim diff --git a/tools/nimsuggest/tests/twithin_macro.nim b/nimsuggest/tests/twithin_macro.nim similarity index 100% rename from tools/nimsuggest/tests/twithin_macro.nim rename to nimsuggest/tests/twithin_macro.nim diff --git a/tools/nimsuggest/tests/twithin_macro_prefix.nim b/nimsuggest/tests/twithin_macro_prefix.nim similarity index 100% rename from tools/nimsuggest/tests/twithin_macro_prefix.nim rename to nimsuggest/tests/twithin_macro_prefix.nim