minor lexer cleanups; fixes #2504

compiler/lexer.nim

@@ -131,24 +131,10 @@ type
 
 var gLinesCompiled*: int  # all lines that have been compiled
 
-proc isKeyword*(kind: TTokType): bool
-proc openLexer*(lex: var TLexer, fileidx: int32, inputstream: PLLStream)
-proc rawGetTok*(L: var TLexer, tok: var TToken)
-  # reads in the next token into tok and skips it
-
 proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
   newLineInfo(L.fileIdx, tok.line, tok.col)
 
-proc closeLexer*(lex: var TLexer)
-proc printTok*(tok: TToken)
-proc tokToStr*(tok: TToken): string
-
-proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream) =
-  openLexer(lex, filename.fileInfoIdx, inputstream)
-
-proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "")
-
-proc isKeyword(kind: TTokType): bool =
+proc isKeyword*(kind: TTokType): bool =
   result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
 
 proc isNimIdentifier*(s: string): bool =
@@ -206,14 +192,17 @@ proc fillToken(L: var TToken) =
   L.base = base10
   L.ident = dummyIdent
 
-proc openLexer(lex: var TLexer, fileIdx: int32, inputstream: PLLStream) =
+proc openLexer*(lex: var TLexer, fileIdx: int32, inputstream: PLLStream) =
   openBaseLexer(lex, inputstream)
   lex.fileIdx = fileidx
   lex.indentAhead = - 1
   lex.currLineIndent = 0
   inc(lex.lineNumber, inputstream.lineOffset)
 
-proc closeLexer(lex: var TLexer) =
+proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream) =
+  openLexer(lex, filename.fileInfoIdx, inputstream)
+
+proc closeLexer*(lex: var TLexer) =
   inc(gLinesCompiled, lex.lineNumber)
   closeBaseLexer(lex)
 
@@ -229,7 +218,7 @@ proc dispMessage(L: TLexer; info: TLineInfo; msg: TMsgKind; arg: string) =
   else:
     L.errorHandler(info, msg, arg)
 
-proc lexMessage(L: TLexer, msg: TMsgKind, arg = "") =
+proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "") =
   L.dispMessage(getLineInfo(L), msg, arg)
 
 proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
@@ -785,7 +774,7 @@ proc skip(L: var TLexer, tok: var TToken) =
       break                   # EndOfFile also leaves the loop
   L.bufpos = pos
 
-proc rawGetTok(L: var TLexer, tok: var TToken) =
+proc rawGetTok*(L: var TLexer, tok: var TToken) =
   fillToken(tok)
   if L.indentAhead >= 0:
     tok.indent = L.indentAhead

compiler/parser.nim

@@ -2031,8 +2031,8 @@ proc parseString*(s: string; filename: string = ""; line: int = 0;
   var parser: TParser
   # XXX for now the builtin 'parseStmt/Expr' functions do not know about strong
   # spaces...
-  openParser(parser, filename, stream, false)
   parser.lex.errorHandler = errorHandler
+  openParser(parser, filename, stream, false)
 
   result = parser.parseAll
   closeParser(parser)

tests/macros/ttryparseexpr.nim

@@ -15,5 +15,6 @@ const
   valid = 45
   a = test("foo&&")
   b = test("valid")
+  c = test("\"") # bug #2504
 
 echo a, " ", b