mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-06-05 11:24:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

implemented extended quoting rules

Browse Source
This commit is contained in:
Andreas Rumpf
2010-01-16 23:16:29 +01:00
parent 0bcdab8395
commit 6c5693e633
11 changed files with 317 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
doc/manual.txt
View File

@@ -215,7 +215,15 @@ String literals can also be delimited by three double quotes
Literals in this form may run for several lines, may contain ``"`` and do not
interpret any escape sequences.
For convenience, when the opening ``"""`` is immediately followed by a newline,
the newline is not included in the string.
the newline is not included in the string. The ending of the string literal is
defined by the pattern ``"""[^"]``, so this:
.. code-block:: nimrod
""""long string within quotes""""
Produces::
"long string within quotes"
Raw string literals
@@ -230,6 +238,19 @@ convenient for regular expressions or Windows paths:
var f = openFile(r"C:\texts\text.txt") # a raw string, so ``\t`` is no tab
To produce a single ``"`` within a raw string literal, it has to be doubled:
.. code-block:: nimrod
r"a""b"
Produces::
a"b
``r""""`` is not possible with this notation, because the three leading
quotes introduce a triple quoted string literal.
Generalized raw string literals
-------------------------------
@@ -291,8 +312,8 @@ prefix), binary (prefix ``0b``), octal (prefix ``0o``) and hexadecimal
There exists a literal for each numerical type that is
defined. The suffix starting with an apostrophe ('\'') is called a
`type suffix`:idx:. Literals without a type prefix are of the type ``int``,
unless the literal contains a dot or an ``E`` in which case it is of
`type suffix`:idx:. Literals without a type suffix are of the type ``int``,
unless the literal contains a dot or ``E|e`` in which case it is of
type ``float``.
The type suffixes are:

6
doc/nimrodc.txt
View File

@@ -42,12 +42,16 @@ Configuration file
The default configuration file is ``nimrod.cfg``. The ``nimrod`` executable
looks for it in the following directories (in this order):
1. ``/home/$user/.config/nimrod.cfg`` (UNIX) or ``$APPDATA/nimrod.cfg`` (Windows)
1. ``/home/$user/.config/nimrod.cfg`` (UNIX) or ``%APPDATA%/nimrod.cfg`` (Windows)
2. ``$nimrod/config/nimrod.cfg`` (UNIX, Windows)
3. ``/etc/nimrod.cfg`` (UNIX)
The search stops as soon as a configuration file has been found. The reading
of ``nimrod.cfg`` can be suppressed by the ``--skipCfg`` command line option.
**Note:** The *project file name* is the name of the ``.nim`` file that is
passed as a command line argument to the compiler.
Configuration settings can be overwritten in a project specific
configuration file that is read automatically. This specific file has to
be in the same directory as the project and be of the same name, except

2
lib/pure/md5.nim
View File

@@ -1,7 +1,7 @@
#
#
# Nimrod's Runtime Library
# (c) Copyright 2009 Andreas Rumpf
# (c) Copyright 2010 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.

76
rod/highlite.nim
View File

@@ -1,7 +1,7 @@
#
#
# The Nimrod Compiler
# (c) Copyright 2008 Andreas Rumpf
# (c) Copyright 2010 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
@@ -57,13 +57,12 @@ proc getSourceLanguage(name: string): TSourceLanguage =
result = langNone
proc initGeneralTokenizer(g: var TGeneralTokenizer, buf: string) =
var pos: int
g.buf = cstring(buf)
g.kind = low(TTokenClass)
g.start = 0
g.length = 0
g.state = low(TTokenClass)
pos = 0 # skip initial whitespace:
var pos = 0 # skip initial whitespace:
while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos)
g.pos = pos
@@ -71,8 +70,7 @@ proc deinitGeneralTokenizer(g: var TGeneralTokenizer) =
nil
proc nimGetKeyword(id: string): TTokenClass =
var i: PIdent
i = getIdent(id)
var i = getIdent(id)
if (i.id >= ord(tokKeywordLow) - ord(tkSymbol)) and
(i.id <= ord(tokKeywordHigh) - ord(tkSymbol)):
result = gtKeyword
@@ -80,8 +78,7 @@ proc nimGetKeyword(id: string): TTokenClass =
result = gtIdentifier
proc nimNumberPostfix(g: var TGeneralTokenizer, position: int): int =
var pos: int
pos = position
var pos = position
if g.buf[pos] == '\'':
inc(pos)
case g.buf[pos]
@@ -99,10 +96,8 @@ proc nimNumberPostfix(g: var TGeneralTokenizer, position: int): int =
result = pos
proc nimNumber(g: var TGeneralTokenizer, position: int): int =
const
decChars = {'0'..'9', '_'}
var pos: int
pos = position
const decChars = {'0'..'9', '_'}
var pos = position
g.kind = gtDecNumber
while g.buf[pos] in decChars: inc(pos)
if g.buf[pos] == '.':
@@ -121,10 +116,7 @@ proc nimNextToken(g: var TGeneralTokenizer) =
hexChars = {'0'..'9', 'A'..'F', 'a'..'f', '_'}
octChars = {'0'..'7', '_'}
binChars = {'0'..'1', '_'}
var
pos: int
id: string
pos = g.pos
var pos = g.pos
g.start = g.pos
if g.state == gtStringLit:
g.kind = gtStringLit
@@ -161,7 +153,7 @@ proc nimNextToken(g: var TGeneralTokenizer) =
g.kind = gtComment
while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos)
of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF':
id = ""
var id = ""
while g.buf[pos] in scanner.SymChars + {'_'}:
add(id, g.buf[pos])
inc(pos)
@@ -175,14 +167,17 @@ proc nimNextToken(g: var TGeneralTokenizer) =
break
of '\"':
inc(pos)
if (g.buf[pos] == '\"') and (g.buf[pos + 1] == '\"'):
if g.buf[pos] == '\"' and g.buf[pos+1] == '\"' and
g.buf[pos+2] != '\"':
inc(pos, 2)
break
else: inc(pos)
else:
g.kind = gtRawData
inc(pos)
while not (g.buf[pos] in {'\0', '\"', '\x0A', '\x0D'}): inc(pos)
while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}):
if g.buf[pos] == '"' and g.buf[pos+1] != '"': break
inc(pos)
if g.buf[pos] == '\"': inc(pos)
else:
g.kind = nimGetKeyword(id)
@@ -228,7 +223,8 @@ proc nimNextToken(g: var TGeneralTokenizer) =
break
of '\"':
inc(pos)
if (g.buf[pos] == '\"') and (g.buf[pos + 1] == '\"'):
if g.buf[pos] == '\"' and g.buf[pos+1] == '\"' and
g.buf[pos+2] != '\"':
inc(pos, 2)
break
else: inc(pos)
@@ -263,10 +259,8 @@ proc nimNextToken(g: var TGeneralTokenizer) =
g.pos = pos
proc generalNumber(g: var TGeneralTokenizer, position: int): int =
const
decChars = {'0'..'9'}
var pos: int
pos = position
const decChars = {'0'..'9'}
var pos = position
g.kind = gtDecNumber
while g.buf[pos] in decChars: inc(pos)
if g.buf[pos] == '.':
@@ -284,12 +278,9 @@ proc generalStrLit(g: var TGeneralTokenizer, position: int): int =
const
decChars = {'0'..'9'}
hexChars = {'0'..'9', 'A'..'F', 'a'..'f'}
var
pos: int
c: Char
pos = position
var pos = position
g.kind = gtStringLit
c = g.buf[pos]
var c = g.buf[pos]
inc(pos) # skip " or '
while true:
case g.buf[pos]
@@ -316,12 +307,11 @@ proc generalStrLit(g: var TGeneralTokenizer, position: int): int =
result = pos
proc isKeyword(x: openarray[string], y: string): int =
var a, b, mid, c: int
a = 0
b = len(x) - 1
var a = 0
var b = len(x) - 1
while a <= b:
mid = (a + b) div 2
c = cmp(x[mid], y)
var mid = (a + b) div 2
var c = cmp(x[mid], y)
if c < 0:
a = mid + 1
elif c > 0:
@@ -331,12 +321,11 @@ proc isKeyword(x: openarray[string], y: string): int =
result = - 1
proc isKeywordIgnoreCase(x: openarray[string], y: string): int =
var a, b, mid, c: int
a = 0
b = len(x) - 1
var a = 0
var b = len(x) - 1
while a <= b:
mid = (a + b) div 2
c = cmpIgnoreCase(x[mid], y)
var mid = (a + b) div 2
var c = cmpIgnoreCase(x[mid], y)
if c < 0:
a = mid + 1
elif c > 0:
@@ -357,10 +346,7 @@ proc clikeNextToken(g: var TGeneralTokenizer, keywords: openarray[string],
octChars = {'0'..'7'}
binChars = {'0'..'1'}
symChars = {'A'..'Z', 'a'..'z', '0'..'9', '_', '\x80'..'\xFF'}
var
pos, nested: int
id: string
pos = g.pos
var pos = g.pos
g.start = g.pos
if g.state == gtStringLit:
g.kind = gtStringLit
@@ -400,7 +386,7 @@ proc clikeNextToken(g: var TGeneralTokenizer, keywords: openarray[string],
while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos)
elif g.buf[pos] == '*':
g.kind = gtLongComment
nested = 0
var nested = 0
inc(pos)
while true:
case g.buf[pos]
@@ -426,7 +412,7 @@ proc clikeNextToken(g: var TGeneralTokenizer, keywords: openarray[string],
else:
g.kind = gtOperator
of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF':
id = ""
var id = ""
while g.buf[pos] in SymChars:
add(id, g.buf[pos])
inc(pos)
@@ -542,4 +528,4 @@ proc getNextToken(g: var TGeneralTokenizer, lang: TSourceLanguage) =
of langC: cNextToken(g)
of langJava: javaNextToken(g)
else: InternalError("getNextToken")

12
rod/lexbase.nim
View File

@@ -1,7 +1,7 @@
#
#
# The Nimrod Compiler
# (c) Copyright 2008 Andreas Rumpf
# (c) Copyright 2010 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
@@ -83,7 +83,8 @@ proc FillBuffer(L: var TBaseLexer) =
toCopy = L.BufLen - L.sentinel - 1
assert(toCopy >= 0)
if toCopy > 0:
MoveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize) # "moveMem" handles overlapping regions
MoveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize)
# "moveMem" handles overlapping regions
charsRead = LLStreamRead(L.stream, addr(L.buf[toCopy]),
(L.sentinel + 1) * chrSize) div chrSize
s = toCopy + charsRead
@@ -142,7 +143,7 @@ proc skip_UTF_8_BOM(L: var TBaseLexer) =
inc(L.bufpos, 3)
inc(L.lineStart, 3)
proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen: int = 8192) =
proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) =
assert(bufLen > 0)
L.bufpos = 0
L.bufLen = bufLen
@@ -158,13 +159,12 @@ proc getColNumber(L: TBaseLexer, pos: int): int =
result = abs(pos - L.lineStart)
proc getCurrentLine(L: TBaseLexer, marker: bool = true): string =
var i: int
result = ""
i = L.lineStart
var i = L.lineStart
while not (L.buf[i] in {CR, LF, EndOfFile}):
add(result, L.buf[i])
inc(i)
result = result & "\n"
if marker:
result = result & RepeatChar(getColNumber(L, L.bufpos)) & '^' & "\n"

3
rod/pnimsyn.nim
View File

@@ -113,9 +113,8 @@ proc parLineInfo(p: TParser): TLineInfo =
result = getLineInfo(p.lex^)
proc indAndComment(p: var TParser, n: PNode) =
var info: TLineInfo
if p.tok.tokType == tkInd:
info = parLineInfo(p)
var info = parLineInfo(p)
getTok(p)
if p.tok.tokType == tkComment: skipComment(p, n)
else: liMessage(info, errInvalidIndentation)

395
rod/rnimsyn.nim
View File

@@ -1,7 +1,7 @@
#
#
# The Nimrod Compiler
# (c) Copyright 2009 Andreas Rumpf
# (c) Copyright 2010 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
@@ -51,8 +51,8 @@ const
LineCommentColumn = 30
proc InitSrcGen(g: var TSrcGen, renderFlags: TRenderFlags) =
g.comStack = @ []
g.tokens = @ []
g.comStack = @[]
g.tokens = @[]
g.indent = 0
g.lineLen = 0
g.pos = 0
@@ -62,8 +62,7 @@ proc InitSrcGen(g: var TSrcGen, renderFlags: TRenderFlags) =
g.pendingNL = - 1
proc addTok(g: var TSrcGen, kind: TTokType, s: string) =
var length: int
length = len(g.tokens)
var length = len(g.tokens)
setlen(g.tokens, length + 1)
g.tokens[length].kind = kind
g.tokens[length].length = int16(len(s))
@@ -128,15 +127,11 @@ proc makeNimString(s: string): string =
add(result, '\"')
proc putComment(g: var TSrcGen, s: string) =
var
i, j, ind, comIndent: int
isCode: bool
com: string
i = 0
comIndent = 1
isCode = (len(s) >= 2) and (s[0 + 1] != ' ')
ind = g.lineLen
com = ""
var i = 0
var comIndent = 1
var isCode = (len(s) >= 2) and (s[0 + 1] != ' ')
var ind = g.lineLen
var com = ""
while true:
case s[i]
of '\0':
@@ -167,7 +162,7 @@ proc putComment(g: var TSrcGen, s: string) =
# we may break the comment into a multi-line comment if the line
# gets too long:
# compute length of the following word:
j = i
var j = i
while s[j] > ' ': inc(j)
if not isCode and (g.lineLen + (j - i) > MaxLineLen):
put(g, tkComment, com)
@@ -181,10 +176,9 @@ proc putComment(g: var TSrcGen, s: string) =
optNL(g)
proc maxLineLength(s: string): int =
var i, linelen: int
result = 0
i = 0
lineLen = 0
var i = 0
var lineLen = 0
while true:
case s[i]
of '\0':
@@ -203,12 +197,9 @@ proc maxLineLength(s: string): int =
inc(i)
proc putRawStr(g: var TSrcGen, kind: TTokType, s: string) =
var
i, hi: int
str: string
i = 0
hi = len(s) + 0 - 1
str = ""
var i = 0
var hi = len(s) + 0 - 1
var str = ""
while i <= hi:
case s[i]
of '\x0D':
@@ -237,8 +228,7 @@ proc containsNL(s: string): bool =
result = false
proc pushCom(g: var TSrcGen, n: PNode) =
var length: int
length = len(g.comStack)
var length = len(g.comStack)
setlen(g.comStack, length + 1)
g.comStack[length] = n
@@ -258,15 +248,15 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool =
(renderDocComments in g.flags) and startsWith(n.comment, "##")
proc gcom(g: var TSrcGen, n: PNode) =
var ml: int
assert(n != nil)
if shouldRenderComment(g, n):
if (g.pendingNL < 0) and (len(g.buf) > 0) and (g.buf[len(g.buf)] != ' '):
put(g, tkSpaces, Space) # Before long comments we cannot make sure that a newline is generated,
# because this might be wrong. But it is no problem in practice.
put(g, tkSpaces, Space)
# Before long comments we cannot make sure that a newline is generated,
# because this might be wrong. But it is no problem in practice.
if (g.pendingNL < 0) and (len(g.buf) > 0) and
(g.lineLen < LineCommentColumn):
ml = maxLineLength(n.comment)
var ml = maxLineLength(n.comment)
if ml + LineCommentColumn <= maxLineLen:
put(g, tkSpaces, repeatChar(LineCommentColumn - g.lineLen))
putComment(g, n.comment) #assert(g.comStack[high(g.comStack)] = n);
@@ -285,30 +275,18 @@ proc litAux(n: PNode, x: biggestInt, size: int): string =
proc atom(n: PNode): string =
var f: float32
case n.kind
of nkEmpty:
result = ""
of nkIdent:
result = n.ident.s
of nkSym:
result = n.sym.name.s
of nkStrLit:
result = makeNimString(n.strVal)
of nkRStrLit:
result = "r\"" & n.strVal & '\"'
of nkTripleStrLit:
result = "\"\"\"" & n.strVal & "\"\"\""
of nkCharLit:
result = '\'' & toNimChar(chr(int(n.intVal))) & '\''
of nkIntLit:
result = litAux(n, n.intVal, 4)
of nkInt8Lit:
result = litAux(n, n.intVal, 1) & "\'i8"
of nkInt16Lit:
result = litAux(n, n.intVal, 2) & "\'i16"
of nkInt32Lit:
result = litAux(n, n.intVal, 4) & "\'i32"
of nkInt64Lit:
result = litAux(n, n.intVal, 8) & "\'i64"
of nkEmpty: result = ""
of nkIdent: result = n.ident.s
of nkSym: result = n.sym.name.s
of nkStrLit: result = makeNimString(n.strVal)
of nkRStrLit: result = "r\"" & replace(n.strVal, "\"", "\"\"") & '\"'
of nkTripleStrLit: result = "\"\"\"" & n.strVal & "\"\"\""
of nkCharLit: result = '\'' & toNimChar(chr(int(n.intVal))) & '\''
of nkIntLit: result = litAux(n, n.intVal, 4)
of nkInt8Lit: result = litAux(n, n.intVal, 1) & "\'i8"
of nkInt16Lit: result = litAux(n, n.intVal, 2) & "\'i16"
of nkInt32Lit: result = litAux(n, n.intVal, 4) & "\'i32"
of nkInt64Lit: result = litAux(n, n.intVal, 8) & "\'i64"
of nkFloatLit:
if n.flags * {nfBase2, nfBase8, nfBase16} == {}: result = $(n.floatVal)
else: result = litAux(n, (cast[PInt64](addr(n.floatVal)))^ , 8)
@@ -323,8 +301,7 @@ proc atom(n: PNode): string =
result = $(n.floatVal) & "\'f64"
else:
result = litAux(n, (cast[PInt64](addr(n.floatVal)))^ , 8) & "\'f64"
of nkNilLit:
result = "nil"
of nkNilLit: result = "nil"
of nkType:
if (n.typ != nil) and (n.typ.sym != nil): result = n.typ.sym.name.s
else: result = "[type node]"
@@ -346,142 +323,87 @@ proc lsons(n: PNode, start: int = 0, theEnd: int = - 1): int =
proc lsub(n: PNode): int =
# computes the length of a tree
var L: int
if n == nil:
return 0
if n.comment != nil:
return maxLineLen + 1
if n == nil: return 0
if n.comment != nil: return maxLineLen + 1
case n.kind
of nkTripleStrLit:
if containsNL(n.strVal): result = maxLineLen + 1
else: result = len(atom(n))
of nkEmpty..pred(nkTripleStrLit), succ(nkTripleStrLit)..nkNilLit:
result = len(atom(n))
of nkCall, nkBracketExpr, nkConv:
result = lsub(n.sons[0]) + lcomma(n, 1) + 2
of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv:
result = lsub(n.sons[1])
of nkCast:
result = lsub(n.sons[0]) + lsub(n.sons[1]) + len("cast[]()")
of nkAddr:
result = lsub(n.sons[0]) + len("addr()")
of nkHiddenAddr, nkHiddenDeref:
result = lsub(n.sons[0])
of nkCommand:
result = lsub(n.sons[0]) + lcomma(n, 1) + 1
of nkExprEqExpr, nkAsgn, nkFastAsgn:
result = lsons(n) + 3
of nkPar, nkCurly, nkBracket:
result = lcomma(n) + 2
of nkSymChoice:
result = lsons(n) + len("()") + sonsLen(n) - 1
of nkTupleTy:
result = lcomma(n) + len("tuple[]")
of nkDotExpr:
result = lsons(n) + 1
of nkBind:
result = lsons(n) + len("bind_")
of nkCheckedFieldExpr:
result = lsub(n.sons[0])
of nkLambda:
result = lsons(n) + len("lambda__=_")
of nkCall, nkBracketExpr, nkConv: result = lsub(n.sons[0]) + lcomma(n, 1) + 2
of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: result = lsub(n[1])
of nkCast: result = lsub(n.sons[0]) + lsub(n.sons[1]) + len("cast[]()")
of nkAddr: result = lsub(n.sons[0]) + len("addr()")
of nkHiddenAddr, nkHiddenDeref: result = lsub(n.sons[0])
of nkCommand: result = lsub(n.sons[0]) + lcomma(n, 1) + 1
of nkExprEqExpr, nkAsgn, nkFastAsgn: result = lsons(n) + 3
of nkPar, nkCurly, nkBracket: result = lcomma(n) + 2
of nkSymChoice: result = lsons(n) + len("()") + sonsLen(n) - 1
of nkTupleTy: result = lcomma(n) + len("tuple[]")
of nkDotExpr: result = lsons(n) + 1
of nkBind: result = lsons(n) + len("bind_")
of nkCheckedFieldExpr: result = lsub(n.sons[0])
of nkLambda: result = lsons(n) + len("lambda__=_")
of nkConstDef, nkIdentDefs:
result = lcomma(n, 0, - 3)
L = sonsLen(n)
var L = sonsLen(n)
if n.sons[L - 2] != nil: result = result + lsub(n.sons[L - 2]) + 2
if n.sons[L - 1] != nil: result = result + lsub(n.sons[L - 1]) + 3
of nkVarTuple:
result = lcomma(n, 0, - 3) + len("() = ") + lsub(lastSon(n))
of nkChckRangeF:
result = len("chckRangeF") + 2 + lcomma(n)
of nkChckRange64:
result = len("chckRange64") + 2 + lcomma(n)
of nkChckRange:
result = len("chckRange") + 2 + lcomma(n)
of nkVarTuple: result = lcomma(n, 0, - 3) + len("() = ") + lsub(lastSon(n))
of nkChckRangeF: result = len("chckRangeF") + 2 + lcomma(n)
of nkChckRange64: result = len("chckRange64") + 2 + lcomma(n)
of nkChckRange: result = len("chckRange") + 2 + lcomma(n)
of nkObjDownConv, nkObjUpConv, nkStringToCString, nkCStringToString,
nkPassAsOpenArray:
result = 2
if sonsLen(n) >= 1: result = result + lsub(n.sons[0])
result = result + lcomma(n, 1)
of nkExprColonExpr:
result = lsons(n) + 2
of nkInfix:
result = lsons(n) + 2
of nkPrefix:
result = lsons(n) + 1
of nkPostfix:
result = lsons(n)
of nkCallStrLit:
result = lsons(n)
of nkPragmaExpr:
result = lsub(n.sons[0]) + lcomma(n, 1)
of nkRange:
result = lsons(n) + 2
of nkDerefExpr:
result = lsub(n.sons[0]) + 2
of nkAccQuoted:
result = lsub(n.sons[0]) + 2
of nkExprColonExpr: result = lsons(n) + 2
of nkInfix: result = lsons(n) + 2
of nkPrefix: result = lsons(n) + 1
of nkPostfix: result = lsons(n)
of nkCallStrLit: result = lsons(n)
of nkPragmaExpr: result = lsub(n.sons[0]) + lcomma(n, 1)
of nkRange: result = lsons(n) + 2
of nkDerefExpr: result = lsub(n.sons[0]) + 2
of nkAccQuoted: result = lsub(n.sons[0]) + 2
of nkIfExpr:
result = lsub(n.sons[0].sons[0]) + lsub(n.sons[0].sons[1]) + lsons(n, 1) +
len("if_:_")
of nkElifExpr:
result = lsons(n) + len("_elif_:_")
of nkElseExpr:
result = lsub(n.sons[0]) + len("_else:_") # type descriptions
of nkTypeOfExpr:
result = lsub(n.sons[0]) + len("type_")
of nkRefTy:
result = lsub(n.sons[0]) + len("ref_")
of nkPtrTy:
result = lsub(n.sons[0]) + len("ptr_")
of nkVarTy:
result = lsub(n.sons[0]) + len("var_")
of nkDistinctTy:
result = lsub(n.sons[0]) + len("Distinct_")
of nkTypeDef:
result = lsons(n) + 3
of nkOfInherit:
result = lsub(n.sons[0]) + len("of_")
of nkProcTy:
result = lsons(n) + len("proc_")
of nkEnumTy:
result = lsub(n.sons[0]) + lcomma(n, 1) + len("enum_")
of nkEnumFieldDef:
result = lsons(n) + 3
of nkElifExpr: result = lsons(n) + len("_elif_:_")
of nkElseExpr: result = lsub(n.sons[0]) + len("_else:_") # type descriptions
of nkTypeOfExpr: result = lsub(n.sons[0]) + len("type_")
of nkRefTy: result = lsub(n.sons[0]) + len("ref_")
of nkPtrTy: result = lsub(n.sons[0]) + len("ptr_")
of nkVarTy: result = lsub(n.sons[0]) + len("var_")
of nkDistinctTy: result = lsub(n.sons[0]) + len("Distinct_")
of nkTypeDef: result = lsons(n) + 3
of nkOfInherit: result = lsub(n.sons[0]) + len("of_")
of nkProcTy: result = lsons(n) + len("proc_")
of nkEnumTy: result = lsub(n.sons[0]) + lcomma(n, 1) + len("enum_")
of nkEnumFieldDef: result = lsons(n) + 3
of nkVarSection:
if sonsLen(n) > 1: result = maxLineLen + 1
else: result = lsons(n) + len("var_")
of nkReturnStmt:
result = lsub(n.sons[0]) + len("return_")
of nkRaiseStmt:
result = lsub(n.sons[0]) + len("raise_")
of nkYieldStmt:
result = lsub(n.sons[0]) + len("yield_")
of nkDiscardStmt:
result = lsub(n.sons[0]) + len("discard_")
of nkBreakStmt:
result = lsub(n.sons[0]) + len("break_")
of nkContinueStmt:
result = lsub(n.sons[0]) + len("continue_")
of nkPragma:
result = lcomma(n) + 4
of nkCommentStmt:
result = len(n.comment)
of nkOfBranch:
result = lcomma(n, 0, - 2) + lsub(lastSon(n)) + len("of_:_")
of nkElifBranch:
result = lsons(n) + len("elif_:_")
of nkElse:
result = lsub(n.sons[0]) + len("else:_")
of nkFinally:
result = lsub(n.sons[0]) + len("finally:_")
of nkGenericParams:
result = lcomma(n) + 2
of nkReturnStmt: result = lsub(n.sons[0]) + len("return_")
of nkRaiseStmt: result = lsub(n.sons[0]) + len("raise_")
of nkYieldStmt: result = lsub(n.sons[0]) + len("yield_")
of nkDiscardStmt: result = lsub(n.sons[0]) + len("discard_")
of nkBreakStmt: result = lsub(n.sons[0]) + len("break_")
of nkContinueStmt: result = lsub(n.sons[0]) + len("continue_")
of nkPragma: result = lcomma(n) + 4
of nkCommentStmt: result = len(n.comment)
of nkOfBranch: result = lcomma(n, 0, - 2) + lsub(lastSon(n)) + len("of_:_")
of nkElifBranch: result = lsons(n) + len("elif_:_")
of nkElse: result = lsub(n.sons[0]) + len("else:_")
of nkFinally: result = lsub(n.sons[0]) + len("finally:_")
of nkGenericParams: result = lcomma(n) + 2
of nkFormalParams:
result = lcomma(n, 1) + 2
if n.sons[0] != nil: result = result + lsub(n.sons[0]) + 2
of nkExceptBranch:
result = lcomma(n, 0, - 2) + lsub(lastSon(n)) + len("except_:_")
result = lcomma(n, 0, -2) + lsub(lastSon(n)) + len("except_:_")
else: result = maxLineLen + 1
proc fits(g: TSrcGen, x: int): bool =
@@ -509,15 +431,12 @@ proc gsub(g: var TSrcGen, n: PNode) =
proc hasCom(n: PNode): bool =
result = false
if n == nil: return
if n.comment != nil:
return true
if n.comment != nil: return true
case n.kind
of nkEmpty..nkNilLit:
nil
of nkEmpty..nkNilLit: nil
else:
for i in countup(0, sonsLen(n) - 1):
if hasCom(n.sons[i]):
return true
if hasCom(n.sons[i]): return true
proc putWithSpace(g: var TSrcGen, kind: TTokType, s: string) =
put(g, kind, s)
@@ -525,12 +444,9 @@ proc putWithSpace(g: var TSrcGen, kind: TTokType, s: string) =
proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
theEnd: int = - 1) =
var
sublen: int
c: bool
for i in countup(start, sonsLen(n) + theEnd):
c = i < sonsLen(n) + theEnd
sublen = lsub(n.sons[i]) + ord(c)
var c = i < sonsLen(n) + theEnd
var sublen = lsub(n.sons[i]) + ord(c)
if not fits(g, sublen) and (ind + sublen < maxLineLen): optNL(g, ind)
gsub(g, n.sons[i])
if c:
@@ -550,19 +466,17 @@ proc gcomma(g: var TSrcGen, n: PNode, c: TContext, start: int = 0,
gcommaAux(g, n, ind, start, theEnd)
proc gcomma(g: var TSrcGen, n: PNode, start: int = 0, theEnd: int = - 1) =
var ind: int
ind = g.lineLen
var ind = g.lineLen
if ind > maxLineLen div 2: ind = g.indent + longIndentWid
gcommaAux(g, n, ind, start, theEnd)
proc gsons(g: var TSrcGen, n: PNode, c: TContext, start: int = 0,
theEnd: int = - 1) =
for i in countup(start, sonsLen(n) + theEnd):
gsub(g, n.sons[i], c)
for i in countup(start, sonsLen(n) + theEnd): gsub(g, n.sons[i], c)
proc gsection(g: var TSrcGen, n: PNode, c: TContext, kind: TTokType, k: string) =
if sonsLen(n) == 0:
return # empty var sections are possible
proc gsection(g: var TSrcGen, n: PNode, c: TContext, kind: TTokType,
k: string) =
if sonsLen(n) == 0: return # empty var sections are possible
putWithSpace(g, kind, k)
gcoms(g)
indentNL(g)
@@ -600,7 +514,6 @@ proc gstmts(g: var TSrcGen, n: PNode, c: TContext) =
proc gif(g: var TSrcGen, n: PNode) =
var
c: TContext
length: int
gsub(g, n.sons[0].sons[0])
initContext(c)
putWithSpace(g, tkColon, ":")
@@ -608,7 +521,7 @@ proc gif(g: var TSrcGen, n: PNode) =
incl(c.flags, rfLongMode)
gcoms(g) # a good place for comments
gstmts(g, n.sons[0].sons[1], c)
length = sonsLen(n)
var length = sonsLen(n)
for i in countup(1, length - 1):
optNL(g)
gsub(g, n.sons[i], c)
@@ -636,10 +549,8 @@ proc gtry(g: var TSrcGen, n: PNode) =
gsons(g, n, c, 1)
proc gfor(g: var TSrcGen, n: PNode) =
var
c: TContext
length: int
length = sonsLen(n)
var c: TContext
var length = sonsLen(n)
putWithSpace(g, tkFor, "for")
initContext(c)
if longMode(n) or
@@ -665,18 +576,16 @@ proc gmacro(g: var TSrcGen, n: PNode) =
gsons(g, n, c, 1)
proc gcase(g: var TSrcGen, n: PNode) =
var
c: TContext
length, last: int
var c: TContext
initContext(c)
length = sonsLen(n)
if n.sons[length - 1].kind == nkElse: last = - 2
else: last = - 1
var length = sonsLen(n)
var last = if n.sons[length-1].kind == nkElse: -2 else: -1
if longMode(n, 0, last): incl(c.flags, rfLongMode)
putWithSpace(g, tkCase, "case")
gsub(g, n.sons[0])
gcoms(g)
optNL(g)
last = 0
gsons(g, n, c, 1, last)
if last == - 2:
initContext(c)
@@ -722,10 +631,8 @@ proc gasm(g: var TSrcGen, n: PNode) =
gsub(g, n.sons[1])
proc gident(g: var TSrcGen, n: PNode) =
var
s: string
t: TTokType
s = atom(n)
var t: TTokType
var s = atom(n)
if (s[0] in scanner.SymChars):
if (n.kind == nkIdent):
if (n.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
@@ -747,36 +654,21 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
if n == nil: return
if n.comment != nil: pushCom(g, n)
case n.kind # atoms:
of nkTripleStrLit:
putRawStr(g, tkTripleStrLit, n.strVal)
of nkEmpty, nkType:
put(g, tkInvalid, atom(n))
of nkSym, nkIdent:
gident(g, n)
of nkIntLit:
put(g, tkIntLit, atom(n))
of nkInt8Lit:
put(g, tkInt8Lit, atom(n))
of nkInt16Lit:
put(g, tkInt16Lit, atom(n))
of nkInt32Lit:
put(g, tkInt32Lit, atom(n))
of nkInt64Lit:
put(g, tkInt64Lit, atom(n))
of nkFloatLit:
put(g, tkFloatLit, atom(n))
of nkFloat32Lit:
put(g, tkFloat32Lit, atom(n))
of nkFloat64Lit:
put(g, tkFloat64Lit, atom(n))
of nkStrLit:
put(g, tkStrLit, atom(n))
of nkRStrLit:
put(g, tkRStrLit, atom(n))
of nkCharLit:
put(g, tkCharLit, atom(n))
of nkNilLit:
put(g, tkNil, atom(n)) # complex expressions
of nkTripleStrLit: putRawStr(g, tkTripleStrLit, n.strVal)
of nkEmpty, nkType: put(g, tkInvalid, atom(n))
of nkSym, nkIdent: gident(g, n)
of nkIntLit: put(g, tkIntLit, atom(n))
of nkInt8Lit: put(g, tkInt8Lit, atom(n))
of nkInt16Lit: put(g, tkInt16Lit, atom(n))
of nkInt32Lit: put(g, tkInt32Lit, atom(n))
of nkInt64Lit: put(g, tkInt64Lit, atom(n))
of nkFloatLit: put(g, tkFloatLit, atom(n))
of nkFloat32Lit: put(g, tkFloat32Lit, atom(n))
of nkFloat64Lit: put(g, tkFloat64Lit, atom(n))
of nkStrLit: put(g, tkStrLit, atom(n))
of nkRStrLit: put(g, tkRStrLit, atom(n))
of nkCharLit: put(g, tkCharLit, atom(n))
of nkNilLit: put(g, tkNil, atom(n)) # complex expressions
of nkCall, nkConv, nkDotCall:
if sonsLen(n) >= 1: gsub(g, n.sons[0])
put(g, tkParLe, "(")
@@ -785,11 +677,10 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
of nkCallStrLit:
gsub(g, n.sons[0])
if n.sons[1].kind == nkRStrLit:
put(g, tkRStrLit, '\"' & n.sons[1].strVal & '\"')
put(g, tkRStrLit, '\"' & replace(n[1].strVal, "\"", "\"\"") & '\"')
else:
gsub(g, n.sons[0])
of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv:
gsub(g, n.sons[0])
of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: gsub(g, n.sons[0])
of nkCast:
put(g, tkCast, "cast")
put(g, tkBracketLe, "[")
@@ -878,7 +769,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
gsub(g, n.sons[codePos])
of nkConstDef, nkIdentDefs:
gcomma(g, n, 0, - 3)
L = sonsLen(n)
var L = sonsLen(n)
if n.sons[L - 2] != nil:
putWithSpace(g, tkColon, ":")
gsub(g, n.sons[L - 2])
@@ -919,8 +810,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
gsub(g, n.sons[1])
of nkDerefExpr:
gsub(g, n.sons[0])
putWithSpace(g, tkHat, "^") # unfortunately this requires a space, because ^. would be
# only one operator
putWithSpace(g, tkHat, "^")
# unfortunately this requires a space, because ^. would be only one operator
of nkAccQuoted:
put(g, tkAccent, "`")
gsub(g, n.sons[0])
@@ -996,28 +887,20 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
put(g, tkSpaces, Space)
putWithSpace(g, tkEquals, "=")
gsub(g, n.sons[1])
of nkStmtList, nkStmtListExpr:
gstmts(g, n, emptyContext)
of nkStmtList, nkStmtListExpr: gstmts(g, n, emptyContext)
of nkIfStmt:
putWithSpace(g, tkIf, "if")
gif(g, n)
of nkWhenStmt, nkRecWhen:
putWithSpace(g, tkWhen, "when")
gif(g, n)
of nkWhileStmt:
gwhile(g, n)
of nkCaseStmt, nkRecCase:
gcase(g, n)
of nkMacroStmt:
gmacro(g, n)
of nkTryStmt:
gtry(g, n)
of nkForStmt:
gfor(g, n)
of nkBlockStmt, nkBlockExpr:
gblock(g, n)
of nkAsmStmt:
gasm(g, n)
of nkWhileStmt: gwhile(g, n)
of nkCaseStmt, nkRecCase: gcase(g, n)
of nkMacroStmt: gmacro(g, n)
of nkTryStmt: gtry(g, n)
of nkForStmt: gfor(g, n)
of nkBlockStmt, nkBlockExpr: gblock(g, n)
of nkAsmStmt: gasm(g, n)
of nkProcDef:
putWithSpace(g, tkProc, "proc")
gproc(g, n)
@@ -1170,8 +1053,7 @@ proc renderModule(n: PNode, filename: string, renderFlags: TRenderFlags = {}) =
if n.sons[i] != nil:
case n.sons[i].kind
of nkTypeSection, nkConstSection, nkVarSection, nkCommentStmt: putNL(g)
else:
nil
else: nil
gcoms(g)
if open(f, filename, fmWrite):
write(f, g.buf)
@@ -1182,10 +1064,9 @@ proc initTokRender(r: var TSrcGen, n: PNode, renderFlags: TRenderFlags = {}) =
gsub(r, n)
proc getNextTok(r: var TSrcGen, kind: var TTokType, literal: var string) =
var length: int
if r.idx < len(r.tokens):
kind = r.tokens[r.idx].kind
length = r.tokens[r.idx].length
var length = r.tokens[r.idx].length
literal = copy(r.buf, r.pos + 0, r.pos + 0 + length - 1)
inc(r.pos, length)
inc(r.idx)

169
rod/scanner.nim
View File

@@ -1,7 +1,7 @@
#
#
# The Nimrod Compiler
# (c) Copyright 2009 Andreas Rumpf
# (c) Copyright 2010 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
@@ -52,15 +52,18 @@ type
tkImplies, tkImport, tkIn, tkInclude, tkIs, tkIsnot, tkIterator, tkLambda,
tkMacro, tkMethod, tkMod, tkNil, tkNot, tkNotin, tkObject, tkOf, tkOr,
tkOut, tkProc, tkPtr, tkRaise, tkRef, tkReturn, tkShl, tkShr, tkTemplate,
tkTry, tkTuple, tkType, tkVar, tkWhen, tkWhile, tkWith, tkWithout, tkXor, tkYield, #[[[end]]]
tkTry, tkTuple, tkType, tkVar, tkWhen, tkWhile, tkWith, tkWithout, tkXor,
tkYield, #[[[end]]]
tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit, tkFloatLit,
tkFloat32Lit, tkFloat64Lit, tkStrLit, tkRStrLit, tkTripleStrLit,
tkCallRStrLit, tkCallTripleStrLit, tkCharLit, tkParLe, tkParRi, tkBracketLe,
tkBracketRi, tkCurlyLe, tkCurlyRi, tkBracketDotLe, tkBracketDotRi, # [. and .]
tkBracketRi, tkCurlyLe, tkCurlyRi,
tkBracketDotLe, tkBracketDotRi, # [. and .]
tkCurlyDotLe, tkCurlyDotRi, # {. and .}
tkParDotLe, tkParDotRi, # (. and .)
tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot, tkHat, tkOpr,
tkComment, tkAccent, tkInd, tkSad, tkDed, # pseudo token types used by the source renderers:
tkComment, tkAccent, tkInd, tkSad,
tkDed, # pseudo token types used by the source renderers:
tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
TTokTypes* = set[TTokType]
@@ -70,16 +73,18 @@ const
tokOperators*: TTokTypes = {tkOpr, tkSymbol, tkBracketLe, tkBracketRi, tkIn,
tkIs, tkIsNot, tkEquals, tkDot, tkHat, tkNot, tkAnd, tkOr, tkXor, tkShl,
tkShr, tkDiv, tkMod, tkNotIn}
TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]", "tkSymbol", #[[[cog
#cog.out(strings)
#]]]
TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]",
"tkSymbol", #[[[cog
#cog.out(strings)
#]]]
"addr", "and", "as", "asm", "bind", "block", "break", "case", "cast",
"const", "continue", "converter", "discard", "distinct", "div", "elif",
"else", "end", "enum", "except", "finally", "for", "from", "generic", "if",
"implies", "import", "in", "include", "is", "isnot", "iterator", "lambda",
"macro", "method", "mod", "nil", "not", "notin", "object", "of", "or",
"out", "proc", "ptr", "raise", "ref", "return", "shl", "shr", "template",
"try", "tuple", "type", "var", "when", "while", "with", "without", "xor", "yield", #[[[end]]]
"try", "tuple", "type", "var", "when", "while", "with", "without", "xor",
"yield", #[[[end]]]
"tkIntLit", "tkInt8Lit", "tkInt16Lit", "tkInt32Lit", "tkInt64Lit",
"tkFloatLit", "tkFloat32Lit", "tkFloat64Lit", "tkStrLit", "tkRStrLit",
"tkTripleStrLit", "tkCallRStrLit", "tkCallTripleStrLit", "tkCharLit", "(",
@@ -213,17 +218,13 @@ proc lexMessage(L: TLexer, msg: TMsgKind, arg: string = "") =
msgs.liMessage(getLineInfo(L), msg, arg)
proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg: string = "") =
var info: TLineInfo
info = newLineInfo(L.filename, L.linenumber, pos - L.lineStart)
var info = newLineInfo(L.filename, L.linenumber, pos - L.lineStart)
msgs.liMessage(info, msg, arg)
proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: TCharSet) =
# matches ([chars]_)*
var
pos: int
buf: cstring
pos = L.bufpos # use registers for pos, buf
buf = L.buf
var pos = L.bufpos # use registers for pos, buf
var buf = L.buf
while true:
if buf[pos] in chars:
add(tok.literal, buf[pos])
@@ -252,11 +253,12 @@ proc GetNumber(L: var TLexer): TToken =
result.tokType = tkIntLit # int literal until we know better
result.literal = ""
result.base = base10 # BUGFIX
pos = L.bufpos # make sure the literal is correct for error messages:
pos = L.bufpos # make sure the literal is correct for error messages:
matchUnderscoreChars(L, result, {'A'..'Z', 'a'..'z', '0'..'9'})
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
add(result.literal, '.')
inc(L.bufpos) #matchUnderscoreChars(L, result, ['A'..'Z', 'a'..'z', '0'..'9'])
inc(L.bufpos)
#matchUnderscoreChars(L, result, ['A'..'Z', 'a'..'z', '0'..'9'])
matchUnderscoreChars(L, result, {'0'..'9'})
if L.buf[L.bufpos] in {'e', 'E'}:
add(result.literal, 'e')
@@ -355,19 +357,15 @@ proc GetNumber(L: var TLexer): TToken =
else: break
else: InternalError(getLineInfo(L), "getNumber")
case result.tokType
of tkIntLit, tkInt64Lit:
result.iNumber = xi
of tkInt8Lit:
result.iNumber = biggestInt(int8(toU8(int(xi))))
of tkInt16Lit:
result.iNumber = biggestInt(toU16(int(xi)))
of tkInt32Lit:
result.iNumber = biggestInt(toU32(xi))
of tkIntLit, tkInt64Lit: result.iNumber = xi
of tkInt8Lit: result.iNumber = biggestInt(int8(toU8(int(xi))))
of tkInt16Lit: result.iNumber = biggestInt(toU16(int(xi)))
of tkInt32Lit: result.iNumber = biggestInt(toU32(xi))
of tkFloat32Lit:
result.fNumber = (cast[PFloat32](addr(xi)))^ # note: this code is endian neutral!
# XXX: Test this on big endian machine!
of tkFloat64Lit:
result.fNumber = (cast[PFloat64](addr(xi)))^
result.fNumber = (cast[PFloat32](addr(xi)))^
# note: this code is endian neutral!
# XXX: Test this on big endian machine!
of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))^
else: InternalError(getLineInfo(L), "getNumber")
elif isFloatLiteral(result.literal) or (result.tokType == tkFloat32Lit) or
(result.tokType == tkFloat64Lit):
@@ -380,12 +378,9 @@ proc GetNumber(L: var TLexer): TToken =
result.tokType = tkInt64Lit
elif result.tokType != tkInt64Lit:
lexMessage(L, errInvalidNumber, result.literal)
except EInvalidValue:
lexMessage(L, errInvalidNumber, result.literal)
except EOverflow:
lexMessage(L, errNumberOutOfRange, result.literal)
except EOutOfRange:
lexMessage(L, errNumberOutOfRange, result.literal)
except EInvalidValue: lexMessage(L, errInvalidNumber, result.literal)
except EOverflow: lexMessage(L, errNumberOutOfRange, result.literal)
except EOutOfRange: lexMessage(L, errNumberOutOfRange, result.literal)
L.bufpos = endpos
proc handleHexChar(L: var TLexer, xi: var int) =
@@ -472,23 +467,22 @@ proc HandleCRLF(L: var TLexer, pos: int): int =
else: result = pos
proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
var
line, line2, pos: int
c: Char
buf: cstring
pos = L.bufPos + 1 # skip "
buf = L.buf # put `buf` in a register
line = L.linenumber # save linenumber for better error message
if (buf[pos] == '\"') and (buf[pos + 1] == '\"'):
var pos = L.bufPos + 1 # skip "
var buf = L.buf # put `buf` in a register
var line = L.linenumber # save linenumber for better error message
if buf[pos] == '\"' and buf[pos+1] == '\"':
tok.tokType = tkTripleStrLit # long string literal:
inc(pos, 2) # skip ""
# skip leading newline:
# skip leading newline:
pos = HandleCRLF(L, pos)
buf = L.buf
while true:
case buf[pos]
of '\"':
if (buf[pos + 1] == '\"') and (buf[pos + 2] == '\"'): break
if buf[pos+1] == '\"' and buf[pos+2] == '\"' and
buf[pos+3] != '\"':
L.bufpos = pos + 3 # skip the three """
break
add(tok.literal, '\"')
Inc(pos)
of CR, LF:
@@ -496,7 +490,7 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
buf = L.buf
tok.literal = tok.literal & tnl
of lexbase.EndOfFile:
line2 = L.linenumber
var line2 = L.linenumber
L.LineNumber = line
lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart)
L.LineNumber = line2
@@ -504,20 +498,23 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
else:
add(tok.literal, buf[pos])
Inc(pos)
L.bufpos = pos + 3 # skip the three """
else:
# ordinary string literal
if rawMode: tok.tokType = tkRStrLit
else: tok.tokType = tkStrLit
while true:
c = buf[pos]
var c = buf[pos]
if c == '\"':
inc(pos) # skip '"'
break
if c in {CR, LF, lexbase.EndOfFile}:
if rawMode and buf[pos+1] == '\"':
inc(pos, 2)
add(tok.literal, '"')
else:
inc(pos) # skip '"'
break
elif c in {CR, LF, lexbase.EndOfFile}:
lexMessage(L, errClosingQuoteExpected)
break
if (c == '\\') and not rawMode:
elif (c == '\\') and not rawMode:
L.bufPos = pos
getEscapedChar(L, tok)
pos = L.bufPos
@@ -527,29 +524,23 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
L.bufpos = pos
proc getCharacter(L: var TLexer, tok: var TToken) =
var c: Char
Inc(L.bufpos) # skip '
c = L.buf[L.bufpos]
var c = L.buf[L.bufpos]
case c
of '\0'..Pred(' '), '\'': lexMessage(L, errInvalidCharacterConstant)
of '\\': getEscapedChar(L, tok)
else:
tok.literal = c & ""
tok.literal = $c
Inc(L.bufpos)
if L.buf[L.bufpos] != '\'': lexMessage(L, errMissingFinalQuote)
inc(L.bufpos) # skip '
proc getSymbol(L: var TLexer, tok: var TToken) =
var
pos: int
c: Char
buf: cstring
h: THash # hashing algorithm inlined
h = 0
pos = L.bufpos
buf = L.buf
var h: THash = 0
var pos = L.bufpos
var buf = L.buf
while true:
c = buf[pos]
var c = buf[pos]
case c
of 'a'..'z', '0'..'9', '\x80'..'\xFF':
h = h +% Ord(c)
@@ -560,8 +551,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
h = h +% Ord(c)
h = h +% h shl 10
h = h xor (h shr 6)
of '_':
nil
of '_': nil
else: break
Inc(pos)
h = h +% h shl 3
@@ -580,16 +570,11 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
else: tok.tokType = tkCallTripleStrLit
proc getOperator(L: var TLexer, tok: var TToken) =
var
pos: int
c: Char
buf: cstring
h: THash # hashing algorithm inlined
pos = L.bufpos
buf = L.buf
h = 0
var pos = L.bufpos
var buf = L.buf
var h: THash = 0
while true:
c = buf[pos]
var c = buf[pos]
if c in OpChars:
h = h +% Ord(c)
h = h +% h shl 10
@@ -606,9 +591,8 @@ proc getOperator(L: var TLexer, tok: var TToken) =
L.bufpos = pos
proc handleIndentation(L: var TLexer, tok: var TToken, indent: int) =
var i: int
tok.indent = indent
i = high(L.indentStack)
var i = high(L.indentStack)
if indent > L.indentStack[i]:
tok.tokType = tkInd
elif indent == L.indentStack[i]:
@@ -625,22 +609,19 @@ proc handleIndentation(L: var TLexer, tok: var TToken, indent: int) =
lexMessage(L, errInvalidIndentation)
proc scanComment(L: var TLexer, tok: var TToken) =
var
buf: cstring
pos, col: int
indent: int
pos = L.bufpos
buf = L.buf # a comment ends if the next line does not start with the # on the same
# column after only whitespace
var pos = L.bufpos
var buf = L.buf
# a comment ends if the next line does not start with the # on the same
# column after only whitespace
tok.tokType = tkComment
col = getColNumber(L, pos)
var col = getColNumber(L, pos)
while true:
while not (buf[pos] in {CR, LF, lexbase.EndOfFile}):
add(tok.literal, buf[pos])
inc(pos)
pos = handleCRLF(L, pos)
buf = L.buf
indent = 0
var indent = 0
while buf[pos] == ' ':
inc(pos)
inc(indent)
@@ -654,11 +635,8 @@ proc scanComment(L: var TLexer, tok: var TToken) =
L.bufpos = pos
proc skip(L: var TLexer, tok: var TToken) =
var
buf: cstring
indent, pos: int
pos = L.bufpos
buf = L.buf
var pos = L.bufpos
var buf = L.buf
while true:
case buf[pos]
of ' ':
@@ -669,7 +647,7 @@ proc skip(L: var TLexer, tok: var TToken) =
of CR, LF:
pos = HandleCRLF(L, pos)
buf = L.buf
indent = 0
var indent = 0
while buf[pos] == ' ':
Inc(pos)
Inc(indent)
@@ -681,7 +659,6 @@ proc skip(L: var TLexer, tok: var TToken) =
L.bufpos = pos
proc rawGetTok(L: var TLexer, tok: var TToken) =
var c: Char
fillToken(tok)
if L.dedent > 0:
dec(L.dedent)
@@ -691,10 +668,10 @@ proc rawGetTok(L: var TLexer, tok: var TToken) =
else:
tok.tokType = tkDed
return
skip(L, tok) # skip
# got an documentation comment or tkIndent, return that:
skip(L, tok)
# got an documentation comment or tkIndent, return that:
if tok.toktype != tkInvalid: return
c = L.buf[L.bufpos]
var c = L.buf[L.bufpos]
if c in SymStartChars - {'r', 'R', 'l'}:
getSymbol(L, tok)
elif c in {'0'..'9'}:

8
tests/tassert.nim
View File

@@ -7,10 +7,10 @@ proc callC() = callA()
try:
callC()
except EAssertionFailed:
write(stdout, "assertion failure!\n")
write(stdout, "assertion failure!")
except:
write(stdout, "unknown exception!\n")
write(stdout, "unknown exception!")
finally:
system.write(stdout, "this shall be always written\n")
system.write(stdout, "this shall be always written")
assert(false)
assert(false) #OUT assertion failure!this shall be always written

20
tests/tinvalidnewseq.nim Executable file
View File

@@ -0,0 +1,20 @@
import regexprs, strutils
type
TURL = tuple[protocol, subdomain, domain, port: string, path: seq[string]]
proc parseURL(url: string): TURL =
#([a-zA-Z]+://)?(\w+?\.)?(\w+)(\.\w+)(:[0-9]+)?(/.+)?
var pattern: string = r"([a-zA-Z]+://)?(\w+?\.)?(\w+)(\.\w+)(:[0-9]+)?(/.+)?"
var m: array[0..6, string] #Array with the matches
newSeq(m, 7) #ERROR
discard regexprs.match(url, pattern, m)
result = (protocol: m[1], subdomain: m[2], domain: m[3] & m[4],
port: m[5], path: m[6].split('/'))
var r: TUrl
r = parseUrl(r"http://google.com/search?var=bleahdhsad")
echo(r.domain)

14
tests/tstrlits.nim Executable file
View File

@@ -0,0 +1,14 @@
# Test the new different string literals
const
tripleEmpty = """"long string"""""""" # "long string """""
rawQuote = r"a"""
raw = r"abc""def"
stdout.write(rawQuote)
stdout.write(tripleEmpty)
stdout.write(raw)
#OUT a""long string"""""abc"def