From 2f95e4d829ed1452f160fa94a9126c71fde2140f Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 7 Dec 2011 00:45:52 +0200 Subject: [PATCH 1/8] Speed-up symbolFiles:on operations on windows --- compiler/main.nim | 22 ++++++++++------------ compiler/rodread.nim | 4 ++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/compiler/main.nim b/compiler/main.nim index 944852bb68..fd29085168 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -15,7 +15,8 @@ import os, lists, condsyms, rodread, rodwrite, ropes, trees, wordrecg, sem, semdata, idents, passes, docgen, extccomp, cgen, ecmasgen, - platform, nimconf, importer, passaux, depends, transf, evals, types, idgen + platform, nimconf, importer, passaux, depends, transf, evals, types, idgen, + tables const has_LLVM_Backend = false @@ -27,19 +28,16 @@ proc MainCommand*() # ------------------ module handling ----------------------------------------- -type - TFileModuleRec = tuple[filename: string, module: PSym] - TFileModuleMap = seq[TFileModuleRec] +var + compMods = initTable[string, PSym]() # all compiled modules -var compMods: TFileModuleMap = @[] # all compiled modules +# This expects a normalized module path +proc registerModule(filename: string, module: PSym) = + compMods[filename] = module -proc registerModule(filename: string, module: PSym) = - compMods.add((filename, module)) - -proc getModule(filename: string): PSym = - for i in countup(0, high(compMods)): - if sameFile(compMods[i].filename, filename): - return compMods[i].module +# This expects a normalized module path +proc getModule(filename: string): PSym = + result = compMods[filename] proc newModule(filename: string): PSym = # We cannot call ``newSym`` here, because we have to circumvent the ID diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 2ab019c82b..9db083181f 100755 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -788,7 +788,7 @@ proc loadMethods(r: PRodReader) = proc getModuleIdx(filename: string): int = for i in countup(0, high(gMods)): - if sameFile(gMods[i].filename, filename): return i + if gMods[i].filename == filename: return i result = len(gMods) setlen(gMods, result + 1) @@ -853,7 +853,7 @@ proc handleSymbolFile(module: PSym, filename: string): PRodReader = proc GetCRC*(filename: string): TCrc32 = for i in countup(0, high(gMods)): - if sameFile(gMods[i].filename, filename): return gMods[i].crc + if gMods[i].filename == filename: return gMods[i].crc result = crcFromFile(filename) #var idx = getModuleIdx(filename) From 446b0421888cf32a58d22d4152dfbf98d6c12a10 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 7 Dec 2011 00:48:43 +0200 Subject: [PATCH 2/8] removed incompatible flags for debugging VCC release builds --- config/nimrod.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/nimrod.cfg b/config/nimrod.cfg index 24732cc820..b8bb77a03a 100755 --- a/config/nimrod.cfg +++ b/config/nimrod.cfg @@ -107,7 +107,7 @@ clang.options.size = "-Os" # Configuration for the Visual C/C++ compiler: vcc.options.linker = "/DEBUG /Zi /Fd\"$projectName.pdb\" /F33554432" # set the stack size to 8 MB -vcc.options.debug = "/RTC1 /Zi /Fd\"$projectName.pdb\"" +vcc.options.debug = "/Zi /Fd\"$projectName.pdb\"" vcc.options.always = "/nologo" vcc.options.speed = "/Ox /arch:SSE2" vcc.options.size = "/O1" From 0e609d2101535b4ca2640b68890b2c575ed3678d Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 7 Dec 2011 00:53:27 +0200 Subject: [PATCH 3/8] New implementation for os.sameFile on Windows Hard-links on Windows are now treated just as they are on POSIX. The new implementation is faster than the previous, but still it's quite slower than fstat (use with caution). --- lib/pure/os.nim | 49 ++++++++++++++++++++++++++++------------- lib/pure/times.nim | 2 +- lib/windows/winlean.nim | 40 +++++++++++++++++++++++++-------- 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index a8fafd5de6..f22a53dd80 100755 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -567,26 +567,45 @@ proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1".} = result = path[0] == '/' proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1".} = - ## Returns True if both pathname arguments refer to the same file or - ## directory (as indicated by device number and i-node number). - ## Raises an exception if an stat() call on either pathname fails. + ## Returns True if both pathname arguments refer to the same physical + ## file or directory. Raises an exception if any of the files does not + ## exist or information about it can not be obtained. + ## + ## This proc will return true if given two alternative hard-linked or + ## sym-linked paths to the same file or directory. when defined(Windows): - var - a, b: TWin32FindData - var resA = findfirstFileA(path1, a) - var resB = findfirstFileA(path2, b) - if resA != -1 and resB != -1: - result = $a.cFileName == $b.cFileName - else: - # work around some ``findfirstFileA`` bugs - result = cmpPaths(path1, path2) == 0 - if resA != -1: findclose(resA) - if resB != -1: findclose(resB) + var success = true + + template OpenHandle(path: expr): expr = + CreateFileA(path, 0'i32, FILE_SHARE_DELETE or FILE_SHARE_READ or + FILE_SHARE_WRITE, nil, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL, 0) + + var f1 = OpenHandle(path1) + var f2 = OpenHandle(path2) + + if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: + var fi1, fi2: TBY_HANDLE_FILE_INFORMATION + + if GetFileInformationByHandle(f1, addr(fi1)) != 0 and + GetFileInformationByHandle(f2, addr(fi2)) != 0: + result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and + fi1.nFileIndexHigh == fi2.nFileIndexHigh and + fi1.nFileIndexLow == fi2.nFileIndexLow + else: success = false + else: success = false + + discard CloseHandle(f1) + discard CloseHandle(f2) + + if not success: + OSError() + else: var a, b: TStat if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: - result = cmpPaths(path1, path2) == 0 # be consistent with Windows + OSError() else: result = a.st_dev == b.st_dev and a.st_ino == b.st_ino diff --git a/lib/pure/times.nim b/lib/pure/times.nim index a4ac596734..f73a48bea4 100755 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -302,7 +302,7 @@ when not defined(ECMAScript): posix_gettimeofday(a) result = toFloat(a.tv_sec) + toFloat(a.tv_usec)*0.00_0001 elif defined(windows): - var f: winlean.Filetime + var f: winlean.TFiletime GetSystemTimeAsFileTime(f) var i64 = rdFileTime(f) - epochDiff var secs = i64 div rateDiff diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 18e287bfa1..acc28e58b1 100755 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -47,6 +47,22 @@ type dwProcessId*: int32 dwThreadId*: int32 + TFILETIME* {.final, pure.} = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT + dwLowDateTime*: DWORD + dwHighDateTime*: DWORD + + TBY_HANDLE_FILE_INFORMATION* {.final, pure.} = object + dwFileAttributes*: DWORD + ftCreationTime*: TFILETIME + ftLastAccessTime*: TFILETIME + ftLastWriteTime*: TFILETIME + dwVolumeSerialNumber*: DWORD + nFileSizeHigh*: DWORD + nFileSizeLow*: DWORD + nNumberOfLinks*: DWORD + nFileIndexHigh*: DWORD + nFileIndexLow*: DWORD + const STARTF_USESHOWWINDOW* = 1'i32 STARTF_USESTDHANDLES* = 256'i32 @@ -149,14 +165,11 @@ const MAX_PATH* = 260 type - FILETIME* {.final, pure.} = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT - dwLowDateTime*: int32 - dwHighDateTime*: int32 TWIN32_FIND_DATA* {.pure.} = object dwFileAttributes*: int32 - ftCreationTime*: FILETIME - ftLastAccessTime*: FILETIME - ftLastWriteTime*: FILETIME + ftCreationTime*: TFILETIME + ftLastAccessTime*: TFILETIME + ftLastWriteTime*: TFILETIME nFileSizeHigh*: int32 nFileSizeLow*: int32 dwReserved0: int32 @@ -192,13 +205,13 @@ proc FreeEnvironmentStringsA*(para1: cstring): int32 {. proc GetCommandLineA*(): CString {.importc, stdcall, dynlib: "kernel32".} -proc rdFileTime*(f: FILETIME): int64 = +proc rdFileTime*(f: TFILETIME): int64 = result = ze64(f.dwLowDateTime) or (ze64(f.dwHighDateTime) shl 32) proc rdFileSize*(f: TWin32FindData): int64 = result = ze64(f.nFileSizeLow) or (ze64(f.nFileSizeHigh) shl 32) -proc GetSystemTimeAsFileTime*(lpSystemTimeAsFileTime: var FileTime) {. +proc GetSystemTimeAsFileTime*(lpSystemTimeAsFileTime: var TFILETIME) {. importc: "GetSystemTimeAsFileTime", dynlib: "kernel32", stdcall.} proc Sleep*(dwMilliseconds: int32){.stdcall, dynlib: "kernel32", @@ -209,12 +222,16 @@ proc ShellExecute*(HWND: THandle, lpOperation, lpFile, nShowCmd: int32): THandle{. stdcall, dynlib: "shell32.dll", importc: "ShellExecuteA".} +proc GetFileInformationByHandle*(hFile: THandle, + lpFileInformation: ptr TBY_HANDLE_FILE_INFORMATION): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "GetFileInformationByHandle".} + const WSADESCRIPTION_LEN* = 256 WSASYS_STATUS_LEN* = 128 FD_SETSIZE* = 64 MSG_PEEK* = 2 - + INADDR_ANY* = 0 INADDR_LOOPBACK* = 0x7F000001 INADDR_BROADCAST* = -1 @@ -410,6 +427,9 @@ const GENERIC_READ* = 0x80000000'i32 GENERIC_ALL* = 0x10000000'i32 FILE_SHARE_READ* = 1'i32 + FILE_SHARE_DELETE* = 4'i32 + FILE_SHARE_WRITE* = 2'i32 + CREATE_ALWAYS* = 2'i32 OPEN_EXISTING* = 3'i32 FILE_BEGIN* = 0'i32 @@ -421,6 +441,8 @@ const FILE_MAP_WRITE* = 2'i32 INVALID_FILE_SIZE* = -1'i32 + FILE_FLAG_BACKUP_SEMANTICS* = 33554432'i32 + proc CreateFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD, lpSecurityAttributes: pointer, dwCreationDisposition, dwFlagsAndAttributes: DWORD, From e13a610ad42ef4d5d0222a2a942682d35e405dc7 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 7 Dec 2011 00:57:52 +0200 Subject: [PATCH 4/8] The test runner will print out encountered exceptions --- lib/pure/unittest.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index e2906dd1aa..f9981195c1 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -70,6 +70,10 @@ template test*(name: expr, body: stmt): stmt = TestSetupIMPL() body + except: + checkpoint("Unhandled exception: " & getCurrentExceptionMsg()) + fail() + finally: TestTeardownIMPL() testDone name, TestStatusIMPL From e4e74034c256f2b35c0b1650044e1598a0ee3697 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Fri, 9 Dec 2011 04:40:59 +0200 Subject: [PATCH 5/8] path canonicalization and proper project relative paths --- compiler/c2nim/clex.nim | 8 ++--- compiler/commands.nim | 10 +++---- compiler/lexer.nim | 8 ++--- compiler/main.nim | 2 +- compiler/msgs.nim | 65 +++++++++++++++++++++++++---------------- compiler/nimrod.nim | 4 +-- compiler/options.nim | 7 +++-- compiler/semstmts.nim | 2 +- compiler/sigmatch.nim | 2 +- 9 files changed, 61 insertions(+), 47 deletions(-) diff --git a/compiler/c2nim/clex.nim b/compiler/c2nim/clex.nim index 5a67f94757..53f230b7b8 100755 --- a/compiler/c2nim/clex.nim +++ b/compiler/c2nim/clex.nim @@ -97,7 +97,7 @@ type next*: ref TToken # for C we need arbitrary look-ahead :-( TLexer* = object of TBaseLexer - filename*: string + fileIdx*: int32 inDirective: bool proc getTok*(L: var TLexer, tok: var TToken) @@ -117,7 +117,7 @@ proc fillToken(L: var TToken) = proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream) = openBaseLexer(lex, inputstream) - lex.filename = filename + lex.fileIdx = filename.fileInfoIdx proc closeLexer*(lex: var TLexer) = inc(gLinesCompiled, lex.LineNumber) @@ -127,13 +127,13 @@ proc getColumn*(L: TLexer): int = result = getColNumber(L, L.bufPos) proc getLineInfo*(L: TLexer): TLineInfo = - result = newLineInfo(L.filename, L.linenumber, getColNumber(L, L.bufpos)) + result = newLineInfo(L.fileIdx, L.linenumber, getColNumber(L, L.bufpos)) proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "") = msgs.GenericMessage(getLineInfo(L), msg, arg) proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") = - var info = newLineInfo(L.filename, L.linenumber, pos - L.lineStart) + var info = newLineInfo(L.fileIdx, L.linenumber, pos - L.lineStart) msgs.GenericMessage(info, msg, arg) proc TokKindToStr*(k: TTokKind): string = diff --git a/compiler/commands.nim b/compiler/commands.nim index bce24c5da7..f07361abe1 100755 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -459,9 +459,7 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) = if strutils.find(switch, '.') >= 0: options.setConfigVar(switch, arg) else: InvalidCmdLineOption(pass, switch, info) -proc ProcessCommand(switch: string, pass: TCmdLinePass) = - var - cmd, arg: string - var info = newLineInfo("command line", 1, 1) - splitSwitch(switch, cmd, arg, pass, info) - ProcessSwitch(cmd, arg, pass, info) +proc ProcessCommand(switch: string, pass: TCmdLinePass) = + var cmd, arg: string + splitSwitch(switch, cmd, arg, pass, gCmdLineInfo) + processSwitch(cmd, arg, pass, gCmdLineInfo) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 653644f8f9..63a9bef87a 100755 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -96,7 +96,7 @@ type # documentation comments are here too TLexer* = object of TBaseLexer - filename*: string + fileIdx*: int32 indentStack*: seq[int] # the indentation stack dedent*: int # counter for DED token generation indentAhead*: int # if > 0 an indendation has already been read @@ -198,7 +198,7 @@ proc fillToken(L: var TToken) = proc openLexer(lex: var TLexer, filename: string, inputstream: PLLStream) = openBaseLexer(lex, inputstream) lex.indentStack = @[0] - lex.filename = filename + lex.fileIdx = filename.fileInfoIdx lex.indentAhead = - 1 inc(lex.Linenumber, inputstream.lineOffset) @@ -210,13 +210,13 @@ proc getColumn(L: TLexer): int = result = getColNumber(L, L.bufPos) proc getLineInfo(L: TLexer): TLineInfo = - result = newLineInfo(L.filename, L.linenumber, getColNumber(L, L.bufpos)) + result = newLineInfo(L.fileIdx, L.linenumber, getColNumber(L, L.bufpos)) proc lexMessage(L: TLexer, msg: TMsgKind, arg = "") = msgs.Message(getLineInfo(L), msg, arg) proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") = - var info = newLineInfo(L.filename, L.linenumber, pos - L.lineStart) + var info = newLineInfo(L.fileIdx, L.linenumber, pos - L.lineStart) msgs.Message(info, msg, arg) proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: TCharSet) = diff --git a/compiler/main.nim b/compiler/main.nim index fd29085168..4d65f84bbe 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -176,7 +176,7 @@ proc CommandSuggest = proc wantMainModule = if gProjectFull.len == 0: - Fatal(newLineInfo("command line", 1, 1), errCommandExpectsFilename) + Fatal(gCmdLineInfo, errCommandExpectsFilename) proc MainCommand = appendStr(searchPaths, options.libpath) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 3090198acd..32afdee3b9 100755 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -8,7 +8,7 @@ # import - options, strutils, os + options, strutils, os, tables type TMsgKind* = enum @@ -384,6 +384,11 @@ const type TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints TNoteKinds* = set[TNoteKind] + + TFileInfo*{.final.} = object + fullPath*: string # This is a canonical full filesystem path + projPath*: string # This is relative to the project's root + TLineInfo*{.final.} = object # This is designed to be as small as possible, # because it is used # in syntax nodes. We safe space here by using @@ -395,7 +400,37 @@ type ERecoverableError* = object of EInvalidValue -proc newLineInfo*(filename: string, line, col: int): TLineInfo +var + filenameToIndexTbl = initTable[string, int32]() + fileInfos: seq[TFileInfo] = @[] + +proc newFileInfo(fullPath, projPath: string): TFileInfo = + result.fullPath = fullPath + result.projPath = projPath + +proc fileInfoIdx*(filename: string): int32 = + var canonical = canonicalizePath(filename) + + if filenameToIndexTbl.hasKey(canonical): + result = filenameToIndexTbl[canonical] + else: + result = fileInfos.len.int32 + fileInfos.add(newFileInfo(canonical, canonical.shortenDir)) + filenameToIndexTbl[canonical] = result + +proc newLineInfo*(filename: string, line, col: int): TLineInfo = + result.fileIndex = filename.fileInfoIdx + result.line = int16(line) + result.col = int16(col) + +proc newLineInfo*(fileInfoIdx: int32, line, col: int): TLineInfo = + result.fileIndex = fileInfoIdx + result.line = int16(line) + result.col = int16(col) + +fileInfos.add(newFileInfo("command line", "")) +var gCmdLineInfo* = newLineInfo(int32(0), 1, 1) + proc raiseRecoverableError*() {.noinline, noreturn.} = raise newException(ERecoverableError, "") @@ -423,9 +458,7 @@ proc UnknownLineInfo*(): TLineInfo = result.fileIndex = -1 var - filenames: seq[tuple[filename: string, fullpath: string]] = @[] msgContext: seq[TLineInfo] = @[] - gCmdLineInfo* = newLineInfo("command line", -1, -1) proc pushInfoContext*(info: TLineInfo) = msgContext.add(info) @@ -433,31 +466,13 @@ proc pushInfoContext*(info: TLineInfo) = proc popInfoContext*() = setlen(msgContext, len(msgContext) - 1) -proc includeFilename*(f: string): int = - for i in countdown(high(filenames), low(filenames)): - if filenames[i].filename == f: - return i - - result = len(filenames) - - var fullpath: string - try: fullpath = expandFilename(f) - except: fullpath = "" - - filenames.add((filename: f, fullpath: fullpath)) - -proc newLineInfo(filename: string, line, col: int): TLineInfo = - result.fileIndex = includeFilename(filename) - result.line = int16(line) - result.col = int16(col) - -proc ToFilename*(info: TLineInfo): string = +proc ToFilename*(info: TLineInfo): string = if info.fileIndex < 0: result = "???" - else: result = filenames[info.fileIndex].filename + else: result = fileInfos[info.fileIndex].projPath proc toFullPath*(info: TLineInfo): string = if info.fileIndex < 0: result = "???" - else: result = filenames[info.fileIndex].fullpath + else: result = fileInfos[info.fileIndex].fullPath proc ToLinenumber*(info: TLineInfo): int {.inline.} = result = info.line diff --git a/compiler/nimrod.nim b/compiler/nimrod.nim index 6a9ae690ec..24dbc06178 100755 --- a/compiler/nimrod.nim +++ b/compiler/nimrod.nim @@ -70,14 +70,14 @@ proc HandleCmdLine() = ProcessCmdLine(passCmd1) if gProjectName != "": try: - gProjectFull = expandFilename(gProjectName) + gProjectFull = canonicalizePath(gProjectName) except EOS: gProjectFull = gProjectName var p = splitFile(gProjectFull) gProjectPath = p.dir gProjectName = p.name else: - gProjectPath = getCurrentDir() + gProjectPath = getCurrentDir() LoadConfigs(DefaultConfig) # load all config files # now process command line arguments again, because some options in the # command line can overwite the config file's settings diff --git a/compiler/options.nim b/compiler/options.nim index ea36e3c6ab..bf01ea0029 100755 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -138,12 +138,13 @@ proc getPrefixDir*(): string = ## gets the application directory result = SplitPath(getAppDir()).head +proc canonicalizePath*(path: string): string = + result = path.expandFilename + when not FileSystemCaseSensitive: result = result.toLower + proc shortenDir*(dir: string): string = ## returns the interesting part of a dir var prefix = getPrefixDir() & dirSep - if startsWith(dir, prefix): - return substr(dir, len(prefix)) - prefix = getCurrentDir() & dirSep if startsWith(dir, prefix): return substr(dir, len(prefix)) prefix = gProjectPath & dirSep diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 2dbfa94973..3766841f17 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -796,7 +796,7 @@ proc evalInclude(c: PContext, n: PNode): PNode = addSon(result, n) for i in countup(0, sonsLen(n) - 1): var f = getModuleFile(n.sons[i]) - var fileIndex = includeFilename(f) + var fileIndex = f.fileInfoIdx if ContainsOrIncl(c.includedFiles, fileIndex): GlobalError(n.info, errRecursiveDependencyX, f) addSon(result, semStmt(c, gIncludeFile(f))) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index cd6cbe8599..a6bde6e400 100755 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -713,7 +713,7 @@ proc matches*(c: PContext, n: PNode, m: var TCandidate) = when false: if sfSystemModule notin c.module.flags: - if includeFilename("temp.nim") == c.module.info.fileIndex: + if fileInfoIdx("temp.nim") == c.module.info.fileIndex: echo "########################" echo m.call.renderTree for i in 1..m.call.len-1: From de4b894541f6304b29c5714b032f415f567c2f3c Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Fri, 9 Dec 2011 15:56:48 +0200 Subject: [PATCH 6/8] proc arguments are in scope when the result type definition is being processed This is intended for type deduction depending on the arguments proc foo(x: type): type(expr(x)) --- compiler/seminst.nim | 3 ++- compiler/semstmts.nim | 6 ++++-- compiler/semtempl.nim | 3 ++- compiler/semtypes.nim | 6 +++++- tests/compile/ttypeselectors.nim | 3 +++ 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index d7dbe623b7..641cf9a890 100755 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -135,7 +135,8 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable, if n.sons[paramsPos].kind != nkEmpty: removeDefaultParamValues(n.sons[ParamsPos]) semParamList(c, n.sons[ParamsPos], nil, result) - addParams(c, result.typ.n) + # XXX: obsoleted - happens in semParamList # + # addParams(c, result.typ.n) else: result.typ = newTypeS(tyProc, c) addSon(result.typ, nil) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 3766841f17..0136724fa1 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -616,7 +616,8 @@ proc semLambda(c: PContext, n: PNode): PNode = illFormedAst(n) # process parameters: if n.sons[paramsPos].kind != nkEmpty: semParamList(c, n.sons[ParamsPos], nil, s) - addParams(c, s.typ.n) + # XXX: obsoleted - happens in semParamList + # addParams(c, s.typ.n) ParamsTypeCheck(c, s.typ) else: s.typ = newTypeS(tyProc, c) @@ -665,7 +666,8 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, n.sons[genericParamsPos] = gp # check for semantics again: semParamList(c, n.sons[ParamsPos], nil, s) - addParams(c, s.typ.n) + # XXX: obsoleted - happens in semParamList + # addParams(c, s.typ.n) else: s.typ = newTypeS(tyProc, c) addSon(s.typ, nil) diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index ee540c5d20..26216ab4c6 100755 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -185,7 +185,8 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = # use ``stmt`` as implicit result type s.typ.sons[0] = newTypeS(tyStmt, c) s.typ.n.sons[0] = newNodeIT(nkType, n.info, s.typ.sons[0]) - addParams(c, s.typ.n) # resolve parameters: + # XXX: obsoleted - happens in semParamList # + # addParams(c, s.typ.n) # resolve parameters: var toBind = initIntSet() n.sons[bodyPos] = resolveTemplateParams(c, n.sons[bodyPos], false, toBind) if s.typ.sons[0].kind notin {tyStmt, tyTypeDesc}: diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 9e78f98e3d..78a95c56b3 100755 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -569,6 +569,8 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, LocalError(a.sons[j].info, errAttemptToRedefine, arg.name.s) addSon(result.n, newSymNode(arg)) addSon(result, typ) + addDecl(c, arg) + if n.sons[0].kind != nkEmpty: var r = paramType(c, n.sons[0], genericParams, cl) # turn explicit 'void' return type into 'nil' because the rest of the @@ -711,11 +713,13 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = of nkDistinctTy: result = semDistinct(c, n, prev) of nkProcTy: checkSonsLen(n, 2) - result = semProcTypeNode(c, n.sons[0], nil, prev) + openScope(c.tab) + result = semProcTypeNode(c, n.sons[0], nil, prev) # dummy symbol for `pragma`: var s = newSymS(skProc, newIdentNode(getIdent("dummy"), n.info), c) s.typ = result pragma(c, s, n.sons[1], procTypePragmas) + closeScope(c.tab) of nkEnumTy: result = semEnum(c, n, prev) of nkType: result = n.typ of nkStmtListType: result = semStmtListType(c, n, prev) diff --git a/tests/compile/ttypeselectors.nim b/tests/compile/ttypeselectors.nim index 7d76cafef8..9ed7e2008b 100644 --- a/tests/compile/ttypeselectors.nim +++ b/tests/compile/ttypeselectors.nim @@ -35,3 +35,6 @@ var y*: type(t2(100)) = "test" proc t6*(x: type(t3(0))): type(t1(0)) = result = $x +proc t7*(x: int): type($x) = + result = "test" + From d171a8b36f10f42d35e64a7ddefa57376b419908 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Sun, 11 Dec 2011 09:41:04 +0200 Subject: [PATCH 7/8] path canonicalization for imported modules, relative paths written in rod files --- compiler/ast.nim | 17 ++++++++++++++++- compiler/depends.nim | 4 ++-- compiler/docgen.nim | 11 +---------- compiler/importer.nim | 28 ++++++++-------------------- compiler/main.nim | 2 +- compiler/msgs.nim | 24 ++++++++++++++++-------- compiler/options.nim | 17 +++++++++++++---- compiler/rodread.nim | 9 ++++++--- compiler/rodwrite.nim | 17 +++++++++-------- compiler/semstmts.nim | 2 +- 10 files changed, 73 insertions(+), 58 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 2b84c415f6..6ac7dc0de9 100755 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -11,7 +11,7 @@ import msgs, hashes, nversion, options, strutils, crc, ropes, idents, lists, - intsets, idgen + intsets, idgen, os const ImportTablePos* = 0 # imported symbols are at level 0 @@ -1030,3 +1030,18 @@ proc getStrOrChar*(a: PNode): string = internalError(a.info, "getStrOrChar") result = "" +proc getModuleName*(n: PNode): string = + # This returns a short relative module name without the nim extension + # e.g. like "system", "importer" or "somepath/module" + # The proc won't perform any checks that the path is actually valid + case n.kind + of nkStrLit, nkRStrLit, nkTripleStrLit: + result = UnixToNativePath(n.strVal) + of nkIdent: + result = n.ident.s + of nkSym: + result = n.sym.name.s + else: + internalError(n.info, "getModuleName") + result = "" + diff --git a/compiler/depends.nim b/compiler/depends.nim index 05d1764366..ac3f13a9b6 100755 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -33,10 +33,10 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode = case n.kind of nkImportStmt: for i in countup(0, sonsLen(n) - 1): - var imported = splitFile(getModuleFile(n.sons[i])).name + var imported = getModuleName(n.sons[i]) addDependencyAux(g.module.name.s, imported) of nkFromStmt: - var imported = splitFile(getModuleFile(n.sons[0])).name + var imported = getModuleName(n.sons[0]) addDependencyAux(g.module.name.s, imported) of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr: for i in countup(0, sonsLen(n) - 1): discard addDotDependency(c, n.sons[i]) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 36eff25763..f22c81a9f6 100755 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -760,21 +760,12 @@ proc renderRstToOut(d: PDoc, n: PRstNode): PRope = proc checkForFalse(n: PNode): bool = result = n.kind == nkIdent and IdentEq(n.ident, "false") -proc getModuleFile(n: PNode): string = - case n.kind - of nkStrLit, nkRStrLit, nkTripleStrLit: result = n.strVal - of nkIdent: result = n.ident.s - of nkSym: result = n.sym.name.s - else: - internalError(n.info, "getModuleFile()") - result = "" - proc traceDeps(d: PDoc, n: PNode) = const k = skModule if d.section[k] != nil: app(d.section[k], ", ") dispA(d.section[k], "$1", - "$1", [toRope(getModuleFile(n))]) + "$1", [toRope(getModuleName(n))]) proc generateDoc(d: PDoc, n: PNode) = case n.kind diff --git a/compiler/importer.nim b/compiler/importer.nim index 6267cb68de..682b852f7c 100755 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -16,25 +16,13 @@ import proc evalImport*(c: PContext, n: PNode): PNode proc evalFrom*(c: PContext, n: PNode): PNode proc importAllSymbols*(c: PContext, fromMod: PSym) -proc getModuleFile*(n: PNode): string -# implementation -proc findModule(info: TLineInfo, modulename: string): string = - # returns path to module - result = options.FindFile(AddFileExt(modulename, nimExt)) - if result == "": Fatal(info, errCannotOpenFile, modulename) - -proc getModuleFile(n: PNode): string = - case n.kind - of nkStrLit, nkRStrLit, nkTripleStrLit: - result = findModule(n.info, UnixToNativePath(n.strVal)) - of nkIdent: - result = findModule(n.info, n.ident.s) - of nkSym: - result = findModule(n.info, n.sym.name.s) - else: - internalError(n.info, "getModuleFile()") - result = "" +proc checkModuleName*(n: PNode): string = + # This returns the full canonical path for a given module import + var modulename = n.getModuleName + result = findModule(modulename) + if result.len == 0: + Fatal(n.info, errCannotOpenFile, modulename) proc rawImportSymbol(c: PContext, s: PSym) = # This does not handle stubs, because otherwise loading on demand would be @@ -103,7 +91,7 @@ proc importAllSymbols(c: PContext, fromMod: PSym) = proc evalImport(c: PContext, n: PNode): PNode = result = n for i in countup(0, sonsLen(n) - 1): - var f = getModuleFile(n.sons[i]) + var f = checkModuleName(n.sons[i]) var m = gImportModule(f) if sfDeprecated in m.flags: Message(n.sons[i].info, warnDeprecated, m.name.s) @@ -114,7 +102,7 @@ proc evalImport(c: PContext, n: PNode): PNode = proc evalFrom(c: PContext, n: PNode): PNode = result = n checkMinSonsLen(n, 2) - var f = getModuleFile(n.sons[0]) + var f = checkModuleName(n.sons[0]) var m = gImportModule(f) n.sons[0] = newSymNode(m) addDecl(c, m) # add symbol to symbol table of module diff --git a/compiler/main.nim b/compiler/main.nim index 4d65f84bbe..0ba07900cf 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -69,7 +69,7 @@ proc importModule(filename: string): PSym = proc CompileModule(filename: string, flags: TSymFlags): PSym = var rd: PRodReader = nil var f = addFileExt(filename, nimExt) - result = newModule(filename) + result = newModule(f) result.flags = result.flags + flags if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}: rd = handleSymbolFile(result, f) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 32afdee3b9..ac6e6e4818 100755 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -409,26 +409,34 @@ proc newFileInfo(fullPath, projPath: string): TFileInfo = result.projPath = projPath proc fileInfoIdx*(filename: string): int32 = - var canonical = canonicalizePath(filename) + var + canonical: string + pseudoPath = false + + try: + canonical = canonicalizePath(filename) + except: + canonical = filename + # The compiler uses "filenames" such as `command line` or `stdin` + # This flag indicates that we are working with such a path here + pseudoPath = true if filenameToIndexTbl.hasKey(canonical): result = filenameToIndexTbl[canonical] else: result = fileInfos.len.int32 - fileInfos.add(newFileInfo(canonical, canonical.shortenDir)) + fileInfos.add(newFileInfo(canonical, if pseudoPath: "" else: canonical.shortenDir)) filenameToIndexTbl[canonical] = result -proc newLineInfo*(filename: string, line, col: int): TLineInfo = - result.fileIndex = filename.fileInfoIdx - result.line = int16(line) - result.col = int16(col) - proc newLineInfo*(fileInfoIdx: int32, line, col: int): TLineInfo = result.fileIndex = fileInfoIdx result.line = int16(line) result.col = int16(col) -fileInfos.add(newFileInfo("command line", "")) +proc newLineInfo*(filename: string, line, col: int): TLineInfo {.inline.} = + result = newLineInfo(filename.fileInfoIdx, line, col) + +fileInfos.add(newFileInfo("", "command line")) var gCmdLineInfo* = newLineInfo(int32(0), 1, 1) proc raiseRecoverableError*() {.noinline, noreturn.} = diff --git a/compiler/options.nim b/compiler/options.nim index bf01ea0029..1c5fc290f3 100755 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -188,19 +188,28 @@ iterator iterSearchPath*(): string = yield it.data it = PStrEntry(it.Next) -proc rawFindFile(f: string): string = +proc rawFindFile(f: string): string = + template ret(e: expr) = + result = e.canonicalizePath + return + if ExistsFile(f): - result = f + ret f else: for it in iterSearchPath(): result = JoinPath(it, f) - if ExistsFile(result): return + if ExistsFile(result): + ret result result = "" proc FindFile*(f: string): string = result = rawFindFile(f) if len(result) == 0: result = rawFindFile(toLower(f)) - + +proc findModule*(modulename: string): string {.inline.} = + # returns path to module + result = FindFile(AddFileExt(modulename, nimExt)) + proc binaryStrSearch*(x: openarray[string], y: string): int = var a = 0 var b = len(x) - 1 diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 9db083181f..a73f5414f0 100755 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -576,7 +576,9 @@ proc processRodFile(r: PRodReader, crc: TCrc32) = L = 0 while r.s[r.pos] > '\x0A' and r.s[r.pos] != ')': setlen(r.files, L + 1) - r.files[L] = decodeStr(r.s, r.pos) + var relativePath = decodeStr(r.s, r.pos) + var resolvedPath = relativePath.findModule + r.files[L] = if resolvedPath.len > 0: resolvedPath else: relativePath inc(r.pos) # skip #10 inc(r.line) inc(L) @@ -628,7 +630,8 @@ proc processRodFile(r: PRodReader, crc: TCrc32) = r.initIdx = r.pos + 2 # "(\10" skipSection(r) else: - MsgWriteln("skipping section: " & $r.pos) + MsgWriteln("skipping section: " & section & + " at " & $r.pos & " in " & r.filename) skipSection(r) if r.s[r.pos] == '\x0A': inc(r.pos) @@ -792,7 +795,7 @@ proc getModuleIdx(filename: string): int = result = len(gMods) setlen(gMods, result + 1) -proc checkDep(filename: string): TReasonForRecompile = +proc checkDep(filename: string): TReasonForRecompile = assert(not isNil(filename)) var idx = getModuleIdx(filename) if gMods[idx].reason != rrEmpty: diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index 154dc13d46..873f97764c 100755 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -87,17 +87,18 @@ proc newRodWriter(modfilename: string, crc: TCrc32, module: PSym): PRodWriter = result.init = "" result.data = newStringOfCap(12_000) -proc addModDep(w: PRodWriter, dep: string) = +proc addModDep(w: PRodWriter, dep: string) = if w.modDeps.len != 0: add(w.modDeps, ' ') encodeVInt(fileIdx(w, dep), w.modDeps) const rodNL = "\x0A" -proc addInclDep(w: PRodWriter, dep: string) = +proc addInclDep(w: PRodWriter, dep: string) = + var resolved = dep.findModule encodeVInt(fileIdx(w, dep), w.inclDeps) add(w.inclDeps, " ") - encodeVInt(crcFromFile(dep), w.inclDeps) + encodeVInt(crcFromFile(resolved), w.inclDeps) add(w.inclDeps, rodNL) proc pushType(w: PRodWriter, t: PType) = @@ -555,21 +556,21 @@ proc process(c: PPassContext, n: PNode): PNode = # addInterfaceSym(w, a.sons[j].sym); # end of nkImportStmt: - for i in countup(0, sonsLen(n) - 1): addModDep(w, getModuleFile(n.sons[i])) + for i in countup(0, sonsLen(n) - 1): addModDep(w, getModuleName(n.sons[i])) addStmt(w, n) of nkFromStmt: - addModDep(w, getModuleFile(n.sons[0])) + addModDep(w, getModuleName(n.sons[0])) addStmt(w, n) of nkIncludeStmt: - for i in countup(0, sonsLen(n) - 1): addInclDep(w, getModuleFile(n.sons[i])) + for i in countup(0, sonsLen(n) - 1): addInclDep(w, getModuleName(n.sons[i])) of nkPragma: addStmt(w, n) else: nil -proc myOpen(module: PSym, filename: string): PPassContext = +proc myOpen(module: PSym, filename: string): PPassContext = if module.id < 0: InternalError("rodwrite: module ID not set") - var w = newRodWriter(filename, rodread.GetCRC(filename), module) + var w = newRodWriter(filename, rodread.GetCRC(module.info.toFullPath), module) rawAddInterfaceSym(w, module) result = w diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 0136724fa1..29bf48c199 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -797,7 +797,7 @@ proc evalInclude(c: PContext, n: PNode): PNode = result = newNodeI(nkStmtList, n.info) addSon(result, n) for i in countup(0, sonsLen(n) - 1): - var f = getModuleFile(n.sons[i]) + var f = checkModuleName(n.sons[i]) var fileIndex = f.fileInfoIdx if ContainsOrIncl(c.includedFiles, fileIndex): GlobalError(n.info, errRecursiveDependencyX, f) From caa4766a33050d7fb89565da128afd2da8272e1c Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Sun, 11 Dec 2011 12:41:06 +0200 Subject: [PATCH 8/8] the current directory is no longer taken into account when resolving module paths --- compiler/ast.nim | 17 +---------------- compiler/docgen.nim | 2 +- compiler/importer.nim | 15 +++++++++++++++ compiler/nimrod.cfg | 1 + compiler/options.nim | 19 ++++++------------- 5 files changed, 24 insertions(+), 30 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 8d1aeeb6a7..419d575627 100755 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -11,7 +11,7 @@ import msgs, hashes, nversion, options, strutils, crc, ropes, idents, lists, - intsets, idgen, os + intsets, idgen const ImportTablePos* = 0 # imported symbols are at level 0 @@ -1001,18 +1001,3 @@ proc getStrOrChar*(a: PNode): string = internalError(a.info, "getStrOrChar") result = "" -proc getModuleName*(n: PNode): string = - # This returns a short relative module name without the nim extension - # e.g. like "system", "importer" or "somepath/module" - # The proc won't perform any checks that the path is actually valid - case n.kind - of nkStrLit, nkRStrLit, nkTripleStrLit: - result = UnixToNativePath(n.strVal) - of nkIdent: - result = n.ident.s - of nkSym: - result = n.sym.name.s - else: - internalError(n.info, "getModuleName") - result = "" - diff --git a/compiler/docgen.nim b/compiler/docgen.nim index f22c81a9f6..0f80187142 100755 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -13,7 +13,7 @@ import ast, astalgo, strutils, hashes, options, nversion, msgs, os, ropes, idents, - wordrecg, math, syntaxes, renderer, lexer, rst, times, highlite + wordrecg, math, syntaxes, renderer, lexer, rst, times, highlite, importer proc CommandDoc*() proc CommandRst2Html*() diff --git a/compiler/importer.nim b/compiler/importer.nim index 682b852f7c..6d502f627a 100755 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -17,6 +17,21 @@ proc evalImport*(c: PContext, n: PNode): PNode proc evalFrom*(c: PContext, n: PNode): PNode proc importAllSymbols*(c: PContext, fromMod: PSym) +proc getModuleName*(n: PNode): string = + # This returns a short relative module name without the nim extension + # e.g. like "system", "importer" or "somepath/module" + # The proc won't perform any checks that the path is actually valid + case n.kind + of nkStrLit, nkRStrLit, nkTripleStrLit: + result = UnixToNativePath(n.strVal) + of nkIdent: + result = n.ident.s + of nkSym: + result = n.sym.name.s + else: + internalError(n.info, "getModuleName") + result = "" + proc checkModuleName*(n: PNode): string = # This returns the full canonical path for a given module import var modulename = n.getModuleName diff --git a/compiler/nimrod.cfg b/compiler/nimrod.cfg index 5168a3bb93..7d8d4d94f0 100755 --- a/compiler/nimrod.cfg +++ b/compiler/nimrod.cfg @@ -2,6 +2,7 @@ --hint[XDeclaredButNotUsed]=off path="llvm" +path="$projectPath/.." @if llvm_gcc or gcc: # GCC, LLVM and Visual C++ have a problem to optimize some modules. diff --git a/compiler/options.nim b/compiler/options.nim index 1c5fc290f3..8aef6288cc 100755 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -186,21 +186,14 @@ iterator iterSearchPath*(): string = var it = PStrEntry(SearchPaths.head) while it != nil: yield it.data - it = PStrEntry(it.Next) + it = PStrEntry(it.Next) proc rawFindFile(f: string): string = - template ret(e: expr) = - result = e.canonicalizePath - return - - if ExistsFile(f): - ret f - else: - for it in iterSearchPath(): - result = JoinPath(it, f) - if ExistsFile(result): - ret result - result = "" + for it in iterSearchPath(): + result = JoinPath(it, f) + if ExistsFile(result): + return result.canonicalizePath + result = "" proc FindFile*(f: string): string = result = rawFindFile(f)