From 02ff5f596c330b68927f843814ecb9b86c2eee67 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 1 Oct 2017 23:34:19 +0200 Subject: [PATCH] implemented new experimental scriptable import mechanism --- compiler/gorgeimpl.nim | 83 ++++++++++++++++++++++ compiler/importer.nim | 50 ++++++++++--- compiler/nimblecmd.nim | 8 +-- compiler/sem.nim | 12 ++-- compiler/vm.nim | 2 +- compiler/vmdeps.nim | 45 +----------- koch.nim | 3 + tools/nimresolve.nim | 158 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 297 insertions(+), 64 deletions(-) create mode 100644 compiler/gorgeimpl.nim create mode 100644 tools/nimresolve.nim diff --git a/compiler/gorgeimpl.nim b/compiler/gorgeimpl.nim new file mode 100644 index 0000000000..4e1ce6d507 --- /dev/null +++ b/compiler/gorgeimpl.nim @@ -0,0 +1,83 @@ +# +# +# The Nim Compiler +# (c) Copyright 2017 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Module that implements ``gorge`` for the compiler as well as +## the scriptable import mechanism. + +import msgs, securehash, os, osproc, streams, strutils, options + +proc readOutput(p: Process): (string, int) = + result[0] = "" + var output = p.outputStream + while not output.atEnd: + result[0].add(output.readLine) + result[0].add("\n") + if result[0].len > 0: + result[0].setLen(result[0].len - "\n".len) + result[1] = p.waitForExit + +proc opGorge*(cmd, input, cache: string, info: TLineInfo): (string, int) = + let workingDir = parentDir(info.toFullPath) + if cache.len > 0:# and optForceFullMake notin gGlobalOptions: + let h = secureHash(cmd & "\t" & input & "\t" & cache) + let filename = options.toGeneratedFile("gorge_" & $h, "txt") + var f: File + if open(f, filename): + result = (f.readAll, 0) + f.close + return + var readSuccessful = false + try: + var p = startProcess(cmd, workingDir, + options={poEvalCommand, poStderrToStdout}) + if input.len != 0: + p.inputStream.write(input) + p.inputStream.close() + result = p.readOutput + readSuccessful = true + # only cache successful runs: + if result[1] == 0: + writeFile(filename, result[0]) + except IOError, OSError: + if not readSuccessful: result = ("", -1) + else: + try: + var p = startProcess(cmd, workingDir, + options={poEvalCommand, poStderrToStdout}) + if input.len != 0: + p.inputStream.write(input) + p.inputStream.close() + result = p.readOutput + except IOError, OSError: + result = ("", -1) + +proc scriptableImport*(pkg, subdir: string; info: TLineInfo): string = + var cmd = getConfigVar("resolver.exe") + if cmd.len == 0: cmd = "nimresolve" + else: cmd = quoteShell(cmd) + cmd.add " --source:" + cmd.add quoteShell(info.toFullPath()) + cmd.add " --stdlib:" + cmd.add quoteShell(options.libpath) + cmd.add " --project:" + cmd.add quoteShell(gProjectFull) + if subdir.len != 0: + cmd.add " --subdir:" + cmd.add quoteShell(subdir) + if options.gNoNimblePath: + cmd.add " --nonimblepath" + cmd.add ' ' + cmd.add quoteShell(pkg) + let (res, exitCode) = opGorge(cmd, "", cmd, info) + if exitCode == 0: + result = res.strip() + elif res.len > 0: + localError(info, res) + else: + localError(info, "cannot resolve: " & (pkg / subdir)) diff --git a/compiler/importer.nim b/compiler/importer.nim index dfc9e84ba6..07f42a1472 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -11,11 +11,22 @@ import intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups, - semdata, passes, renderer + semdata, passes, renderer, gorgeimpl proc evalImport*(c: PContext, n: PNode): PNode proc evalFrom*(c: PContext, n: PNode): PNode +proc lookupPackage(pkg, subdir: PNode): string = + let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: "" + case pkg.kind + of nkStrLit, nkRStrLit, nkTripleStrLit: + result = scriptableImport(pkg.strVal, sub, pkg.info) + of nkIdent: + result = scriptableImport(pkg.ident.s, sub, pkg.info) + else: + localError(pkg.info, "package name must be an identifier or string literal") + result = "" + proc getModuleName*(n: PNode): string = # This returns a short relative module name without the nim extension # e.g. like "system", "importer" or "somepath/module" @@ -31,16 +42,33 @@ proc getModuleName*(n: PNode): string = result = n.ident.s of nkSym: result = n.sym.name.s - of nkInfix, nkPrefix: - if n.sons[0].kind == nkIdent and n.sons[0].ident.id == getIdent("as").id: + of nkInfix: + let n0 = n[0] + let n1 = n[1] + if n0.kind == nkIdent and n0.ident.id == getIdent("as").id: # XXX hack ahead: n.kind = nkImportAs n.sons[0] = n.sons[1] n.sons[1] = n.sons[2] n.sons.setLen(2) return getModuleName(n.sons[0]) - # hacky way to implement 'x / y /../ z': - result = renderTree(n, {renderNoComments}).replace(" ") + if n1.kind == nkPrefix and n1[0].kind == nkIdent and n1[0].ident.s == "$": + if n0.kind == nkIdent and n0.ident.s == "/": + result = lookupPackage(n1[1], n[2]) + else: + localError(n.info, "only '/' supported with $package notation") + result = "" + else: + # hacky way to implement 'x / y /../ z': + result = getModuleName(n1) + result.add renderTree(n0, {renderNoComments}) + result.add getModuleName(n[2]) + of nkPrefix: + if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$": + result = lookupPackage(n[1], nil) + else: + # hacky way to implement 'x / y /../ z': + result = renderTree(n, {renderNoComments}).replace(" ") of nkDotExpr: result = renderTree(n, {renderNoComments}).replace(".", "/") of nkImportAs: @@ -209,12 +237,14 @@ proc evalImport(c: PContext, n: PNode): PNode = for i in countup(0, sonsLen(n) - 1): let it = n.sons[i] if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket: - let sep = renderTree(it.sons[0], {renderNoComments}) - let dir = renderTree(it.sons[1], {renderNoComments}) + let sep = it[0] + let dir = it[1] + let a = newNodeI(nkInfix, it.info) + a.add sep + a.add dir + a.add sep # dummy entry, replaced in the loop for x in it[2]: - let f = renderTree(x, {renderNoComments}) - let a = newStrNode(nkStrLit, (dir & sep & f).replace(" ")) - a.info = it.info + a.sons[2] = x impMod(c, a) else: impMod(c, it) diff --git a/compiler/nimblecmd.nim b/compiler/nimblecmd.nim index ab63f9e12b..39c3a17e75 100644 --- a/compiler/nimblecmd.nim +++ b/compiler/nimblecmd.nim @@ -16,11 +16,11 @@ proc addPath*(path: string, info: TLineInfo) = options.searchPaths.insert(path, 0) type - Version = distinct string + Version* = distinct string -proc `$`(ver: Version): string {.borrow.} +proc `$`*(ver: Version): string {.borrow.} -proc newVersion(ver: string): Version = +proc newVersion*(ver: string): Version = doAssert(ver.len == 0 or ver[0] in {'#', '\0'} + Digits, "Wrong version: " & ver) return Version(ver) @@ -28,7 +28,7 @@ proc newVersion(ver: string): Version = proc isSpecial(ver: Version): bool = return ($ver).len > 0 and ($ver)[0] == '#' -proc `<`(ver: Version, ver2: Version): bool = +proc `<`*(ver: Version, ver2: Version): bool = ## This is synced from Nimble's version module. # Handling for special versions such as "#head" or "#branch". diff --git a/compiler/sem.nim b/compiler/sem.nim index 21fd0b3836..ebfdafea72 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -495,13 +495,15 @@ proc isImportSystemStmt(n: PNode): bool = case n.kind of nkImportStmt: for x in n: - let f = checkModuleName(x, false) + if x.kind == nkIdent: + let f = checkModuleName(x, false) + if f == magicsys.systemModule.info.fileIndex: + return true + of nkImportExceptStmt, nkFromStmt: + if n[0].kind == nkIdent: + let f = checkModuleName(n[0], false) if f == magicsys.systemModule.info.fileIndex: return true - of nkImportExceptStmt, nkFromStmt: - let f = checkModuleName(n[0], false) - if f == magicsys.systemModule.info.fileIndex: - return true else: discard proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = diff --git a/compiler/vm.nim b/compiler/vm.nim index 79bfb911de..08605cad12 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -19,7 +19,7 @@ import ast except getstr import strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes, parser, vmdeps, idents, trees, renderer, options, transf, parseutils, - vmmarshal + vmmarshal, gorgeimpl from semfold import leValueConv, ordinalValToString from evaltempl import evalTemplate diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 2b3cfeeeb4..29c1129b5b 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -7,50 +7,7 @@ # distribution, for details about the copyright. # -import ast, types, msgs, os, osproc, streams, options, idents, securehash - -proc readOutput(p: Process): (string, int) = - result[0] = "" - var output = p.outputStream - while not output.atEnd: - result[0].add(output.readLine) - result[0].add("\n") - if result[0].len > 0: - result[0].setLen(result[0].len - "\n".len) - result[1] = p.waitForExit - -proc opGorge*(cmd, input, cache: string, info: TLineInfo): (string, int) = - let workingDir = parentDir(info.toFullPath) - if cache.len > 0:# and optForceFullMake notin gGlobalOptions: - let h = secureHash(cmd & "\t" & input & "\t" & cache) - let filename = options.toGeneratedFile("gorge_" & $h, "txt") - var f: File - if open(f, filename): - result = (f.readAll, 0) - f.close - return - var readSuccessful = false - try: - var p = startProcess(cmd, workingDir, - options={poEvalCommand, poStderrToStdout}) - if input.len != 0: - p.inputStream.write(input) - p.inputStream.close() - result = p.readOutput - readSuccessful = true - writeFile(filename, result[0]) - except IOError, OSError: - if not readSuccessful: result = ("", -1) - else: - try: - var p = startProcess(cmd, workingDir, - options={poEvalCommand, poStderrToStdout}) - if input.len != 0: - p.inputStream.write(input) - p.inputStream.close() - result = p.readOutput - except IOError, OSError: - result = ("", -1) +import ast, types, msgs, os, streams, options, idents proc opSlurp*(file: string, info: TLineInfo, module: PSym): string = try: diff --git a/koch.nim b/koch.nim index 6ae45fcb7d..6b267ce461 100644 --- a/koch.nim +++ b/koch.nim @@ -262,6 +262,9 @@ proc buildTools(latest: bool) = let nimgrepExe = "bin/nimgrep".exe nimexec "c -o:" & nimgrepExe & " tools/nimgrep.nim" when defined(windows): buildVccTool() + + nimexec "c -o:" & ("bin/nimresolve".exe) & " tools/nimresolve.nim" + buildNimble(latest) proc nsis(args: string) = diff --git a/tools/nimresolve.nim b/tools/nimresolve.nim new file mode 100644 index 0000000000..9af24df5a6 --- /dev/null +++ b/tools/nimresolve.nim @@ -0,0 +1,158 @@ +# +# +# The Nim Compiler +# (c) Copyright 2017 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Standard tool that resolves import paths. + +import + os, strutils, parseopt + +import "../compiler/nimblecmd" + +# You can change these constants to build you own adapted resolver. +const + considerParentDirs = not defined(noParentProjects) + considerNimbleDirs = not defined(noNimbleDirs) + +const + Version = "1.0" + Usage = "nimresolve - Nim Resolve Package Path Version " & Version & """ + + (c) 2017 Andreas Rumpf +Usage: + nimresolve [options] package +Options: + --source:FILE the file that requests to resolve 'package' + --stdlib:PATH the path to use for the standard library + --project:FILE the main '.nim' file that was passed to the Nim compiler + --subdir:EXPR the subdir part in: 'import $pkg / subdir' + --noNimblePath do not search the Nimble path to resolve the package +""" + +proc writeHelp() = + stdout.write(Usage) + stdout.flushFile() + quit(0) + +proc writeVersion() = + stdout.write(Version & "\n") + stdout.flushFile() + quit(0) + +type + Task = object + source, stdlib, subdir, project, pkg: string + noNimblePath: bool + +proc findInNimbleDir(t: Task; dir: string): bool = + var best = "" + var bestv = "" + for k, p in os.walkDir(dir, relative=true): + if k == pcDir and p.len > t.pkg.len+1 and + p[t.pkg.len] == '-' and p.startsWith(t.pkg): + let (_, a) = getPathVersion(p) + if bestv.len == 0 or bestv < a: + bestv = a + best = dir / p + + if best.len > 0: + var f: File + if open(f, best / changeFileExt(t.pkg, ".nimble-link")): + # the second line contains what we're interested in, see: + # https://github.com/nim-lang/nimble#nimble-link + var override = "" + discard readLine(f, override) + discard readLine(f, override) + close(f) + if not override.isAbsolute(): + best = best / override + else: + best = override + let f = if t.subdir.len == 0: t.pkg else: t.subdir + let res = addFileExt(best / f, "nim") + if best.len > 0 and fileExists(res): + echo res + result = true + +const stdlibDirs = [ + "pure", "core", "arch", + "pure/collections", + "pure/concurrency", "impure", + "wrappers", "wrappers/linenoise", + "windows", "posix", "js"] + +proc resolve(t: Task) = + template attempt(a) = + let x = addFileExt(a, "nim") + if fileExists(x): + echo x + return + + case t.pkg + of "stdlib": + if t.subdir.len == 0: + echo t.stdlib + return + else: + for candidate in stdlibDirs: + attempt(t.stdlib / candidate / t.subdir) + of "root": + let root = t.project.splitFile.dir + if t.subdir.len == 0: + echo root + return + else: + attempt(root / t.subdir) + else: + when considerParentDirs: + var p = parentDir(t.source.splitFile.dir) + # support 'import $karax': + let f = if t.subdir.len == 0: t.pkg else: t.subdir + + while p.len > 0: + let dir = p / t.pkg + if dirExists(dir): + attempt(dir / f) + # 2nd attempt: try to use 'karax/karax' + attempt(dir / t.pkg / f) + # 3rd attempt: try to use 'karax/src/karax' + attempt(dir / "src" / f) + attempt(dir / "src" / t.pkg / f) + p = parentDir(p) + + when considerNimbleDirs: + if not t.noNimblePath: + if findInNimbleDir(t, getHomeDir() / ".nimble" / "pkgs"): return + when not defined(windows): + if findInNimbleDir(t, "/opt/nimble/pkgs"): return + + quit "cannot resolve: " & (t.pkg / t.subdir) + +proc main = + var t: Task + t.subdir = "" + for kind, key, val in getopt(): + case kind + of cmdArgument: + t.pkg = key + of cmdLongoption, cmdShortOption: + case normalize(key) + of "source": t.source = val + of "stdlib": t.stdlib = val + of "project": t.project = val + of "subdir": t.subdir = val + of "nonimblepath": t.noNimblePath = true + of "help", "h": writeHelp() + of "version", "v": writeVersion() + else: writeHelp() + of cmdEnd: assert(false) # cannot happen + if t.pkg.len == 0: + quit "[Error] no package to resolve." + resolve(t) + +main()