implemented new experimental scriptable import mechanism

compiler/gorgeimpl.nim

@@ -0,0 +1,83 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Module that implements ``gorge`` for the compiler as well as
+## the scriptable import mechanism.
+
+import msgs, securehash, os, osproc, streams, strutils, options
+
+proc readOutput(p: Process): (string, int) =
+  result[0] = ""
+  var output = p.outputStream
+  while not output.atEnd:
+    result[0].add(output.readLine)
+    result[0].add("\n")
+  if result[0].len > 0:
+    result[0].setLen(result[0].len - "\n".len)
+  result[1] = p.waitForExit
+
+proc opGorge*(cmd, input, cache: string, info: TLineInfo): (string, int) =
+  let workingDir = parentDir(info.toFullPath)
+  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
+    let h = secureHash(cmd & "\t" & input & "\t" & cache)
+    let filename = options.toGeneratedFile("gorge_" & $h, "txt")
+    var f: File
+    if open(f, filename):
+      result = (f.readAll, 0)
+      f.close
+      return
+    var readSuccessful = false
+    try:
+      var p = startProcess(cmd, workingDir,
+                           options={poEvalCommand, poStderrToStdout})
+      if input.len != 0:
+        p.inputStream.write(input)
+        p.inputStream.close()
+      result = p.readOutput
+      readSuccessful = true
+      # only cache successful runs:
+      if result[1] == 0:
+        writeFile(filename, result[0])
+    except IOError, OSError:
+      if not readSuccessful: result = ("", -1)
+  else:
+    try:
+      var p = startProcess(cmd, workingDir,
+                           options={poEvalCommand, poStderrToStdout})
+      if input.len != 0:
+        p.inputStream.write(input)
+        p.inputStream.close()
+      result = p.readOutput
+    except IOError, OSError:
+      result = ("", -1)
+
+proc scriptableImport*(pkg, subdir: string; info: TLineInfo): string =
+  var cmd = getConfigVar("resolver.exe")
+  if cmd.len == 0: cmd = "nimresolve"
+  else: cmd = quoteShell(cmd)
+  cmd.add " --source:"
+  cmd.add quoteShell(info.toFullPath())
+  cmd.add " --stdlib:"
+  cmd.add quoteShell(options.libpath)
+  cmd.add "  --project:"
+  cmd.add quoteShell(gProjectFull)
+  if subdir.len != 0:
+    cmd.add " --subdir:"
+    cmd.add quoteShell(subdir)
+  if options.gNoNimblePath:
+    cmd.add " --nonimblepath"
+  cmd.add ' '
+  cmd.add quoteShell(pkg)
+  let (res, exitCode) = opGorge(cmd, "", cmd, info)
+  if exitCode == 0:
+    result = res.strip()
+  elif res.len > 0:
+    localError(info, res)
+  else:
+    localError(info, "cannot resolve: " & (pkg / subdir))

compiler/importer.nim

@@ -11,11 +11,22 @@
 
 import
   intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups,
-  semdata, passes, renderer
+  semdata, passes, renderer, gorgeimpl
 
 proc evalImport*(c: PContext, n: PNode): PNode
 proc evalFrom*(c: PContext, n: PNode): PNode
 
+proc lookupPackage(pkg, subdir: PNode): string =
+  let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
+  case pkg.kind
+  of nkStrLit, nkRStrLit, nkTripleStrLit:
+    result = scriptableImport(pkg.strVal, sub, pkg.info)
+  of nkIdent:
+    result = scriptableImport(pkg.ident.s, sub, pkg.info)
+  else:
+    localError(pkg.info, "package name must be an identifier or string literal")
+    result = ""
+
 proc getModuleName*(n: PNode): string =
   # This returns a short relative module name without the nim extension
   # e.g. like "system", "importer" or "somepath/module"
@@ -31,16 +42,33 @@ proc getModuleName*(n: PNode): string =
     result = n.ident.s
   of nkSym:
     result = n.sym.name.s
-  of nkInfix, nkPrefix:
-    if n.sons[0].kind == nkIdent and n.sons[0].ident.id == getIdent("as").id:
+  of nkInfix:
+    let n0 = n[0]
+    let n1 = n[1]
+    if n0.kind == nkIdent and n0.ident.id == getIdent("as").id:
       # XXX hack ahead:
       n.kind = nkImportAs
       n.sons[0] = n.sons[1]
       n.sons[1] = n.sons[2]
       n.sons.setLen(2)
       return getModuleName(n.sons[0])
-    # hacky way to implement 'x / y /../ z':
-    result = renderTree(n, {renderNoComments}).replace(" ")
+    if n1.kind == nkPrefix and n1[0].kind == nkIdent and n1[0].ident.s == "$":
+      if n0.kind == nkIdent and n0.ident.s == "/":
+        result = lookupPackage(n1[1], n[2])
+      else:
+        localError(n.info, "only '/' supported with $package notation")
+        result = ""
+    else:
+      # hacky way to implement 'x / y /../ z':
+      result = getModuleName(n1)
+      result.add renderTree(n0, {renderNoComments})
+      result.add getModuleName(n[2])
+  of nkPrefix:
+    if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$":
+      result = lookupPackage(n[1], nil)
+    else:
+      # hacky way to implement 'x / y /../ z':
+      result = renderTree(n, {renderNoComments}).replace(" ")
   of nkDotExpr:
     result = renderTree(n, {renderNoComments}).replace(".", "/")
   of nkImportAs:
@@ -209,12 +237,14 @@ proc evalImport(c: PContext, n: PNode): PNode =
   for i in countup(0, sonsLen(n) - 1):
     let it = n.sons[i]
     if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
-      let sep = renderTree(it.sons[0], {renderNoComments})
-      let dir = renderTree(it.sons[1], {renderNoComments})
+      let sep = it[0]
+      let dir = it[1]
+      let a = newNodeI(nkInfix, it.info)
+      a.add sep
+      a.add dir
+      a.add sep # dummy entry, replaced in the loop
       for x in it[2]:
-        let f = renderTree(x, {renderNoComments})
-        let a = newStrNode(nkStrLit, (dir & sep & f).replace(" "))
-        a.info = it.info
+        a.sons[2] = x
         impMod(c, a)
     else:
       impMod(c, it)

compiler/nimblecmd.nim

@@ -16,11 +16,11 @@ proc addPath*(path: string, info: TLineInfo) =
     options.searchPaths.insert(path, 0)
 
 type
-  Version = distinct string
+  Version* = distinct string
 
-proc `$`(ver: Version): string {.borrow.}
+proc `$`*(ver: Version): string {.borrow.}
 
-proc newVersion(ver: string): Version =
+proc newVersion*(ver: string): Version =
   doAssert(ver.len == 0 or ver[0] in {'#', '\0'} + Digits,
            "Wrong version: " & ver)
   return Version(ver)
@@ -28,7 +28,7 @@ proc newVersion(ver: string): Version =
 proc isSpecial(ver: Version): bool =
   return ($ver).len > 0 and ($ver)[0] == '#'
 
-proc `<`(ver: Version, ver2: Version): bool =
+proc `<`*(ver: Version, ver2: Version): bool =
   ## This is synced from Nimble's version module.
 
   # Handling for special versions such as "#head" or "#branch".

compiler/sem.nim

@@ -495,13 +495,15 @@ proc isImportSystemStmt(n: PNode): bool =
   case n.kind
   of nkImportStmt:
     for x in n:
-      let f = checkModuleName(x, false)
+      if x.kind == nkIdent:
+        let f = checkModuleName(x, false)
+        if f == magicsys.systemModule.info.fileIndex:
+          return true
+  of nkImportExceptStmt, nkFromStmt:
+    if n[0].kind == nkIdent:
+      let f = checkModuleName(n[0], false)
       if f == magicsys.systemModule.info.fileIndex:
         return true
-  of nkImportExceptStmt, nkFromStmt:
-    let f = checkModuleName(n[0], false)
-    if f == magicsys.systemModule.info.fileIndex:
-      return true
   else: discard
 
 proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =

compiler/vm.nim

@@ -19,7 +19,7 @@ import ast except getstr
 import
   strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes,
   parser, vmdeps, idents, trees, renderer, options, transf, parseutils,
-  vmmarshal
+  vmmarshal, gorgeimpl
 
 from semfold import leValueConv, ordinalValToString
 from evaltempl import evalTemplate

compiler/vmdeps.nim

@@ -7,50 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-import ast, types, msgs, os, osproc, streams, options, idents, securehash
-
-proc readOutput(p: Process): (string, int) =
-  result[0] = ""
-  var output = p.outputStream
-  while not output.atEnd:
-    result[0].add(output.readLine)
-    result[0].add("\n")
-  if result[0].len > 0:
-    result[0].setLen(result[0].len - "\n".len)
-  result[1] = p.waitForExit
-
-proc opGorge*(cmd, input, cache: string, info: TLineInfo): (string, int) =
-  let workingDir = parentDir(info.toFullPath)
-  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
-    let h = secureHash(cmd & "\t" & input & "\t" & cache)
-    let filename = options.toGeneratedFile("gorge_" & $h, "txt")
-    var f: File
-    if open(f, filename):
-      result = (f.readAll, 0)
-      f.close
-      return
-    var readSuccessful = false
-    try:
-      var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
-      if input.len != 0:
-        p.inputStream.write(input)
-        p.inputStream.close()
-      result = p.readOutput
-      readSuccessful = true
-      writeFile(filename, result[0])
-    except IOError, OSError:
-      if not readSuccessful: result = ("", -1)
-  else:
-    try:
-      var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
-      if input.len != 0:
-        p.inputStream.write(input)
-        p.inputStream.close()
-      result = p.readOutput
-    except IOError, OSError:
-      result = ("", -1)
+import ast, types, msgs, os, streams, options, idents
 
 proc opSlurp*(file: string, info: TLineInfo, module: PSym): string =
   try:

koch.nim

@@ -262,6 +262,9 @@ proc buildTools(latest: bool) =
   let nimgrepExe = "bin/nimgrep".exe
   nimexec "c -o:" & nimgrepExe & " tools/nimgrep.nim"
   when defined(windows): buildVccTool()
+
+  nimexec "c -o:" & ("bin/nimresolve".exe) & " tools/nimresolve.nim"
+
   buildNimble(latest)
 
 proc nsis(args: string) =

tools/nimresolve.nim

@@ -0,0 +1,158 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Standard tool that resolves import paths.
+
+import
+  os, strutils, parseopt
+
+import "../compiler/nimblecmd"
+
+# You can change these constants to build you own adapted resolver.
+const
+  considerParentDirs = not defined(noParentProjects)
+  considerNimbleDirs = not defined(noNimbleDirs)
+
+const
+  Version = "1.0"
+  Usage = "nimresolve - Nim Resolve Package Path Version " & Version & """
+
+  (c) 2017 Andreas Rumpf
+Usage:
+  nimresolve [options] package
+Options:
+  --source:FILE       the file that requests to resolve 'package'
+  --stdlib:PATH       the path to use for the standard library
+  --project:FILE      the main '.nim' file that was passed to the Nim compiler
+  --subdir:EXPR       the subdir part in: 'import $pkg / subdir'
+  --noNimblePath      do not search the Nimble path to resolve the package
+"""
+
+proc writeHelp() =
+  stdout.write(Usage)
+  stdout.flushFile()
+  quit(0)
+
+proc writeVersion() =
+  stdout.write(Version & "\n")
+  stdout.flushFile()
+  quit(0)
+
+type
+  Task = object
+    source, stdlib, subdir, project, pkg: string
+    noNimblePath: bool
+
+proc findInNimbleDir(t: Task; dir: string): bool =
+  var best = ""
+  var bestv = ""
+  for k, p in os.walkDir(dir, relative=true):
+    if k == pcDir and p.len > t.pkg.len+1 and
+        p[t.pkg.len] == '-' and p.startsWith(t.pkg):
+      let (_, a) = getPathVersion(p)
+      if bestv.len == 0 or bestv < a:
+        bestv = a
+        best = dir / p
+
+  if best.len > 0:
+    var f: File
+    if open(f, best / changeFileExt(t.pkg, ".nimble-link")):
+      # the second line contains what we're interested in, see:
+      # https://github.com/nim-lang/nimble#nimble-link
+      var override = ""
+      discard readLine(f, override)
+      discard readLine(f, override)
+      close(f)
+      if not override.isAbsolute():
+        best = best / override
+      else:
+        best = override
+  let f = if t.subdir.len == 0: t.pkg else: t.subdir
+  let res = addFileExt(best / f, "nim")
+  if best.len > 0 and fileExists(res):
+    echo res
+    result = true
+
+const stdlibDirs = [
+  "pure", "core", "arch",
+  "pure/collections",
+  "pure/concurrency", "impure",
+  "wrappers", "wrappers/linenoise",
+  "windows", "posix", "js"]
+
+proc resolve(t: Task) =
+  template attempt(a) =
+    let x = addFileExt(a, "nim")
+    if fileExists(x):
+      echo x
+      return
+
+  case t.pkg
+  of "stdlib":
+    if t.subdir.len == 0:
+      echo t.stdlib
+      return
+    else:
+      for candidate in stdlibDirs:
+        attempt(t.stdlib / candidate / t.subdir)
+  of "root":
+    let root = t.project.splitFile.dir
+    if t.subdir.len == 0:
+      echo root
+      return
+    else:
+      attempt(root / t.subdir)
+  else:
+    when considerParentDirs:
+      var p = parentDir(t.source.splitFile.dir)
+      # support 'import $karax':
+      let f = if t.subdir.len == 0: t.pkg else: t.subdir
+
+      while p.len > 0:
+        let dir = p / t.pkg
+        if dirExists(dir):
+          attempt(dir / f)
+          # 2nd attempt: try to use 'karax/karax'
+          attempt(dir / t.pkg / f)
+          # 3rd attempt: try to use 'karax/src/karax'
+          attempt(dir / "src" / f)
+          attempt(dir / "src" / t.pkg / f)
+        p = parentDir(p)
+
+    when considerNimbleDirs:
+      if not t.noNimblePath:
+        if findInNimbleDir(t, getHomeDir() / ".nimble" / "pkgs"): return
+        when not defined(windows):
+          if findInNimbleDir(t, "/opt/nimble/pkgs"): return
+
+  quit "cannot resolve: " & (t.pkg / t.subdir)
+
+proc main =
+  var t: Task
+  t.subdir = ""
+  for kind, key, val in getopt():
+    case kind
+    of cmdArgument:
+      t.pkg = key
+    of cmdLongoption, cmdShortOption:
+      case normalize(key)
+      of "source": t.source = val
+      of "stdlib": t.stdlib = val
+      of "project": t.project = val
+      of "subdir": t.subdir = val
+      of "nonimblepath": t.noNimblePath = true
+      of "help", "h": writeHelp()
+      of "version", "v": writeVersion()
+      else: writeHelp()
+    of cmdEnd: assert(false) # cannot happen
+  if t.pkg.len == 0:
+    quit "[Error] no package to resolve."
+  resolve(t)
+
+main()