Atlas: --project switch and better graph representation (#21971)

atlas/atlas.nim

@@ -10,7 +10,7 @@
 ## a Nimble dependency and its dependencies recursively.
 
 import std / [parseopt, strutils, os, osproc, tables, sets, json, jsonutils,
-  parsecfg, streams, terminal, strscans]
+  parsecfg, streams, terminal, strscans, hashes]
 import parse_requires, osutils, packagesjson, compiledpatterns
 
 from unicode import nil
@@ -56,10 +56,12 @@ Options:
   --cfgHere             also create/maintain a nim.cfg in the current
                         working directory
   --workspace=DIR       use DIR as workspace
+  --project=DIR         use DIR as the current project
   --genlock             generate a lock file (use with `clone` and `update`)
   --uselock             use the lock file for the build
   --autoinit            auto initialize a workspace
   --colors=on|off       turn on|off colored output
+  --showGraph           show the dependency graph
   --version             show the version
   --help                show this help
 """
@@ -76,7 +78,7 @@ proc writeVersion() =
 
 const
   MockupRun = defined(atlasTests)
-  TestsDir = "tools/atlas/tests"
+  TestsDir = "atlas/tests"
 
 type
   LockOption = enum
@@ -97,13 +99,16 @@ type
     name: PackageName
     url, commit: string
     rel: DepRelation # "requires x < 1.0" is silly, but Nimble allows it so we have too.
+    self: int # position in the graph
     parents: seq[int] # why we need this dependency
+    active: bool
   DepGraph = object
     nodes: seq[Dependency]
     processed: Table[string, int] # the key is (url / commit)
+    byName: Table[PackageName, seq[int]]
 
   AtlasContext = object
-    projectDir, workspace, depsDir: string
+    projectDir, workspace, depsDir, currentDir: string
     hasPackageList: bool
     keepCommits: bool
     cfgHere: bool
@@ -115,12 +120,15 @@ type
     lockFileToWrite: seq[LockFileEntry]
     lockFileToUse: Table[string, LockFileEntry]
     when MockupRun:
-      currentDir: string
       step: int
       mockupSuccess: bool
     noColors: bool
+    showGraph: bool
 
-proc `==`(a, b: CfgPath): bool {.borrow.}
+proc `==`*(a, b: CfgPath): bool {.borrow.}
+
+proc `==`*(a, b: PackageName): bool {.borrow.}
+proc hash*(a: PackageName): Hash {.borrow.}
 
 const
   InvalidCommit = "<invalid commit>"
@@ -188,7 +196,6 @@ proc cloneUrl(c: var AtlasContext; url, dest: string; cloneUsingHttps: bool): st
 
 template withDir*(c: var AtlasContext; dir: string; body: untyped) =
   when MockupRun:
-    c.currentDir = dir
     body
   else:
     let oldDir = getCurrentDir()
@@ -243,7 +250,9 @@ proc info(c: var AtlasContext; p: PackageName; arg: string) =
   else:
     stdout.styledWriteLine(fgGreen, styleBright, "[Info] ", resetStyle, fgCyan, "(", p.string, ")", resetStyle, " ", arg)
 
-template projectFromCurrentDir(): PackageName = PackageName(getCurrentDir().splitPath.tail)
+template projectFromCurrentDir(): PackageName = PackageName(c.currentDir.splitPath.tail)
+
+proc readableFile(s: string): string = relativePath(s, getCurrentDir())
 
 proc sameVersionAs(tag, ver: string): bool =
   const VersionChars = {'0'..'9', '.'}
@@ -414,6 +423,30 @@ proc toName(p: string): PackageName =
   else:
     result = PackageName p
 
+proc generateDepGraph(c: var AtlasContext; g: DepGraph) =
+  proc repr(w: Dependency): string = w.url / w.commit
+
+  var dotGraph = ""
+  for i in 0 ..< g.nodes.len:
+    dotGraph.addf("\"$1\" [label=\"$2\"];\n", [g.nodes[i].repr, if g.nodes[i].active: "" else: "unused"])
+  for i in 0 ..< g.nodes.len:
+    for p in items g.nodes[i].parents:
+      if p >= 0:
+        dotGraph.addf("\"$1\" -> \"$2\";\n", [g.nodes[p].repr, g.nodes[i].repr])
+  let dotFile = c.workspace / "deps.dot"
+  writeFile(dotFile, "digraph deps {\n$1}\n" % dotGraph)
+  let graphvizDotPath = findExe("dot")
+  if graphvizDotPath.len == 0:
+    #echo("gendepend: Graphviz's tool dot is required, " &
+    #  "see https://graphviz.org/download for downloading")
+    discard
+  else:
+    discard execShellCmd("dot -Tpng -odeps.png " & quoteShell(dotFile))
+
+proc showGraph(c: var AtlasContext; g: DepGraph) =
+  if c.showGraph:
+    generateDepGraph c, g
+
 proc needsCommitLookup(commit: string): bool {.inline.} =
   '.' in commit or commit == InvalidCommit
 
@@ -452,7 +485,12 @@ proc dependencyDir(c: AtlasContext; w: Dependency): string =
   if not dirExists(result):
     result = c.depsDir / w.name.string
 
-proc checkoutCommit(c: var AtlasContext; w: Dependency) =
+proc selectNode(c: var AtlasContext; g: var DepGraph; w: Dependency) =
+  # all other nodes of the same project name are not active
+  for e in items g.byName[w.name]:
+    g.nodes[e].active = e == w.self
+
+proc checkoutCommit(c: var AtlasContext; g: var DepGraph; w: Dependency) =
   let dir = dependencyDir(c, w)
   withDir c, dir:
     if c.lockOption == genLock:
@@ -484,8 +522,10 @@ proc checkoutCommit(c: var AtlasContext; w: Dependency) =
               # conflict resolution: pick the later commit:
               if mergeBase == currentCommit:
                 checkoutGitCommit(c, w.name, requiredCommit)
+                selectNode c, g, w
             else:
               checkoutGitCommit(c, w.name, requiredCommit)
+              selectNode c, g, w
               when false:
                 warn c, w.name, "do not know which commit is more recent:",
                   currentCommit, "(current) or", w.commit, " =", requiredCommit, "(required)"
@@ -521,16 +561,20 @@ proc addUniqueDep(c: var AtlasContext; g: var DepGraph; parent: int;
     if g.processed.hasKey(key):
       g.nodes[g.processed[key]].parents.addUnique parent
     else:
-      g.processed[key] = g.nodes.len
+      let self = g.nodes.len
+      g.byName.mgetOrPut(toName(pkgName), @[]).add self
+      g.processed[key] = self
       if lockfile.contains(pkgName):
         g.nodes.add Dependency(name: toName(pkgName),
                             url: lockfile[pkgName].url,
                             commit: lockfile[pkgName].commit,
                             rel: normal,
+                            self: self,
                             parents: @[parent])
       else:
         g.nodes.add Dependency(name: toName(pkgName), url: url, commit: tokens[2],
                                rel: toDepRelation(tokens[1]),
+                               self: self,
                                parents: @[parent])
 
 template toDestDir(p: PackageName): string = p.string
@@ -582,6 +626,37 @@ proc collectNewDeps(c: var AtlasContext; g: var DepGraph; parent: int;
   else:
     result = CfgPath toDestDir(dep.name)
 
+proc selectDir(a, b: string): string = (if dirExists(a): a else: b)
+
+const
+  FileProtocol = "file://"
+  ThisVersion = "current_version.atlas"
+
+proc copyFromDisk(c: var AtlasContext; w: Dependency) =
+  let destDir = toDestDir(w.name)
+  var u = w.url.substr(FileProtocol.len)
+  if u.startsWith("./"): u = c.workspace / u.substr(2)
+  copyDir(selectDir(u & "@" & w.commit, u), destDir)
+  writeFile destDir / ThisVersion, w.commit
+  #echo "WRITTEN ", destDir / ThisVersion
+
+proc isNewerVersion(a, b: string): bool =
+  # only used for testing purposes.
+  if a == InvalidCommit or b == InvalidCommit:
+    return true
+  var amajor, aminor, apatch, bmajor, bminor, bpatch: int
+  if scanf(a, "$i.$i.$i", amajor, aminor, apatch):
+    assert scanf(b, "$i.$i.$i", bmajor, bminor, bpatch)
+    result = (amajor, aminor, apatch) > (bmajor, bminor, bpatch)
+  else:
+    assert scanf(a, "$i.$i", amajor, aminor)
+    assert scanf(b, "$i.$i", bmajor, bminor)
+    result = (amajor, aminor) > (bmajor, bminor)
+
+proc isLaterCommit(destDir, version: string): bool =
+  let oldVersion = try: readFile(destDir / ThisVersion).strip: except: "0.0"
+  result = isNewerVersion(version, oldVersion)
+
 proc traverseLoop(c: var AtlasContext; g: var DepGraph; startIsDep: bool): seq[CfgPath] =
   result = @[]
   var i = 0
@@ -592,11 +667,23 @@ proc traverseLoop(c: var AtlasContext; g: var DepGraph; startIsDep: bool): seq[C
 
     if not dirExists(c.workspace / destDir) and not dirExists(c.depsDir / destDir):
       withDir c, (if i != 0 or startIsDep: c.depsDir else: c.workspace):
-        let err = cloneUrl(c, w.url, destDir, false)
-        if err != "":
-          error c, w.name, err
+        if w.url.startsWith(FileProtocol):
+          copyFromDisk c, w
+        else:
+          let err = cloneUrl(c, w.url, destDir, false)
+          if err != "":
+            error c, w.name, err
+    # assume this is the selected version, it might get overwritten later:
+    selectNode c, g, w
     if oldErrors == c.errors:
-      if not c.keepCommits: checkoutCommit(c, w)
+      if not c.keepCommits:
+        if not w.url.startsWith(FileProtocol):
+          checkoutCommit(c, g, w)
+        else:
+          withDir c, (if i != 0 or startIsDep: c.depsDir else: c.workspace):
+            if isLaterCommit(destDir, w.commit):
+              copyFromDisk c, w
+              selectNode c, g, w
       # even if the checkout fails, we can make use of the somewhat
       # outdated .nimble file to clone more of the most likely still relevant
       # dependencies:
@@ -606,7 +693,8 @@ proc traverseLoop(c: var AtlasContext; g: var DepGraph; startIsDep: bool): seq[C
 proc traverse(c: var AtlasContext; start: string; startIsDep: bool): seq[CfgPath] =
   # returns the list of paths for the nim.cfg file.
   let url = toUrl(c, start)
-  var g = DepGraph(nodes: @[Dependency(name: toName(start), url: url, commit: "")])
+  var g = DepGraph(nodes: @[Dependency(name: toName(start), url: url, commit: "", self: 0)])
+  g.byName.mgetOrPut(toName(start), @[]).add 0
 
   if url == "":
     error c, toName(start), "cannot resolve package name"
@@ -618,6 +706,7 @@ proc traverse(c: var AtlasContext; start: string; startIsDep: bool): seq[CfgPath
   result = traverseLoop(c, g, startIsDep)
   if c.lockOption == genLock:
     writeFile c.projectDir / LockFileName, toJson(c.lockFileToWrite).pretty
+  showGraph c, g
 
 const
   configPatternBegin = "############# begin Atlas config section ##########\n"
@@ -638,11 +727,12 @@ proc patchNimCfg(c: var AtlasContext; deps: seq[CfgPath]; cfgPath: string) =
     c.mockupSuccess = true
   else:
     let cfg = cfgPath / "nim.cfg"
+    assert cfgPath.len > 0
     if cfgPath.len > 0 and not dirExists(cfgPath):
       error(c, c.projectDir.PackageName, "could not write the nim.cfg")
     elif not fileExists(cfg):
       writeFile(cfg, cfgContent)
-      info(c, projectFromCurrentDir(), "created: " & cfg)
+      info(c, projectFromCurrentDir(), "created: " & cfg.readableFile)
     else:
       let content = readFile(cfg)
       let start = content.find(configPatternBegin)
@@ -657,7 +747,7 @@ proc patchNimCfg(c: var AtlasContext; deps: seq[CfgPath]; cfgPath: string) =
         # do not touch the file if nothing changed
         # (preserves the file date information):
         writeFile(cfg, cfgContent)
-        info(c, projectFromCurrentDir(), "updated: " & cfg)
+        info(c, projectFromCurrentDir(), "updated: " & cfg.readableFile)
 
 proc fatal*(msg: string) =
   when defined(debug):
@@ -665,36 +755,21 @@ proc fatal*(msg: string) =
   quit "[Error] " & msg
 
 proc findSrcDir(c: var AtlasContext): string =
-  for nimbleFile in walkPattern("*.nimble"):
+  for nimbleFile in walkPattern(c.currentDir / "*.nimble"):
     let nimbleInfo = extractRequiresInfo(c, nimbleFile)
-    return nimbleInfo.srcDir
-  return ""
-
-proc generateDepGraph(g: DepGraph) =
-  # currently unused.
-  var dotGraph = ""
-  for i in 0 ..< g.nodes.len:
-    for p in items g.nodes[i].parents:
-      if p >= 0:
-        dotGraph.addf("\"$1\" -> \"$2\";\n", [g.nodes[p].name.string, g.nodes[i].name.string])
-  writeFile("deps.dot", "digraph deps {\n$1}\n" % dotGraph)
-  let graphvizDotPath = findExe("dot")
-  if graphvizDotPath.len == 0:
-    #echo("gendepend: Graphviz's tool dot is required, " &
-    #  "see https://graphviz.org/download for downloading")
-    discard
-  else:
-    discard execShellCmd("dot -Tpng -odeps.png deps.dot")
+    return c.currentDir / nimbleInfo.srcDir
+  return c.currentDir
 
 proc installDependencies(c: var AtlasContext; nimbleFile: string; startIsDep: bool) =
   # 1. find .nimble file in CWD
   # 2. install deps from .nimble
   var g = DepGraph(nodes: @[])
   let (_, pkgname, _) = splitFile(nimbleFile)
-  let dep = Dependency(name: toName(pkgname), url: "", commit: "")
+  let dep = Dependency(name: toName(pkgname), url: "", commit: "", self: 0)
   discard collectDeps(c, g, -1, dep, nimbleFile)
   let paths = traverseLoop(c, g, startIsDep)
-  patchNimCfg(c, paths, if c.cfgHere: getCurrentDir() else: findSrcDir(c))
+  patchNimCfg(c, paths, if c.cfgHere: c.currentDir else: findSrcDir(c))
+  showGraph c, g
 
 proc updateDir(c: var AtlasContext; dir, filter: string) =
   for kind, file in walkDir(dir):
@@ -718,14 +793,14 @@ proc updateDir(c: var AtlasContext; dir, filter: string) =
               error c, pkg, "could not fetch current branch name"
 
 proc patchNimbleFile(c: var AtlasContext; dep: string): string =
-  let thisProject = getCurrentDir().splitPath.tail
+  let thisProject = c.currentDir.splitPath.tail
   let oldErrors = c.errors
   let url = toUrl(c, dep)
   result = ""
   if oldErrors != c.errors:
     warn c, toName(dep), "cannot resolve package name"
   else:
-    for x in walkFiles("*.nimble"):
+    for x in walkFiles(c.currentDir / "*.nimble"):
       if result.len == 0:
         result = x
       else:
@@ -754,16 +829,16 @@ proc patchNimbleFile(c: var AtlasContext; dep: string): string =
       if result.len > 0:
         let oldContent = readFile(result)
         writeFile result, oldContent & "\n" & line
-        info(c, toName(thisProject), "updated: " & result)
+        info(c, toName(thisProject), "updated: " & result.readableFile)
       else:
-        result = thisProject & ".nimble"
+        result = c.currentDir / thisProject & ".nimble"
         writeFile result, line
-        info(c, toName(thisProject), "created: " & result)
+        info(c, toName(thisProject), "created: " & result.readableFile)
     else:
-      info(c, toName(thisProject), "up to date: " & result)
+      info(c, toName(thisProject), "up to date: " & result.readableFile)
 
-proc detectWorkspace(): string =
-  result = getCurrentDir()
+proc detectWorkspace(currentDir: string): string =
+  result = currentDir
   while result.len > 0:
     if fileExists(result / AtlasWorkspace):
       return result
@@ -777,8 +852,8 @@ proc absoluteDepsDir(workspace, value: string): string =
   else:
     result = workspace / value
 
-proc autoWorkspace(): string =
-  result = getCurrentDir()
+proc autoWorkspace(currentDir: string): string =
+  result = currentDir
   while result.len > 0 and dirExists(result / ".git"):
     result = result.parentDir()
 
@@ -868,7 +943,7 @@ proc setupNimEnv(c: var AtlasContext; nimVersion: string) =
       error c, toName("nim"), "cannot parse version requirement"
       return
   let csourcesVersion =
-    if nimVersion.isDevel or (major >= 1 and minor >= 9) or major >= 2:
+    if nimVersion.isDevel or (major == 1 and minor >= 9) or major >= 2:
       # already uses csources_v2
       "csources_v2"
     elif major == 0:
@@ -892,7 +967,7 @@ proc setupNimEnv(c: var AtlasContext; nimVersion: string) =
   withDir c, c.workspace / nimDest:
     let nimExe = "bin" / "nim".addFileExt(ExeExt)
     copyFileWithPermissions nimExe0, nimExe
-    let dep = Dependency(name: toName(nimDest), rel: normal, commit: nimVersion)
+    let dep = Dependency(name: toName(nimDest), rel: normal, commit: nimVersion, self: 0)
     if not nimVersion.isDevel:
       let commit = versionToCommit(c, dep)
       if commit.len == 0:
@@ -926,10 +1001,10 @@ proc main =
       fatal action & " command takes no arguments"
 
   template projectCmd() =
-    if getCurrentDir() == c.workspace or getCurrentDir() == c.depsDir:
+    if c.projectDir == c.workspace or c.projectDir == c.depsDir:
       fatal action & " command must be executed in a project, not in the workspace"
 
-  var c = AtlasContext(projectDir: getCurrentDir(), workspace: "")
+  var c = AtlasContext(projectDir: getCurrentDir(), currentDir: getCurrentDir(), workspace: "")
   var autoinit = false
   for kind, key, val in getopt():
     case kind
@@ -953,6 +1028,11 @@ proc main =
           createWorkspaceIn c.workspace, c.depsDir
         else:
           writeHelp()
+      of "project":
+        if isAbsolute(val):
+          c.currentDir = val
+        else:
+          c.currentDir = getCurrentDir() / val
       of "deps":
         if val.len > 0:
           c.depsDir = val
@@ -960,6 +1040,7 @@ proc main =
           writeHelp()
       of "cfghere": c.cfgHere = true
       of "autoinit": autoinit = true
+      of "showgraph": c.showGraph = true
       of "genlock":
         if c.lockOption != useLock:
           c.lockOption = genLock
@@ -982,14 +1063,14 @@ proc main =
     if not dirExists(c.workspace): fatal "Workspace directory '" & c.workspace & "' not found."
   elif action != "init":
     when MockupRun:
-      c.workspace = autoWorkspace()
+      c.workspace = autoWorkspace(c.currentDir)
     else:
-      c.workspace = detectWorkspace()
+      c.workspace = detectWorkspace(c.currentDir)
       if c.workspace.len > 0:
         readConfig c
-        info c, toName(c.workspace), "is the current workspace"
+        info c, toName(c.workspace.readableFile), "is the current workspace"
       elif autoinit:
-        c.workspace = autoWorkspace()
+        c.workspace = autoWorkspace(c.currentDir)
         createWorkspaceIn c.workspace, c.depsDir
       elif action notin ["search", "list"]:
         fatal "No workspace found. Run `atlas init` if you want this current directory to be your workspace."
@@ -1006,7 +1087,7 @@ proc main =
   of "clone", "update":
     singleArg()
     let deps = traverse(c, args[0], startIsDep = false)
-    patchNimCfg c, deps, if c.cfgHere: getCurrentDir() else: findSrcDir(c)
+    patchNimCfg c, deps, if c.cfgHere: c.currentDir else: findSrcDir(c)
     when MockupRun:
       if not c.mockupSuccess:
         fatal "There were problems."

atlas/tests/ws_conflict/atlas.workspace

@@ -0,0 +1,2 @@
+deps=""
+overrides="url.rules"

atlas/tests/ws_conflict/source/apkg/apkg.nimble

@@ -0,0 +1,4 @@
+# require first b and then c
+
+requires "https://github.com/bpkg >= 1.0"
+requires "https://github.com/cpkg >= 1.0"

atlas/tests/ws_conflict/source/bpkg@1.0/bpkg.nimble

@@ -0,0 +1 @@
+requires "https://github.com/cpkg >= 2.0"

atlas/tests/ws_conflict/source/cpkg@1.0/cpkg.nimble

@@ -0,0 +1 @@
+# No dependency here!

atlas/tests/ws_conflict/source/cpkg@2.0/cpkg.nimble

@@ -0,0 +1 @@
+requires "https://github.com/dpkg >= 1.0"

atlas/tests/ws_conflict/url.rules

@@ -0,0 +1 @@
+https://github.com/$+ -> file://./source/$#

koch.nim

@@ -179,7 +179,7 @@ proc bundleWinTools(args: string) =
   buildVccTool(args)
   nimCompile("tools/nimgrab.nim", options = "-d:ssl " & args)
   nimCompile("tools/nimgrep.nim", options = args)
-  nimCompile("tools/atlas/atlas.nim", options = args)
+  nimCompile("atlas/atlas.nim", options = args)
   nimCompile("testament/testament.nim", options = args)
   when false:
     # not yet a tool worth including
@@ -236,7 +236,7 @@ proc buildTools(args: string = "") =
       "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args,
       outputName = "nim_dbg")
 
-  nimCompileFold("Compile atlas", "tools/atlas/atlas.nim", options = "-d:release " & args,
+  nimCompileFold("Compile atlas", "atlas/atlas.nim", options = "-d:release " & args,
       outputName = "atlas")
 
 proc testTools(args: string = "") =
@@ -245,7 +245,7 @@ proc testTools(args: string = "") =
   when defined(windows): buildVccTool(args)
   bundleNimpretty(args)
   nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args)
-  nimCompileFold("Compile atlas", "tools/atlas/atlas.nim", options = "-d:release " & args,
+  nimCompileFold("Compile atlas", "atlas/atlas.nim", options = "-d:release " & args,
       outputName = "atlas")
 
 proc nsis(latest: bool; args: string) =
@@ -612,7 +612,7 @@ proc runCI(cmd: string) =
       execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest")
       execFold("Run nimsuggest tests", "nim r nimsuggest/tester")
 
-    execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls")
+    execFold("Run atlas tests", "nim c -r -d:atlasTests atlas/atlas.nim clone https://github.com/disruptek/balls")
 
     kochExecFold("Testing booting in refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib")