mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-06-07 04:14:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

added Atlas helper tool (#18497)

Browse Source 
* added Atlas helper tool
* further improvements
This commit is contained in:
Andreas Rumpf
2021-07-16 07:42:35 +02:00
committed by GitHub
parent 5e6680406f
commit 089e741ce4
6 changed files with 751 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
koch.nim
View File

@@ -222,7 +222,13 @@ proc buildTools(args: string = "") =
# `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler
# `-d:debug` should be changed to a flag that doesn't require re-compiling nim
# `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces
nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, outputName = "nim_dbg")
nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options =
"--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args,
outputName = "nim_dbg")
nimCompileFold("Compile atlas", "tools/atlas/atlas.nim", options = "-d:release " & args,
outputName = "atlas")
proc nsis(latest: bool; args: string) =
bundleNimbleExe(latest, args)

77
tools/atlas/atlas.md Normal file
View File

@@ -0,0 +1,77 @@
# Atlas Package Cloner
Atlas is a simple package cloner tool that automates some of the
workflows and needs for Nim's stdlib evolution.
Atlas is compatible with Nimble in the sense that it supports the Nimble
file format.
## How it works
Atlas uses git commits internally; version requirements are translated
to git commits via `git show-ref --tags`.
Atlas uses URLs internally; Nimble package names are translated to URLs
via Nimble's `packages.json` file.
Atlas does not call the Nim compiler for a build, instead it creates/patches
a `nim.cfg` file for the compiler. For example:
```
############# begin Atlas config section ##########
--noNimblePath
--path:"../nimx"
--path:"../sdl2/src"
--path:"../opengl/src"
############# end Atlas config section ##########
```
The version selection is deterministic, it picks up the *minimum* required
version. Thanks to this design, lock files are not required.
## Dependencies
Dependencies are neither installed globally, nor locally into the current
project. Instead a "workspace" is used. The workspace is the nearest parent
directory of the current directory that does not contain a `.git` subdirectory.
Dependencies are managed as **siblings**, not as children. Dependencies are
kept as git repositories.
Thanks to this setup, it's easy to develop multiple projects at the same time.
A project plus its dependencies are stored in a workspace:
$workspace / main project
$workspace / dependency A
$workspace / dependency B
No attempts are being made at keeping directory hygiene inside the
workspace, you're supposed to create appropriate `$workspace` directories
at your own leisure.
## Commands
Atlas supports the following commands:
### Clone <url>
Clones a URL and all of its dependencies (recursively) into the workspace.
Creates or patches a `nim.cfg` file with the required `--path` entries.
### Clone <package name>
The `<package name>` is translated into an URL via `packages.json` and
then `clone <url>` is performed.
### Search <term term2 term3 ...>
Search the package index `packages.json` for a package that the given terms
in its description (or name or list of tags).

390
tools/atlas/atlas.nim Normal file
View File

@@ -0,0 +1,390 @@
#
# Atlas Package Cloner
# (c) Copyright 2021 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Simple tool to automate frequent workflows: Can "clone"
## a Nimble dependency and its dependencies recursively.
import std/[parseopt, strutils, os, osproc, sequtils, unicode, tables, sets]
import parse_requires, osutils, packagesjson
const
Version = "0.1"
Usage = "atlas - Nim Package Manager Version " & Version & """
(c) 2021 Andreas Rumpf
Usage:
atlas [options] [command] [arguments]
Command:
clone url|pkgname clone a package and all of its dependencies
search keyw keywB... search for package that contains the given keywords
Options:
--keepCommits do not perform any `git checkouts`
--version show the version
--help show this help
"""
proc writeHelp() =
stdout.write(Usage)
stdout.flushFile()
quit(0)
proc writeVersion() =
stdout.write(Version & "\n")
stdout.flushFile()
quit(0)
type
PackageName = distinct string
DepRelation = enum
normal, strictlyLess, strictlyGreater
Dependency = object
name: PackageName
url, commit: string
rel: DepRelation # "requires x < 1.0" is silly, but Nimble allows it so we have too.
AtlasContext = object
projectDir, workspace: string
hasPackageList: bool
keepCommits: bool
p: Table[string, string] # name -> url mapping
processed: HashSet[string] # the key is (url / commit)
errors: int
const
InvalidCommit = "<invalid commit>"
proc toDepRelation(s: string): DepRelation =
case s
of "<": strictlyLess
of ">": strictlyGreater
else: normal
proc isCleanGit(dir: string): string =
result = ""
let (outp, status) = osproc.execCmdEx("git diff")
if outp.len != 0:
result = "'git diff' not empty"
elif status != 0:
result = "'git diff' returned non-zero"
proc message(c: var AtlasContext; category: string; p: PackageName; args: varargs[string]) =
var msg = category & "(" & p.string & ")"
for a in args:
msg.add ' '
msg.add a
stdout.writeLine msg
inc c.errors
proc warn(c: var AtlasContext; p: PackageName; args: varargs[string]) =
message(c, "[Warning] ", p, args)
proc error(c: var AtlasContext; p: PackageName; args: varargs[string]) =
message(c, "[Error] ", p, args)
proc sameVersionAs(tag, ver: string): bool =
const VersionChars = {'0'..'9', '.'}
proc safeCharAt(s: string; i: int): char {.inline.} =
if i >= 0 and i < s.len: s[i] else: '\0'
let idx = find(tag, ver)
if idx >= 0:
# we found the version as a substring inside the `tag`. But we
# need to watch out the the boundaries are not part of a
# larger/different version number:
result = safeCharAt(tag, idx-1) notin VersionChars and
safeCharAt(tag, idx+ver.len) notin VersionChars
proc versionToCommit(d: Dependency): string =
let (outp, status) = osproc.execCmdEx("git show-ref --tags")
if status == 0:
var useNextOne = false
for line in splitLines(outp):
let commitsAndTags = strutils.splitWhitespace(line)
if commitsAndTags.len == 2:
case d.rel
of normal:
if commitsAndTags[1].sameVersionAs(d.commit):
return commitsAndTags[0]
of strictlyLess:
if d.commit == InvalidCommit or not commitsAndTags[1].sameVersionAs(d.commit):
return commitsAndTags[0]
of strictlyGreater:
if commitsAndTags[1].sameVersionAs(d.commit):
useNextOne = true
elif useNextOne:
return commitsAndTags[0]
return ""
proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) =
let (outp, status) = osproc.execCmdEx("git checkout " & quoteShell(commit))
if status != 0:
error(c, p, "could not checkout commit", commit)
proc gitPull(c: var AtlasContext; p: PackageName) =
let (_, status) = osproc.execCmdEx("git pull")
if status != 0:
error(c, p, "could not 'git pull'")
proc updatePackages(c: var AtlasContext) =
if dirExists(c.workspace / PackagesDir):
withDir(c.workspace / PackagesDir):
gitPull(c, PackageName PackagesDir)
else:
withDir c.workspace:
let err = cloneUrl("https://github.com/nim-lang/packages", PackagesDir, false)
if err != "":
error c, PackageName(PackagesDir), err
proc fillPackageLookupTable(c: var AtlasContext) =
if not c.hasPackageList:
c.hasPackageList = true
updatePackages(c)
let plist = getPackages(c.workspace)
for entry in plist:
c.p[unicode.toLower entry.name] = entry.url
proc toUrl(c: var AtlasContext; p: string): string =
if p.isUrl:
result = p
else:
fillPackageLookupTable(c)
result = c.p.getOrDefault(unicode.toLower p)
if result.len == 0:
inc c.errors
proc toName(p: string): PackageName =
if p.isUrl:
result = PackageName splitFile(p).name
else:
result = PackageName p
proc needsCommitLookup(commit: string): bool {.inline} =
'.' in commit or commit == InvalidCommit
proc checkoutCommit(c: var AtlasContext; w: Dependency) =
let dir = c.workspace / w.name.string
withDir dir:
if w.commit.len == 0 or cmpIgnoreCase(w.commit, "#head") == 0:
gitPull(c, w.name)
else:
let err = isCleanGit(dir)
if err != "":
warn c, w.name, err
else:
let requiredCommit = if needsCommitLookup(w.commit): versionToCommit(w) else: w.commit
let (cc, status) = osproc.execCmdEx("git log -n 1 --format=%H")
let currentCommit = strutils.strip(cc)
if requiredCommit == "" or status != 0:
if requiredCommit == "" and w.commit == InvalidCommit:
warn c, w.name, "package has no tagged releases"
else:
warn c, w.name, "cannot find specified version/commit", w.commit
else:
if currentCommit != requiredCommit:
# checkout the later commit:
# git merge-base --is-ancestor <commit> <commit>
let (mergeBase, status) = osproc.execCmdEx("git merge-base " &
currentCommit.quoteShell & " " & requiredCommit.quoteShell)
if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit):
# conflict resolution: pick the later commit:
if mergeBase == currentCommit:
checkoutGitCommit(c, w.name, requiredCommit)
else:
checkoutGitCommit(c, w.name, requiredCommit)
when false:
warn c, w.name, "do not know which commit is more recent:",
currentCommit, "(current) or", w.commit, " =", requiredCommit, "(required)"
proc findNimbleFile(c: AtlasContext; dep: Dependency): string =
result = c.workspace / dep.name.string / (dep.name.string & ".nimble")
if not fileExists(result):
result = ""
for x in walkFiles(c.workspace / dep.name.string / "*.nimble"):
if result.len == 0:
result = x
else:
# ambiguous .nimble file
return ""
proc addUniqueDep(c: var AtlasContext; work: var seq[Dependency];
tokens: seq[string]) =
let oldErrors = c.errors
let url = toUrl(c, tokens[0])
if oldErrors != c.errors:
warn c, toName(tokens[0]), "cannot resolve package name"
elif not c.processed.containsOrIncl(url / tokens[2]):
work.add Dependency(name: toName(tokens[0]), url: url, commit: tokens[2],
rel: toDepRelation(tokens[1]))
proc collectNewDeps(c: var AtlasContext; work: var seq[Dependency];
dep: Dependency; result: var seq[string];
isMainProject: bool) =
let nimbleFile = findNimbleFile(c, dep)
if nimbleFile != "":
let nimbleInfo = extractRequiresInfo(nimbleFile)
for r in nimbleInfo.requires:
var tokens: seq[string] = @[]
for token in tokenizeRequires(r):
tokens.add token
if tokens.len == 1:
# nimx uses dependencies like 'requires "sdl2"'.
# Via this hack we map them to the first tagged release.
# (See the `isStrictlySmallerThan` logic.)
tokens.add "<"
tokens.add InvalidCommit
elif tokens.len == 2 and tokens[1].startsWith("#"):
# Dependencies can also look like 'requires "sdl2#head"
var commit = tokens[1]
tokens[1] = "=="
tokens.add commit
if tokens.len >= 3 and cmpIgnoreCase(tokens[0], "nim") != 0:
c.addUniqueDep work, tokens
result.add dep.name.string / nimbleInfo.srcDir
else:
result.add dep.name.string
proc clone(c: var AtlasContext; start: string): seq[string] =
# non-recursive clone.
let oldErrors = c.errors
var work = @[Dependency(name: toName(start), url: toUrl(c, start), commit: "")]
if oldErrors != c.errors:
error c, toName(start), "cannot resolve package name"
return
c.projectDir = work[0].name.string
result = @[]
var i = 0
while i < work.len:
let w = work[i]
let oldErrors = c.errors
if not dirExists(c.workspace / w.name.string):
withDir c.workspace:
let err = cloneUrl(w.url, w.name.string, false)
if err != "":
error c, w.name, err
if oldErrors == c.errors:
if not c.keepCommits: checkoutCommit(c, w)
# even if the checkout fails, we can make use of the somewhat
# outdated .nimble file to clone more of the most likely still relevant
# dependencies:
collectNewDeps(c, work, w, result, i == 0)
inc i
const
configPatternBegin = "############# begin Atlas config section ##########\n"
configPatternEnd = "############# end Atlas config section ##########\n"
proc patchNimCfg(c: AtlasContext; deps: seq[string]) =
var paths = "--noNimblePath\n"
for d in deps:
paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n"
let cfg = c.projectDir / "nim.cfg"
var cfgContent = configPatternBegin & paths & configPatternEnd
if not fileExists(cfg):
writeFile(cfg, cfgContent)
else:
let content = readFile(cfg)
let start = content.find(configPatternBegin)
if start >= 0:
cfgContent = content.substr(0, start-1) & cfgContent
let theEnd = content.find(configPatternEnd, start)
if theEnd >= 0:
cfgContent.add content.substr(theEnd+len(configPatternEnd))
else:
cfgContent = content & "\n" & cfgContent
if cfgContent != content:
# do not touch the file if nothing changed
# (preserves the file date information):
writeFile(cfg, cfgContent)
proc error*(msg: string) =
when defined(debug):
writeStackTrace()
quit "[Error] " & msg
proc main =
var action = ""
var args: seq[string] = @[]
template singleArg() =
if args.len != 1:
error action & " command takes a single package name"
template noArgs() =
if args.len != 0:
error action & " command takes no arguments"
var c = AtlasContext(
projectDir: getCurrentDir(),
workspace: getCurrentDir())
for kind, key, val in getopt():
case kind
of cmdArgument:
if action.len == 0:
action = key.normalize
else:
args.add key
of cmdLongOption, cmdShortOption:
case normalize(key)
of "help", "h": writeHelp()
of "version", "v": writeVersion()
of "keepcommits": c.keepCommits = true
else: writeHelp()
of cmdEnd: assert false, "cannot happen"
while c.workspace.len > 0 and dirExists(c.workspace / ".git"):
c.workspace = c.workspace.parentDir()
case action
of "":
error "No action."
of "clone":
singleArg()
let deps = clone(c, args[0])
patchNimCfg c, deps
if c.errors > 0:
error "There were problems."
of "refresh":
noArgs()
updatePackages(c)
of "search", "list":
updatePackages(c)
search getPackages(c.workspace), args
else:
error "Invalid action: " & action
when isMainModule:
main()
when false:
# some testing code for the `patchNimCfg` logic:
var c = AtlasContext(
projectDir: getCurrentDir(),
workspace: getCurrentDir().parentDir)
patchNimCfg(c, @[PackageName"abc", PackageName"xyz"])
when false:
assert sameVersionAs("v0.2.0", "0.2.0")
assert sameVersionAs("v1", "1")
assert sameVersionAs("1.90", "1.90")
assert sameVersionAs("v1.2.3-zuzu", "1.2.3")
assert sameVersionAs("foo-1.2.3.4", "1.2.3.4")
assert not sameVersionAs("foo-1.2.3.4", "1.2.3")
assert not sameVersionAs("foo", "1.2.3")
assert not sameVersionAs("", "1.2.3")

59
tools/atlas/osutils.nim Normal file
View File

@@ -0,0 +1,59 @@
## OS utilities like 'withDir'.
## (c) 2021 Andreas Rumpf
import os, strutils, osproc
template withDir*(dir, body) =
let oldDir = getCurrentDir()
try:
setCurrentDir(dir)
body
finally:
setCurrentDir(oldDir)
proc isUrl*(x: string): bool =
x.startsWith("git://") or x.startsWith("https://") or x.startsWith("http://")
proc cloneUrl*(url, dest: string; cloneUsingHttps: bool): string =
## Returns an error message on error or else "".
result = ""
var modUrl =
if url.startsWith("git://") and cloneUsingHttps:
"https://" & url[6 .. ^1]
else: url
# github + https + trailing url slash causes a
# checkout/ls-remote to fail with Repository not found
var isGithub = false
if modUrl.contains("github.com") and modUrl.endswith("/"):
modUrl = modUrl[0 .. ^2]
isGithub = true
let (_, exitCode) = execCmdEx("git ls-remote --quiet --tags " & modUrl)
var xcode = exitCode
if isGithub and exitCode != QuitSuccess:
# retry multiple times to avoid annoying github timeouts:
for i in 0..4:
os.sleep(4000)
xcode = execCmdEx("git ls-remote --quiet --tags " & modUrl)[1]
if xcode == QuitSuccess: break
if xcode == QuitSuccess:
# retry multiple times to avoid annoying github timeouts:
let cmd = "git clone " & modUrl & " " & dest
for i in 0..4:
if execShellCmd(cmd) == 0: return ""
os.sleep(4000)
result = "exernal program failed: " & cmd
elif not isGithub:
let (_, exitCode) = execCmdEx("hg identify " & modUrl)
if exitCode == QuitSuccess:
let cmd = "hg clone " & modUrl & " " & dest
for i in 0..4:
if execShellCmd(cmd) == 0: return ""
os.sleep(4000)
result = "exernal program failed: " & cmd
else:
result = "Unable to identify url: " & modUrl
else:
result = "Unable to identify url: " & modUrl

117
tools/atlas/packagesjson.nim Normal file
View File

@@ -0,0 +1,117 @@
import std / [json, os, sets, strutils]
import osutils
type
Package* = ref object
# Required fields in a package.
name*: string
url*: string # Download location.
license*: string
downloadMethod*: string
description*: string
tags*: seq[string] # \
# From here on, optional fields set to the empty string if not available.
version*: string
dvcsTag*: string
web*: string # Info url for humans.
proc optionalField(obj: JsonNode, name: string, default = ""): string =
if hasKey(obj, name) and obj[name].kind == JString:
result = obj[name].str
else:
result = default
proc requiredField(obj: JsonNode, name: string): string =
result = optionalField(obj, name, "")
proc fromJson*(obj: JSonNode): Package =
result = Package()
result.name = obj.requiredField("name")
if result.name.len == 0: return nil
result.version = obj.optionalField("version")
result.url = obj.requiredField("url")
if result.url.len == 0: return nil
result.downloadMethod = obj.requiredField("method")
if result.downloadMethod.len == 0: return nil
result.dvcsTag = obj.optionalField("dvcs-tag")
result.license = obj.optionalField("license")
result.tags = @[]
for t in obj["tags"]:
result.tags.add(t.str)
result.description = obj.requiredField("description")
result.web = obj.optionalField("web")
const PackagesDir* = "packages"
proc getPackages*(workspaceDir: string): seq[Package] =
result = @[]
var uniqueNames = initHashSet[string]()
var jsonFiles = 0
for kind, path in walkDir(workspaceDir / PackagesDir):
if kind == pcFile and path.endsWith(".json"):
inc jsonFiles
let packages = json.parseFile(path)
for p in packages:
let pkg = p.fromJson()
if pkg != nil and not uniqueNames.containsOrIncl(pkg.name):
result.add(pkg)
proc `$`*(pkg: Package): string =
result = pkg.name & ":\n"
result &= " url: " & pkg.url & " (" & pkg.downloadMethod & ")\n"
result &= " tags: " & pkg.tags.join(", ") & "\n"
result &= " description: " & pkg.description & "\n"
result &= " license: " & pkg.license & "\n"
if pkg.web.len > 0:
result &= " website: " & pkg.web & "\n"
proc search*(pkgList: seq[Package]; terms: seq[string]) =
var found = false
template onFound =
echo pkg
echo("")
found = true
break forPackage
for pkg in pkgList:
if terms.len > 0:
block forPackage:
for term in terms:
let word = term.toLower
# Search by name.
if word in pkg.name.toLower:
onFound()
# Search by tag.
for tag in pkg.tags:
if word in tag.toLower:
onFound()
else:
echo(pkg)
echo(" ")
if not found and terms.len > 0:
echo("No package found.")
type PkgCandidates* = array[3, seq[Package]]
proc determineCandidates*(pkgList: seq[Package];
terms: seq[string]): PkgCandidates =
result[0] = @[]
result[1] = @[]
result[2] = @[]
for pkg in pkgList:
block termLoop:
for term in terms:
let word = term.toLower
if word == pkg.name.toLower:
result[0].add pkg
break termLoop
elif word in pkg.name.toLower:
result[1].add pkg
break termLoop
else:
for tag in pkg.tags:
if word in tag.toLower:
result[2].add pkg
break termLoop

101
tools/atlas/parse_requires.nim Normal file
View File

@@ -0,0 +1,101 @@
## Utility API for Nim package managers.
## (c) 2021 Andreas Rumpf
import std / strutils
import ".." / compiler / [ast, idents, msgs, syntaxes, options, pathutils]
type
NimbleFileInfo* = object
requires*: seq[string]
srcDir*: string
tasks*: seq[(string, string)]
proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) =
case n.kind
of nkStmtList, nkStmtListExpr:
for child in n:
extract(child, conf, result)
of nkCallKinds:
if n[0].kind == nkIdent:
case n[0].ident.s
of "requires":
for i in 1..<n.len:
var ch = n[i]
while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0: ch = ch.lastSon
if ch.kind in {nkStrLit..nkTripleStrLit}:
result.requires.add ch.strVal
else:
localError(conf, ch.info, "'requires' takes string literals")
of "task":
if n.len >= 3 and n[1].kind == nkIdent and n[2].kind in {nkStrLit..nkTripleStrLit}:
result.tasks.add((n[1].ident.s, n[2].strVal))
else: discard
of nkAsgn, nkFastAsgn:
if n[0].kind == nkIdent and cmpIgnoreCase(n[0].ident.s, "srcDir") == 0:
if n[1].kind in {nkStrLit..nkTripleStrLit}:
result.srcDir = n[1].strVal
else:
localError(conf, n[1].info, "assignments to 'srcDir' must be string literals")
else:
discard
proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo =
## Extract the `requires` information from a Nimble file. This does **not**
## evaluate the Nimble file. Errors are produced on stderr/stdout and are
## formatted as the Nim compiler does it. The parser uses the Nim compiler
## as an API. The result can be empty, this is not an error, only parsing
## errors are reported.
var conf = newConfigRef()
conf.foreignPackageNotes = {}
conf.notes = {}
conf.mainPackageNotes = {}
let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile)
var parser: Parser
if setupParser(parser, fileIdx, newIdentCache(), conf):
extract(parseAll(parser), conf, result)
closeParser(parser)
const Operators* = {'<', '>', '=', '&', '@', '!', '^'}
proc token(s: string; idx: int; lit: var string): int =
var i = idx
if i >= s.len: return i
while s[i] in Whitespace: inc(i)
case s[i]
of Letters, '#':
lit.add s[i]
inc i
while i < s.len and s[i] notin (Whitespace + {'@', '#'}):
lit.add s[i]
inc i
of '0'..'9':
while i < s.len and s[i] in {'0'..'9', '.'}:
lit.add s[i]
inc i
of '"':
inc i
while i < s.len and s[i] != '"':
lit.add s[i]
inc i
inc i
of Operators:
while i < s.len and s[i] in Operators:
lit.add s[i]
inc i
else:
lit.add s[i]
inc i
result = i
iterator tokenizeRequires*(s: string): string =
var start = 0
var tok = ""
while start < s.len:
tok.setLen 0
start = token(s, start, tok)
yield tok
when isMainModule:
for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"):
echo x