From 98073624f71e768250907d71f9b5752c0aa56be5 Mon Sep 17 00:00:00 2001 From: narimiran Date: Thu, 22 Oct 2020 14:48:03 +0200 Subject: [PATCH] update koch so the nightlies will be able to bundle Nimble - Previously, `winrelease` called `bundleNimbleExe`, which tries to clone nimble to `dist/nimble` unconditionally. - Since nightlies source tarball bundles nimble, git couldn't clone because `dist/nimble` was already there. This leads to 1.0.x couldn't be built with nimble bundled. - The fix backports the logic from Nim >= 1.2.8 which can check if `dist/nimble` was bundled by the source archive. With this fix we can re-enable bundling of nimble to the source archive for 1.0.x. --- koch.nim | 56 ++++++++++++++------------------------------------ tools/deps.nim | 43 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 41 deletions(-) create mode 100644 tools/deps.nim diff --git a/koch.nim b/koch.nim index 60434cec84..29fe60ee31 100644 --- a/koch.nim +++ b/koch.nim @@ -27,6 +27,7 @@ import os, strutils, parseopt, osproc, streams import tools / kochdocs +import tools / deps const VersionAsString = system.NimVersion @@ -115,6 +116,7 @@ proc copyExe(source, dest: string) = const compileNimInst = "tools/niminst/niminst" + distDir = "dist" proc csource(args: string) = nimexec(("cc $1 -r $3 --var:version=$2 --var:mingw=none csource " & @@ -122,45 +124,17 @@ proc csource(args: string) = [args, VersionAsString, compileNimInst]) proc bundleC2nim(args: string) = - if not dirExists("dist/c2nim/.git"): - exec("git clone https://github.com/nim-lang/c2nim.git dist/c2nim") + cloneDependency(distDir, "https://github.com/nim-lang/c2nim.git") nimCompile("dist/c2nim/c2nim", options = "--noNimblePath --path:. " & args) proc bundleNimbleExe(latest: bool, args: string) = - if not dirExists("dist/nimble/.git"): - exec("git clone https://github.com/nim-lang/nimble.git dist/nimble") - if not latest: - withDir("dist/nimble"): - exec("git fetch") - exec("git checkout " & NimbleStableCommit) + let commit = if latest: "HEAD" else: NimbleStableCommit + cloneDependency(distDir, "https://github.com/nim-lang/nimble.git", + commit = commit, allowBundled = true) # installer.ini expects it under $nim/bin nimCompile("dist/nimble/src/nimble.nim", - options = "-d:release --nilseqs:on " & args) - -proc buildNimble(latest: bool, args: string) = - # if koch is used for a tar.xz, build the dist/nimble we shipped - # with the tarball: - var installDir = "dist/nimble" - if not latest and dirExists(installDir) and not dirExists("dist/nimble/.git"): - discard "don't do the git dance" - else: - if not dirExists("dist/nimble/.git"): - if dirExists(installDir): - var id = 0 - while dirExists("dist/nimble" & $id): - inc id - installDir = "dist/nimble" & $id - exec("git clone https://github.com/nim-lang/nimble.git " & installDir) - withDir(installDir): - if latest: - exec("git checkout -f master") - exec("git pull") - else: - exec("git fetch") - exec("git checkout " & NimbleStableCommit) - nimCompile(installDir / "src/nimble.nim", - options = "--noNimblePath --nilseqs:on -d:release " & args) + options = "-d:release --noNimblePath --nilseqs:on " & args) proc bundleNimsuggest(args: string) = nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim", @@ -318,10 +292,10 @@ proc boot(args: string) = # in order to use less memory, we split the build into two steps: # --compileOnly produces a $project.json file and does not run GCC/Clang. # jsonbuild then uses the $project.json file to build the Nim binary. - exec "$# $# $# $# --nimcache:$# --compileOnly compiler" / "nim.nim" % - [nimi, bootOptions, extraOption, args, smartNimcache] - exec "$# jsonscript $# --nimcache:$# compiler" / "nim.nim" % - [nimi, args, smartNimcache] + exec "$# $# $# --nimcache:$# $# --compileOnly compiler" / "nim.nim" % + [nimi, bootOptions, extraOption, smartNimcache, args] + exec "$# jsonscript --nimcache:$# $# compiler" / "nim.nim" % + [nimi, smartNimcache, args] if sameFileContent(output, i.thVersion): copyExe(output, finalDest) @@ -479,7 +453,7 @@ proc runCI(cmd: string) = when defined(posix): # appveyor (on windows) didn't run this kochExecFold("Boot", "boot") # boot without -d:nimHasLibFFI to make sure this still works - kochExecFold("Boot in release mode", "boot -d:release -d:danger") + kochExecFold("Boot in release mode", "boot -d:release") ## build nimble early on to enable remainder to depend on it if needed kochExecFold("Build Nimble", "nimble") @@ -491,7 +465,7 @@ proc runCI(cmd: string) = if getEnv("NIM_TEST_PACKAGES", "false") == "true": execFold("Test selected Nimble packages", "nim c -r testament/testament cat nimble-packages") else: - buildTools() # altenatively, kochExec "tools --toolsNoNimble" + buildTools() ## run tests execFold("Test nimscript", "nim e tests/test_nimscript.nims") @@ -628,13 +602,13 @@ when isMainModule: of "temp": temp(op.cmdLineRest) of "xtemp": xtemp(op.cmdLineRest) of "wintools": bundleWinTools(op.cmdLineRest) - of "nimble": buildNimble(latest, op.cmdLineRest) + of "nimble": bundleNimbleExe(latest, op.cmdLineRest) of "nimsuggest": bundleNimsuggest(op.cmdLineRest) of "toolsnonimble": buildTools(op.cmdLineRest) of "tools": buildTools(op.cmdLineRest) - buildNimble(latest, op.cmdLineRest) + bundleNimbleExe(latest, op.cmdLineRest) of "pushcsource", "pushcsources": pushCsources() of "valgrind": valgrind(op.cmdLineRest) of "c2nim": bundleC2nim(op.cmdLineRest) diff --git a/tools/deps.nim b/tools/deps.nim new file mode 100644 index 0000000000..b238f12ed4 --- /dev/null +++ b/tools/deps.nim @@ -0,0 +1,43 @@ +import os, uri, strformat, osproc, strutils + +proc exec(cmd: string) = + echo "deps.cmd: " & cmd + let status = execShellCmd(cmd) + doAssert status == 0, cmd + +proc execEx(cmd: string): tuple[output: TaintedString, exitCode: int] = + echo "deps.cmd: " & cmd + execCmdEx(cmd, {poStdErrToStdOut, poUsePath, poEvalCommand}) + +proc isGitRepo(dir: string): bool = + # This command is used to get the relative path to the root of the repository. + # Using this, we can verify whether a folder is a git repository by checking + # whether the command success and if the output is empty. + let (output, status) = execEx fmt"git -C {quoteShell(dir)} rev-parse --show-cdup" + # On Windows there will be a trailing newline on success, remove it. + # The value of a successful call typically won't have a whitespace (it's + # usually a series of ../), so we know that it's safe to unconditionally + # remove trailing whitespaces from the result. + result = status == 0 and output.strip() == "" + +const commitHead* = "HEAD" + +proc cloneDependency*(destDirBase: string, url: string, commit = commitHead, + appendRepoName = true, allowBundled = false) = + let destDirBase = destDirBase.absolutePath + let p = url.parseUri.path + let name = p.splitFile.name + var destDir = destDirBase + if appendRepoName: destDir = destDir / name + let destDir2 = destDir.quoteShell + if not dirExists(destDir): + # note: old code used `destDir / .git` but that wouldn't prevent git clone + # from failing + exec fmt"git clone -q {url} {destDir2}" + if isGitRepo(destDir): + exec fmt"git -C {destDir2} fetch -q" + exec fmt"git -C {destDir2} checkout -q {commit}" + elif allowBundled: + discard "this dependency was bundled with Nim, don't do anything" + else: + quit "FAILURE: " & destdir & " already exists but is not a git repo"