From ba2aa474a1e96acb06190236fbc2896d2e170494 Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Tue, 26 May 2015 12:51:31 +0300 Subject: [PATCH 1/4] Implement support for XZ dist tarball Motivation ---------- Currenly tarballs for linux created manually. ZIP files are not suitable, because they do not preserve unix file permissions. Modification ------------ Implement 'koch xz' command to produce dist tarball in xz format. Also the same command implemented in niminst.nim. Result ------ Now it is easy to create source distribution for git stapshot and use it further in packaging scripts. --- koch.nim | 8 ++++++++ tools/niminst/niminst.nim | 42 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/koch.nim b/koch.nim index a951b422fd..c6db6893c3 100644 --- a/koch.nim +++ b/koch.nim @@ -47,6 +47,7 @@ Possible Commands: csource [options] builds the C sources for installation pdf builds the PDF documentation zip builds the installation ZIP package + xz builds the installation XZ package nsis [options] builds the NSIS Setup installer (for Windows) tests [options] run the testsuite update updates nim to the latest version from github @@ -112,6 +113,12 @@ proc targz(args: string) = exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim targz compiler/installer.ini" % ["tools" / "niminst" / "niminst".exe, VersionAsString]) +proc xz(args: string) = + exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % + [VersionAsString, compileNimInst, findNim()]) + exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim xz compiler/installer.ini" % + ["tools" / "niminst" / "niminst".exe, VersionAsString]) + proc buildTool(toolname, args: string) = exec("$# cc $# $#" % [findNim(), args, toolname]) copyFile(dest="bin"/ splitFile(toolname).name.exe, source=toolname.exe) @@ -368,6 +375,7 @@ of cmdArgument: of "csource", "csources": csource(op.cmdLineRest) of "zip": zip(op.cmdLineRest) of "targz": targz(op.cmdLineRest) + of "xz": xz(op.cmdLineRest) of "nsis": nsis(op.cmdLineRest) of "install": install(op.cmdLineRest) of "test", "tests": tests(op.cmdLineRest) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index f0ae45484f..aed4624f52 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -36,6 +36,7 @@ type actionScripts # action: create install and deinstall scripts actionZip, # action: create zip file actionTargz, # action: create targz file + actionXz, # action: create xz file actionDeb # action: prepare deb package FileCategory = enum @@ -173,6 +174,7 @@ proc parseCmdLine(c: var ConfigData) = of "scripts": incl(c.actions, actionScripts) of "zip": incl(c.actions, actionZip) of "targz": incl(c.actions, actionTargz) + of "xz": incl(c.actions, actionXz) of "inno": incl(c.actions, actionInno) of "nsis": incl(c.actions, actionNsis) of "deb": incl(c.actions, actionDeb) @@ -600,6 +602,44 @@ proc targzDist(c: var ConfigData) = finally: setCurrentDir(oldDir) +proc xzDist(c: var ConfigData) = + let proj = toLower(c.name) & "-" & c.version + var n = "$#.tar.xz" % proj + let tmpDir = if c.outdir.len == 0: "build" else: c.outdir + + template processFile(z, dest, src) = + let s = src + let d = dest + echo "Copying ", s, " to ", tmpDir / d + let destdir = tmpdir / d.splitFile.dir + if not dirExists(destdir): createDir(destdir) + copyFile(s, tmpDir / d) + + processFile(z, proj / buildBatFile32, "build" / buildBatFile32) + processFile(z, proj / buildBatFile64, "build" / buildBatFile64) + processFile(z, proj / buildShFile, "build" / buildShFile) + processFile(z, proj / makeFile, "build" / makeFile) + processFile(z, proj / installShFile, installShFile) + processFile(z, proj / deinstallShFile, deinstallShFile) + for f in walkFiles(c.libpath / "lib/*.h"): + processFile(z, proj / "c_code" / extractFilename(f), f) + for osA in 1..c.oses.len: + for cpuA in 1..c.cpus.len: + var dir = buildDir(osA, cpuA) + for k, f in walkDir("build" / dir): + if k == pcFile: processFile(z, proj / dir / extractFilename(f), f) + + for cat in items({fcConfig..fcOther, fcUnix}): + for f in items(c.cat[cat]): processFile(z, proj / f, f) + + let oldDir = getCurrentDir() + setCurrentDir(tmpDir) + try: + if execShellCmd("tar Jcf $1.tar.xz $1" % proj) != 0: + echo("External program failed") + finally: + setCurrentDir(oldDir) + # -- prepare build files for .deb creation proc debDist(c: var ConfigData) = @@ -668,5 +708,7 @@ if actionZip in c.actions: quit("libzip is not installed") if actionTargz in c.actions: targzDist(c) +if actionXz in c.actions: + xzDist(c) if actionDeb in c.actions: debDist(c) From 4caf5482d43fb8230779363c2e7405d25d428b85 Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Tue, 26 May 2015 13:27:02 +0300 Subject: [PATCH 2/4] Remove outdated targz from niminst --- koch.nim | 7 ------- tools/niminst/niminst.nim | 44 --------------------------------------- 2 files changed, 51 deletions(-) diff --git a/koch.nim b/koch.nim index c6db6893c3..fc292401a0 100644 --- a/koch.nim +++ b/koch.nim @@ -107,12 +107,6 @@ proc zip(args: string) = exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" % ["tools/niminst/niminst".exe, VersionAsString]) -proc targz(args: string) = - exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % - [VersionAsString, compileNimInst, findNim()]) - exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim targz compiler/installer.ini" % - ["tools" / "niminst" / "niminst".exe, VersionAsString]) - proc xz(args: string) = exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % [VersionAsString, compileNimInst, findNim()]) @@ -374,7 +368,6 @@ of cmdArgument: of "pdf": pdf() of "csource", "csources": csource(op.cmdLineRest) of "zip": zip(op.cmdLineRest) - of "targz": targz(op.cmdLineRest) of "xz": xz(op.cmdLineRest) of "nsis": nsis(op.cmdLineRest) of "install": install(op.cmdLineRest) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index aed4624f52..0afef125b1 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -35,7 +35,6 @@ type actionNsis, # action: create NSIS installer actionScripts # action: create install and deinstall scripts actionZip, # action: create zip file - actionTargz, # action: create targz file actionXz, # action: create xz file actionDeb # action: prepare deb package @@ -173,7 +172,6 @@ proc parseCmdLine(c: var ConfigData) = of "csource": incl(c.actions, actionCSource) of "scripts": incl(c.actions, actionScripts) of "zip": incl(c.actions, actionZip) - of "targz": incl(c.actions, actionTargz) of "xz": incl(c.actions, actionXz) of "inno": incl(c.actions, actionInno) of "nsis": incl(c.actions, actionNsis) @@ -562,46 +560,6 @@ when haveZipLib: else: quit("Cannot open for writing: " & n) -proc targzDist(c: var ConfigData) = - let proj = toLower(c.name) & "-" & c.version - var n = "$#.tar.gz" % proj - let tmpDir = if c.outdir.len == 0: "build" else: c.outdir - - template processFile(z, dest, src) = - let s = src - let d = dest - echo "Copying ", s, " to ", tmpDir / d - let destdir = tmpdir / d.splitFile.dir - if not dirExists(destdir): createDir(destdir) - copyFile(s, tmpDir / d) - - processFile(z, proj / buildBatFile32, "build" / buildBatFile32) - processFile(z, proj / buildBatFile64, "build" / buildBatFile64) - processFile(z, proj / buildShFile, "build" / buildShFile) - processFile(z, proj / makeFile, "build" / makeFile) - processFile(z, proj / installShFile, installShFile) - processFile(z, proj / deinstallShFile, deinstallShFile) - for f in walkFiles(c.libpath / "lib/*.h"): - processFile(z, proj / "c_code" / extractFilename(f), f) - for osA in 1..c.oses.len: - for cpuA in 1..c.cpus.len: - var dir = buildDir(osA, cpuA) - for k, f in walkDir("build" / dir): - if k == pcFile: processFile(z, proj / dir / extractFilename(f), f) - - for cat in items({fcConfig..fcOther, fcUnix}): - for f in items(c.cat[cat]): processFile(z, proj / f, f) - - let oldDir = getCurrentDir() - setCurrentDir(tmpDir) - try: - #if execShellCmd("7z a -ttar $1.tar $1" % proj) != 0 or - # execShellCmd("7z a -tgzip $1.tar.gz $1.tar" % proj) != 0 or - if execShellCmd("7z a -tzip $1.zip $1" % proj) != 0: - echo("External program failed") - finally: - setCurrentDir(oldDir) - proc xzDist(c: var ConfigData) = let proj = toLower(c.name) & "-" & c.version var n = "$#.tar.xz" % proj @@ -706,8 +664,6 @@ if actionZip in c.actions: zipDist(c) else: quit("libzip is not installed") -if actionTargz in c.actions: - targzDist(c) if actionXz in c.actions: xzDist(c) if actionDeb in c.actions: From 0fd0ace00c5b6cf347cb82b460973c5d6490d45a Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Tue, 26 May 2015 13:43:21 +0300 Subject: [PATCH 3/4] Set file permissions when prepare XZ tarball --- tools/niminst/niminst.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index 0afef125b1..6123259f50 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -571,7 +571,7 @@ proc xzDist(c: var ConfigData) = echo "Copying ", s, " to ", tmpDir / d let destdir = tmpdir / d.splitFile.dir if not dirExists(destdir): createDir(destdir) - copyFile(s, tmpDir / d) + copyFileWithPermissions(s, tmpDir / d) processFile(z, proj / buildBatFile32, "build" / buildBatFile32) processFile(z, proj / buildBatFile64, "build" / buildBatFile64) From a333cf939fae154ebd562494d5854e0b60ed11d3 Mon Sep 17 00:00:00 2001 From: Sergey Avseyev Date: Tue, 26 May 2015 13:51:22 +0300 Subject: [PATCH 4/4] Use maximum compression level for XZ tarball --- tools/niminst/niminst.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index 6123259f50..99befa92d6 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -593,7 +593,7 @@ proc xzDist(c: var ConfigData) = let oldDir = getCurrentDir() setCurrentDir(tmpDir) try: - if execShellCmd("tar Jcf $1.tar.xz $1" % proj) != 0: + if execShellCmd("XZ_OPT=-9 tar Jcf $1.tar.xz $1" % proj) != 0: echo("External program failed") finally: setCurrentDir(oldDir)