niminst: faster way to deduplicate files

tools/niminst/niminst.nim

@@ -14,7 +14,8 @@ when haveZipLib:
   import zipfiles
 
 import
-  os, osproc, strutils, parseopt, parsecfg, strtabs, streams, debcreation
+  os, osproc, strutils, parseopt, parsecfg, strtabs, streams, debcreation,
+  securehash
 
 const
   maxOS = 20 # max number of OSes
@@ -476,23 +477,23 @@ proc writeFile(filename, content, newline: string) =
   else:
     quit("Cannot open for writing: " & filename)
 
-proc removeDuplicateFiles(c: var ConfigData) =
-  for osA in countdown(c.oses.len, 1):
-    for cpuA in countdown(c.cpus.len, 1):
+proc deduplicateFiles(c: var ConfigData) =
+  var tab = newStringTable()
+  let build = getOutputDir(c)
+  for osA in countup(1, c.oses.len):
+    for cpuA in countup(1, c.cpus.len):
       if c.cfiles[osA][cpuA].isNil: c.cfiles[osA][cpuA] = @[]
       if c.explicitPlatforms and not c.platforms[osA][cpuA]: continue
-      for i in 0..c.cfiles[osA][cpuA].len-1:
-        var dup = c.cfiles[osA][cpuA][i]
-        var f = extractFilename(dup)
-        for osB in 1..c.oses.len:
-          for cpuB in 1..c.cpus.len:
-            if osB != osA or cpuB != cpuA:
-              var orig = buildDir(osB, cpuB) / f
-              if existsFile(orig) and existsFile(dup) and
-                  sameFileContent(orig, dup):
-                # file is identical, so delete duplicate:
-                removeFile(dup)
-                c.cfiles[osA][cpuA][i] = orig
+      for dup in mitems(c.cfiles[osA][cpuA]):
+        let key = $secureHashFile(build / dup)
+        let val = build / buildDir(osA, cpuA) / extractFilename(dup)
+        let orig = tab.getOrDefault(key)
+        if orig.len > 0:
+          # file is identical, so delete duplicate:
+          removeFile(dup)
+          dup = orig
+        else:
+          tab[key] = val
 
 proc writeInstallScripts(c: var ConfigData) =
   if c.installScript:
@@ -536,7 +537,7 @@ proc srcdist(c: var ConfigData) =
         copyFile(dest=dest, source=c.cfiles[osA][cpuA][i])
         c.cfiles[osA][cpuA][i] = relDest
   # second pass: remove duplicate files
-  removeDuplicateFiles(c)
+  deduplicateFiles(c)
   writeFile(getOutputDir(c) / buildShFile, generateBuildShellScript(c), "\10")
   inclFilePermissions(getOutputDir(c) / buildShFile, {fpUserExec, fpGroupExec, fpOthersExec})
   writeFile(getOutputDir(c) / makeFile, generateMakefile(c), "\10")