compute checksum of nim files early in the pipelines (#23268)

related https://github.com/nim-lang/Nim/issues/21717 configs will be
resolved later

(cherry picked from commit 519d976f62)

compiler/extccomp.nim

@@ -19,7 +19,7 @@ import std/[os, osproc, streams, sequtils, times, strtabs, json, jsonutils, suga
 import std / strutils except addf
 
 when defined(nimPreviewSlimSystem):
-  import std/syncio
+  import std/[syncio, assertions]
 
 import ../dist/checksums/src/checksums/sha1
 
@@ -992,7 +992,7 @@ type BuildCache = object
   depfiles: seq[(string, string)]
   nimexe: string
 
-proc writeJsonBuildInstructions*(conf: ConfigRef) =
+proc writeJsonBuildInstructions*(conf: ConfigRef; deps: StringTableRef) =
   var linkFiles = collect(for it in conf.externalToLink:
     var it = it
     if conf.noAbsolutePaths: it = it.extractFilename
@@ -1013,10 +1013,14 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
     currentDir: getCurrentDir())
   if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
     bcache.cmdline = conf.commandLine
-    bcache.depfiles = collect(for it in conf.m.fileInfos:
+    for it in conf.m.fileInfos:
       let path = it.fullPath.string
       if isAbsolute(path): # TODO: else?
-        (path, $secureHashFile(path)))
+        if path in deps:
+          bcache.depfiles.add (path, deps[path])
+        else: # backup for configs etc.
+          bcache.depfiles.add (path, $secureHashFile(path))
+
     bcache.nimexe = hashNimExe()
   conf.jsonBuildFile = conf.jsonBuildInstructionsFile
   conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty)

compiler/main.nim

@@ -151,7 +151,7 @@ proc commandCompileToC(graph: ModuleGraph) =
     extccomp.callCCompiler(conf)
     # for now we do not support writing out a .json file with the build instructions when HCR is on
     if not conf.hcrOn:
-      extccomp.writeJsonBuildInstructions(conf)
+      extccomp.writeJsonBuildInstructions(conf, graph.cachedFiles)
     if optGenScript in graph.config.globalOptions:
       writeDepsFile(graph)
     if optGenCDeps in graph.config.globalOptions:

compiler/modulegraphs.nim

@@ -11,7 +11,7 @@
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
 
-import intsets, tables, hashes
+import std/[intsets, tables, hashes, strtabs]
 import ../dist/checksums/src/checksums/md5
 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages
 import ic / [packed_ast, ic]
@@ -129,6 +129,8 @@ type
     idgen*: IdGenerator
     operators*: Operators
 
+    cachedFiles*: StringTableRef
+
   TPassContext* = object of RootObj # the pass's context
     idgen*: IdGenerator
   PPassContext* = ref TPassContext
@@ -480,6 +482,7 @@ proc initModuleGraphFields(result: ModuleGraph) =
   result.symBodyHashes = initTable[int, SigHash]()
   result.operators = initOperators(result)
   result.emittedTypeInfo = initTable[string, FileIndex]()
+  result.cachedFiles = newStringTable()
 
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()

compiler/modules.nim

@@ -14,6 +14,9 @@ import
   idents, lexer, syntaxes, modulegraphs,
   lineinfos, pathutils
 
+import ../dist/checksums/src/checksums/sha1
+import std/strtabs
+
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
 
@@ -42,6 +45,8 @@ proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
   result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
   graph.addDep(s, fileIdx)
   graph.addIncludeDep(s.position.FileIndex, fileIdx)
+  let path = toFullPath(graph.config, fileIdx)
+  graph.cachedFiles[path] = $secureHashFile(path)
 
 proc wantMainModule*(conf: ConfigRef) =
   if conf.projectFull.isEmpty:

compiler/pipelines.nim

@@ -5,10 +5,12 @@ import sem, cgen, modulegraphs, ast, llstream, parser, msgs,
 
 import pipelineutils
 
+import ../dist/checksums/src/checksums/sha1
+
 when not defined(leanCompiler):
   import jsgen, docgen2
 
-import std/[syncio, objectdollar, assertions, tables, strutils]
+import std/[syncio, objectdollar, assertions, tables, strutils, strtabs]
 import renderer
 import ic/replayer
 
@@ -225,7 +227,10 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
   if result == nil:
     var cachedModules: seq[FileIndex]
     result = moduleFromRodFile(graph, fileIdx, cachedModules)
-    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+    let path = toFullPath(graph.config, fileIdx)
+    let filename = AbsoluteFile path
+    if fileExists(filename): # it could be a stdinfile
+      graph.cachedFiles[path] = $secureHashFile(path)
     if result == nil:
       result = newModule(graph, fileIdx)
       result.flags.incl flags