fixes #19863; move sha1, md5 to nimble packages for 2.0 (#21702)

* move sha1, md5 to nimble packages

* boot the compiler

* fixes tests

* build the documentation

* fixes docs

* lol, I forgot koch.nim

* add `nimHasChecksums` define

* clone checksums but maybe copying is better

* bump nimble hash

* use ChecksumsStableCommit

* fixes tests

* deprecate them

* fixes paths

* fixes koch

changelogs/changelog_2_0_0.md

@@ -126,6 +126,8 @@
   - `std/db_mysql` => `db_connector/db_mysql`
   - `std/db_postgres` => `db_connector/db_postgres`
   - `std/db_odbc` => `db_connector/db_odbc`
+  - `std/md5` => `checksums/md5`
+  - `std/sha1` => `checksums/sha1`
 
 - Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of
   `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`.

compiler/ccgtypes.nim

@@ -12,7 +12,7 @@
 # ------------------------- Name Mangling --------------------------------
 
 import sighashes, modulegraphs
-import std/md5
+import ../dist/checksums/src/checksums/md5
 
 proc isKeyword(w: PIdent): bool =
   # Nim and C++ share some keywords

compiler/condsyms.nim

@@ -154,3 +154,5 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasGenericDefine")
   defineSymbol("nimHasDefineAliases")
   defineSymbol("nimHasWarnBareExcept")
+
+  defineSymbol("nimHasChecksums")

compiler/extccomp.nim

@@ -14,13 +14,15 @@
 
 import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths
 
-import std/[os, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils]
+import std/[os, osproc, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils]
 
 import std / strutils except addf
 
 when defined(nimPreviewSlimSystem):
   import std/syncio
 
+import ../dist/checksums/src/checksums/sha1
+
 type
   TInfoCCProp* = enum         # properties of the C compiler:
     hasSwitchRange,           # CC allows ranges in switch statements (GNU C)

compiler/gorgeimpl.nim

@@ -9,12 +9,14 @@
 
 ## Module that implements ``gorge`` for the compiler.
 
-import msgs, std / sha1, os, osproc, streams, options,
+import msgs, os, osproc, streams, options,
   lineinfos, pathutils
 
 when defined(nimPreviewSlimSystem):
   import std/syncio
 
+import ../dist/checksums/src/checksums/sha1
+
 proc readOutput(p: Process): (string, int) =
   result[0] = ""
   var output = p.outputStream

compiler/ic/ic.nim

@@ -7,13 +7,15 @@
 #    distribution, for details about the copyright.
 #
 
-import hashes, tables, intsets, std/sha1
+import hashes, tables, intsets
 import packed_ast, bitabs, rodfiles
 import ".." / [ast, idents, lineinfos, msgs, ropes, options,
   pathutils, condsyms, packages, modulepaths]
 #import ".." / [renderer, astalgo]
 from os import removeFile, isAbsolute
 
+import ../../dist/checksums/src/checksums/sha1
+
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions, formatfloat]
 

compiler/installer.ini

@@ -78,6 +78,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/checksums"
 
 Files: "tests"
 

compiler/main.nim

@@ -13,7 +13,7 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  std/[strutils, os, times, tables, sha1, with, json],
+  std/[strutils, os, times, tables, with, json],
   llstream, ast, lexer, syntaxes, options, msgs,
   condsyms,
   idents, extccomp,
@@ -29,6 +29,8 @@ when defined(nimPreviewSlimSystem):
 import ic / [cbackend, integrity, navigator]
 from ic / ic import rodViewer
 
+import ../dist/checksums/src/checksums/sha1
+
 import pipelines
 
 when not defined(leanCompiler):

compiler/modulegraphs.nim

@@ -11,7 +11,8 @@
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
 
-import intsets, tables, hashes, md5
+import intsets, tables, hashes
+import ../dist/checksums/src/checksums/md5
 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages
 import ic / [packed_ast, ic]
 

compiler/nimblecmd.nim

@@ -10,11 +10,13 @@
 ## Implements some helper procs for Nimble (Nim's package manager) support.
 
 import parseutils, strutils, os, options, msgs, sequtils, lineinfos, pathutils,
-  std/sha1, tables
+  tables
 
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions]
 
+import ../dist/checksums/src/checksums/sha1
+
 proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   if not conf.searchPaths.contains(path):
     conf.searchPaths.insert(path, 0)

compiler/passes.nim

@@ -24,7 +24,7 @@ import ic/replayer
 export skipCodegen, resolveMod, prepareConfigNotes
 
 when defined(nimsuggest):
-  import std/sha1
+  import ../dist/checksums/src/checksums/sha1
 
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions]

compiler/sighashes.nim

@@ -9,9 +9,10 @@
 
 ## Computes hash values for routine (proc, method etc) signatures.
 
-import ast, tables, ropes, md5, modulegraphs, options, msgs, pathutils
+import ast, tables, ropes, modulegraphs, options, msgs, pathutils
 from hashes import Hash
 import types
+import ../dist/checksums/src/checksums/md5
 
 
 when defined(nimPreviewSlimSystem):

koch.nim

@@ -10,9 +10,10 @@
 #
 
 const
-  NimbleStableCommit = "7efb226ef908297e8791cade20d991784b4e8bfc" # master
+  NimbleStableCommit = "168416290e49023894fc26106799d6f1fc964a2d" # master
   # examples of possible values: #head, #ea82b54, 1.2.3
   FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014"
+  ChecksumsStableCommit = "3fa15df7d27ecef624ed932d60f63d6a8949618d"
   HeadHash = "#head"
 when not defined(windows):
   const
@@ -148,6 +149,8 @@ proc bundleNimbleExe(latest: bool, args: string) =
   let commit = if latest: "HEAD" else: NimbleStableCommit
   cloneDependency(distDir, "https://github.com/nim-lang/nimble.git",
                   commit = commit, allowBundled = true)
+  cloneDependency(distDir / "nimble" / distDir, "https://github.com/nim-lang/checksums.git",
+                commit = ChecksumsStableCommit, allowBundled = true) # or copy it from dist?
   # installer.ini expects it under $nim/bin
   nimCompile("dist/nimble/src/nimble.nim",
              options = "-d:release --mm:refc --noNimblePath " & args)
@@ -181,7 +184,12 @@ proc bundleWinTools(args: string) =
     nimCompile(r"tools\downloader.nim",
                options = r"--cc:vcc --app:gui -d:ssl --noNimblePath --path:..\ui " & args)
 
+proc bundleChecksums(latest: bool) =
+  let commit = if latest: "HEAD" else: ChecksumsStableCommit
+  cloneDependency(distDir, "https://github.com/nim-lang/checksums.git", commit)
+
 proc zip(latest: bool; args: string) =
+  bundleChecksums(latest)
   bundleNimbleExe(latest, args)
   bundleNimsuggest(args)
   bundleNimpretty(args)
@@ -239,6 +247,7 @@ proc testTools(args: string = "") =
       outputName = "atlas")
 
 proc nsis(latest: bool; args: string) =
+  bundleChecksums(latest)
   bundleNimbleExe(latest, args)
   bundleNimsuggest(args)
   bundleWinTools(args)
@@ -301,6 +310,9 @@ proc boot(args: string) =
   let smartNimcache = (if "release" in args or "danger" in args: "nimcache/r_" else: "nimcache/d_") &
                       hostOS & "_" & hostCPU
 
+  if not dirExists("dist/checksums"):
+    bundleChecksums(false)
+
   let nimStart = findStartNim().quoteShell()
   for i in 0..2:
     let defaultCommand = if useCpp: "cpp" else: "c"
@@ -451,6 +463,9 @@ proc temp(args: string) =
       result[1].add " " & quoteShell(args[i])
       inc i
 
+  if not dirExists("dist/checksums"):
+    bundleChecksums(false)
+
   let d = getAppDir()
   let output = d / "compiler" / "nim".exe
   let finalDest = d / "bin" / "nim_temp".exe
@@ -711,6 +726,8 @@ when isMainModule:
       of "tools":
         buildTools(op.cmdLineRest)
         bundleNimbleExe(latest, op.cmdLineRest)
+      of "checksums":
+        bundleChecksums(latest)
       of "pushcsource":
         quit "use this instead: https://github.com/nim-lang/csources_v1/blob/master/push_c_code.nim"
       of "valgrind": valgrind(op.cmdLineRest)

lib/deprecated/pure/securehash.nim

@@ -1,6 +0,0 @@
-## This module is a deprecated alias for the `sha1` module. Deprecated since 0.18.1.
-
-{.deprecated: "use `std/sha1` instead".}
-
-import "../std/sha1"
-export sha1

lib/pure/md5.nim

@@ -18,6 +18,8 @@
 ## * `hashes module<hashes.html>`_ for efficient computations of hash values
 ##   for diverse Nim types
 
+{.deprecated: "use command `nimble install checksums` and import `checksums/md5` instead".}
+
 when defined(nimHasStyleChecks):
   {.push styleChecks: off.}
 
@@ -343,4 +345,4 @@ proc md5Final*(c: var MD5Context, digest: var MD5Digest) =
 
 
 when defined(nimHasStyleChecks):
-  {.pop.} #{.push styleChecks: off.}
+  {.pop.} #{.push styleChecks: off.}

lib/std/sha1.nim

@@ -26,6 +26,9 @@ runnableExamples("-r:off"):
     b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C")
   assert a == b, "files don't match"
 
+
+{.deprecated: "use command `nimble install checksums` and import `checksums/sha1` instead".}
+
 import strutils
 from endians import bigEndian32, bigEndian64
 
@@ -281,4 +284,4 @@ proc `==`*(a, b: SecureHash): bool =
 
 proc isValidSha1Hash*(s: string): bool =
   ## Checks if a string is a valid sha1 hash sum.
-  s.len == 40 and allCharsInSet(s, HexDigits)
+  s.len == 40 and allCharsInSet(s, HexDigits)

nimsuggest/nimsuggest.nim

@@ -11,9 +11,10 @@ import compiler/renderer
 import strformat
 import algorithm
 import tables
-import std/sha1
 import times
 
+import ../dist/checksums/src/checksums/sha1
+
 ## Nimsuggest is a tool that helps to give editors IDE like capabilities.
 
 when not defined(nimcore):

testament/testament.nim

@@ -12,13 +12,15 @@
 import
   strutils, pegs, os, osproc, streams, json, std/exitprocs,
   backend, parseopt, specs, htmlgen, browsers, terminal,
-  algorithm, times, md5, azure, intsets, macros
+  algorithm, times, azure, intsets, macros
 from std/sugar import dup
 import compiler/nodejs
 import lib/stdtest/testutils
 from lib/stdtest/specialpaths import splitTestFile
 from std/private/gitutils import diffStrings
 
+import ../dist/checksums/src/checksums/md5
+
 proc trimUnitSep(x: var string) =
   let L = x.len
   if L > 0 and x[^1] == '\31':

tests/effects/tstrict_funcs_imports.nim

@@ -66,7 +66,6 @@ import
   macros,
   marshal,
   math,
-  md5,
   memfiles,
   mersenne,
   mimetypes,
@@ -155,7 +154,6 @@ import std/[
   monotimes,
   packedsets,
   setutils,
-  sha1,
   socketstreams,
   stackframes,
   sums,

tests/js/tstdlib_imports.nim

@@ -62,7 +62,7 @@ import std/[
   htmlgen,
 
   # Hashing:
-  base64, hashes, md5,
+  base64, hashes,
   # fails due to cstring cast/endians import: oids
   # fails due to copyMem/endians import: sha1
 

tests/manyloc/keineschweine/enet_server/server_utils.nim

@@ -1,4 +1,6 @@
-import enet, sg_packets, estreams, md5, zlib_helpers, client_helpers, strutils,
+import ../../../../dist/checksums/src/checksums/md5
+
+import enet, sg_packets, estreams, zlib_helpers, client_helpers, strutils,
   idgen, sg_assets, tables, os
 type
   PClient* = ref object

tests/manyloc/keineschweine/lib/client_helpers.nim

@@ -1,6 +1,8 @@
+import ../../../../dist/checksums/src/checksums/md5
+
 import
   tables, sg_packets, enet, estreams, sg_gui, sfml,
-  zlib_helpers, md5, sg_assets, os
+  zlib_helpers, sg_assets, os
 type
   PServer* = ptr TServer
   TServer* = object

tests/manyloc/keineschweine/lib/sg_assets.nim

@@ -1,6 +1,8 @@
+import ../../../../dist/checksums/src/checksums/md5
+
 import
   re, json, strutils, tables, math, os, math_helpers,
-  sg_packets, md5, zlib_helpers
+  sg_packets, zlib_helpers
 
 when defined(NoSFML):
   import server_utils

tests/manyloc/keineschweine/lib/sg_packets.nim

@@ -1,4 +1,6 @@
-import genpacket_enet, nativesockets, net, md5, enet
+import ../../../../dist/checksums/src/checksums/md5
+
+import genpacket_enet, nativesockets, net, enet
 defPacketImports()
 
 type

tests/manyloc/keineschweine/server/old_dirserver.nim

@@ -1,9 +1,9 @@
 ## directory server
 ## handles client authorization and assets
-
+import ../../../dist/checksums/src/checksums/md5
 import
   sockets, times, streams, streams_enh, tables, json, os,
-  sg_packets, sg_assets, md5, server_utils, map_filter
+  sg_packets, sg_assets, server_utils, map_filter
 type
   THandler = proc(client: PCLient; stream: PStream)
 var

tests/manyloc/keineschweine/server/old_server_utils.nim

@@ -1,5 +1,7 @@
+import ../../../dist/checksums/src/checksums/md5
+
 import
-  streams, md5, sockets,
+  streams, sockets,
   sg_packets, zlib_helpers, idgen
 type
   TClientType* = enum

tests/manyloc/keineschweine/server/sg_lobby.nim

@@ -1,6 +1,7 @@
+import ../../../dist/checksums/src/checksums/md5
 
 import
-  sockets, streams, tables, times, math, strutils, json, os, md5,
+  sockets, streams, tables, times, math, strutils, json, os,
   sfml, sfml_vector, sfml_colors,
   streams_enh, input_helpers, zlib_helpers, client_helpers, sg_packets, sg_assets, sg_gui
 type

tests/stdlib/tmd5.nim

@@ -1,18 +0,0 @@
-discard """
-  matrix: "--mm:refc; --mm:orc"
-  targets: "c cpp js"
-"""
-
-import md5
-import std/assertions
-
-proc main() {.raises: [].} =
-  doAssert(getMD5("Franz jagt im komplett verwahrlosten Taxi quer durch Bayern") ==
-    "a3cca2b2aa1e3b5b3b5aad99a8529074")
-  doAssert(getMD5("Frank jagt im komplett verwahrlosten Taxi quer durch Bayern") ==
-    "7e716d0e702df0505fc72e2b89467910")
-  doAssert($toMD5("") == "d41d8cd98f00b204e9800998ecf8427e")
-
-main()
-
-static: main()

tests/stdlib/tsha1.nim

@@ -1,28 +0,0 @@
-discard """
-  matrix: "--mm:refc; --mm:orc"
-"""
-
-import std/sha1
-import std/assertions
-
-let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]")
-doAssert hash1 == hash1
-doAssert parseSecureHash($hash1) == hash1
-
-template checkVector(s, exp: string) =
-  doAssert secureHash(s) == parseSecureHash(exp)
-
-checkVector("", "da39a3ee5e6b4b0d3255bfef95601890afd80709")
-checkVector("abc", "a9993e364706816aba3e25717850c26c9cd0d89d")
-checkVector("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
-            "84983e441c3bd26ebaae4aa1f95129e5e54670f1")
-
-proc testIsValidSha1Hash =
-  doAssert not isValidSha1Hash("")
-  doAssert not isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD11")
-  doAssert not isValidSha1Hash("042G4BE2B90ED0672E717D71850ABDB0A2D19CD1")
-  doAssert isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD1")
-  doAssert isValidSha1Hash("042d4be2b90ed0672e717d71850abdb0a2d19cd1")
-  doAssert isValidSha1Hash("042d4be2b90ed0672e717D71850ABDB0A2D19CD1")
-
-testIsValidSha1Hash()

tests/test_nimscript.nims

@@ -64,7 +64,7 @@ import std/[
   htmlgen,
 
   # Hashing:
-  base64, hashes, md5,
+  base64, hashes,
   # fails due to cstring cast/times import/endians import: oids
   # fails due to copyMem/endians import: sha1
 

tests/vm/tsignaturehash.nim

@@ -1,7 +1,7 @@
 # test sym digest is computable at compile time
 
 import macros, algorithm
-import md5
+import ../../dist/checksums/src/checksums/md5
 
 macro testmacro(s: typed{nkSym}): string =
   let s = getMD5(signaturehash(s) & " - " & symBodyHash(s))

tools/kochdocs.nim

@@ -171,16 +171,6 @@ pkgs/db_connector/src/db_connector/odbcsql.nim
 pkgs/db_connector/src/db_connector/private/dbutils.nim
 """.splitWhitespace()
 
-proc findName(name: string): string =
-  doAssert name[0..4] == "pkgs/"
-  var i = 5
-  while i < name.len:
-    if name[i] != '/':
-      inc i
-      result.add name[i]
-    else:
-      break
-
 when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo):
   proc isRelativeTo(path, base: string): bool =
     let path = path.normalizedPath

tools/niminst/niminst.nim

@@ -8,9 +8,9 @@
 #
 
 import
-  os, strutils, parseopt, parsecfg, strtabs, streams, debcreation,
-  std / sha1
+  os, strutils, parseopt, parsecfg, strtabs, streams, debcreation
 
+import ../../dist/checksums/src/checksums/sha1
 
 when defined(nimPreviewSlimSystem):
   import std/syncio