build the documentation of official packages (#20986)

* remove db stuffs

* remove punycode

* remove

* fixes script

* add cloner

* patches

* disable

* patch

* fixes external packages

* disable two packages

* preview documentation build

* try again

* fixes URL

* fixes a bug

* simplify

* fixes documentaion

* fixes

* Apply suggestions from code review

tools/kochdocs.nim

@@ -2,6 +2,9 @@
 
 import std/[os, strutils, osproc, sets, pathnorm, sequtils]
 
+import officialpackages
+export exec
+
 when defined(nimPreviewSlimSystem):
   import std/assertions
 
@@ -48,18 +51,6 @@ proc findNimImpl*(): tuple[path: string, ok: bool] =
 
 proc findNim*(): string = findNimImpl().path
 
-proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") =
-  let prevPath = getEnv("PATH")
-  if additionalPath.len > 0:
-    var absolute = additionalPath
-    if not absolute.isAbsolute:
-      absolute = getCurrentDir() / absolute
-    echo("Adding to $PATH: ", absolute)
-    putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute)
-  echo(cmd)
-  if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
-  putEnv("PATH", prevPath)
-
 template inFold*(desc, body) =
   if existsEnv("GITHUB_ACTIONS"):
     echo "::group::" & desc
@@ -131,11 +122,7 @@ mm.md
 """.splitWhitespace().mapIt("doc" / it)
 
   withoutIndex = """
-lib/wrappers/mysql.nim
-lib/wrappers/sqlite3.nim
-lib/wrappers/postgres.nim
 lib/wrappers/tinyc.nim
-lib/wrappers/odbcsql.nim
 lib/wrappers/pcre.nim
 lib/wrappers/openssl.nim
 lib/posix/posix.nim
@@ -165,6 +152,35 @@ lib/posix/posix_openbsd_amd64.nim
 lib/posix/posix_haiku.nim
 """.splitWhitespace()
 
+  officialPackagesList = """
+pkgs/asyncftpclient/src/asyncftpclient.nim
+pkgs/smtp/src/smtp.nim
+pkgs/punycode/src/punycode.nim
+pkgs/db_connector/src/db_connector/db_common.nim
+pkgs/db_connector/src/db_connector/db_mysql.nim
+pkgs/db_connector/src/db_connector/db_odbc.nim
+pkgs/db_connector/src/db_connector/db_postgres.nim
+pkgs/db_connector/src/db_connector/db_sqlite.nim
+""".splitWhitespace()
+
+  officialPackagesListWithoutIndex = """
+pkgs/db_connector/src/db_connector/mysql.nim
+pkgs/db_connector/src/db_connector/sqlite3.nim
+pkgs/db_connector/src/db_connector/postgres.nim
+pkgs/db_connector/src/db_connector/odbcsql.nim
+pkgs/db_connector/src/db_connector/private/dbutils.nim
+""".splitWhitespace()
+
+proc findName(name: string): string =
+  doAssert name[0..4] == "pkgs/"
+  var i = 5
+  while i < name.len:
+    if name[i] != '/':
+      inc i
+      result.add name[i]
+    else:
+      break
+
 when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo):
   proc isRelativeTo(path, base: string): bool =
     let path = path.normalizedPath
@@ -248,7 +264,8 @@ proc buildDoc(nimArgs, destPath: string) =
   # call nim for the documentation:
   let rst2html = getMd2html()
   var
-    commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len)
+    commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len +
+              officialPackagesList.len + officialPackagesListWithoutIndex.len)
     i = 0
   let nim = findNim().quoteShell()
   for d in items(rst2html):
@@ -269,6 +286,19 @@ proc buildDoc(nimArgs, destPath: string) =
       destPath / changeFileExt(splitFile(d).name, "html"), d]
     i.inc
 
+
+  for d in items(officialPackagesList):
+    var nimArgs2 = nimArgs
+    if d.isRelativeTo("compiler"): doAssert false
+    commands[i] = nim & " doc $# --outdir:$# --index:on $#" %
+      [nimArgs2, destPath, d]
+    i.inc
+  for d in items(officialPackagesListWithoutIndex):
+    commands[i] = nim & " doc $# -o:$# $#" %
+      [nimArgs,
+      destPath / changeFileExt(splitFile(d).name, "html"), d]
+    i.inc
+
   mexec(commands)
   exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath])
     # caveat: this works so long it's called before `buildDocPackages` which
@@ -318,6 +348,7 @@ proc buildJS(): string =
 proc buildDocsDir*(args: string, dir: string) =
   let args = nimArgs & " " & args
   let docHackJsSource = buildJS()
+  gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector"])
   createDir(dir)
   buildDocSamples(args, dir)
   buildDoc(args, dir) # bottleneck

tools/officialpackages.nim

@@ -0,0 +1,21 @@
+import std/[strformat, paths, dirs, envvars]
+from std/os import execShellCmd
+
+proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") =
+  let prevPath = getEnv("PATH")
+  if additionalPath.len > 0:
+    var absolute = Path(additionalPath)
+    if not absolute.isAbsolute:
+      absolute = getCurrentDir() / absolute
+    echo("Adding to $PATH: ", string(absolute))
+    putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & string(absolute))
+  echo(cmd)
+  if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
+  putEnv("PATH", prevPath)
+
+proc gitClonePackages*(names: seq[string]) =
+  if not dirExists(Path"pkgs"):
+    createDir(Path"pkgs")
+  for name in names:
+    if not dirExists(Path"pkgs" / Path(name)):
+      exec fmt"git clone https://github.com/nim-lang/{name} pkgs/{name}"