reuse config/build_config.txt for all bootstrap scripts (posix + windows + ci); use build_all.bat in 1 CI, fix bug in build_all.bat (#17899)

* reuse config/build_config.txt for all bootstrap scripts (posix + windows + ci)
* ci_docs: use build_all.bat in CI (just in that pipeline) to ensure it keeps working
* fixup
* fix pre-existing bug in build_all.bat
* fixup
* cp => copy /y
* auto-generate build_all.bat, build_all.sh
* fixup

.github/workflows/ci_docs.yml

@@ -70,13 +70,21 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Build csourcesAny'
+      - name: 'Build csourcesAny (posix)'
+        # this would work on windows and other CI use this on windows,
+        # but we ensure here that `ci/build_autogen.bat` keeps working on windows.
+        if: runner.os != 'Windows'
         shell: bash
         run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc
           # was previously using caching via `actions/cache@v1` but this wasn't
           # used in other CI pipelines and it's unclear the added complexity
           # was worth the saving; can be revisited if needed.
 
+      - name: 'Build csourcesAny (windows)'
+        if: runner.os == 'Windows'
+        shell: cmd
+        run: ci/build_autogen.bat
+
       - name: 'Build koch'
         shell: bash
         run: nim c koch

build_all.bat

@@ -1,20 +1,29 @@
 @echo off
-rem build development version of the compiler; can be rerun safely
-rem TODO: call nimBuildCsourcesIfNeeded or auto-generate this file (from a nim script)
-rem to avoid duplication.
-if not exist csources_v1 (
-  git clone --depth 1 https://github.com/nim-lang/csources_v1.git
+rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
+rem Build development version of the compiler; can be rerun safely
+rem bare bones version of ci/funs.sh adapted for windows.
+
+rem Read in some common shared variables (shared with other tools),
+rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
+for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
+SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
+echo "building from csources: %nim_csources%"
+
+if not exist %nim_csourcesDir% (
+  git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir%
 )
-if not exist bin\nim.exe (
-  cd csources_v1
-  git checkout a8a5241f9475099c823cfe1a5e0ca4022ac201ff
-  if PROCESSOR_ARCHITECTURE == AMD64 (
+
+if not exist %nim_csources% (
+  cd %nim_csourcesDir%
+  git checkout %nim_csourcesHash%
+  echo "%PROCESSOR_ARCHITECTURE%"
+  if "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
     SET ARCH=64
   )
   CALL build.bat
   cd ..
+  copy /y bin\nim.exe  %nim_csources%
 )
-bin\nim.exe c --skipUserCfg --skipParentCfg koch
-koch.exe boot -d:release --skipUserCfg --skipParentCfg
-koch.exe tools --skipUserCfg --skipParentCfg
-
+ bin\nim.exe c --skipUserCfg --skipParentCfg --hints:off koch
+ koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
+ koch tools --skipUserCfg --skipParentCfg --hints:off

build_all.sh

@@ -1,4 +1,5 @@
 #! /bin/sh
+rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
 
 # build development version of the compiler; can be rerun safely.
 # arguments can be passed, e.g.:
@@ -10,7 +11,7 @@ set -e # exit on first error
 . ci/funs.sh
 nimBuildCsourcesIfNeeded "$@"
 
-# Note: if fails, may need to update csourcesAny manually
-echo_run bin/nim c --skipUserCfg --skipParentCfg koch
-echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
-echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
+echo_run bin/nim c --skipUserCfg --skipParentCfg --hints:off koch
+echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
+echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off
+

ci/build_autogen.bat

@@ -0,0 +1,26 @@
+@echo off
+rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
+rem Build development version of the compiler; can be rerun safely
+rem bare bones version of ci/funs.sh adapted for windows.
+
+rem Read in some common shared variables (shared with other tools),
+rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
+for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
+SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
+echo "building from csources: %nim_csources%"
+
+if not exist %nim_csourcesDir% (
+  git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir%
+)
+
+if not exist %nim_csources% (
+  cd %nim_csourcesDir%
+  git checkout %nim_csourcesHash%
+  echo "%PROCESSOR_ARCHITECTURE%"
+  if "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
+    SET ARCH=64
+  )
+  CALL build.bat
+  cd ..
+  copy /y bin\nim.exe  %nim_csources%
+)

ci/funs.sh

@@ -32,9 +32,7 @@ nimIsCiSkip(){
 }
 
 nimDefineVars(){
-  nim_csourcesDir=csources_v1 # where we clone
-  nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git
-  nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff
+  . config/build_config.txt
   nim_csources=bin/nim_csources_$nim_csourcesHash
 }
 
@@ -88,9 +86,11 @@ nimBuildCsourcesIfNeeded(){
       if test -d "$nim_csourcesDir"; then
         echo "$nim_csourcesDir exists."
       else
-        # depth 1: adjust as needed in case useful for `git bisect`
+        # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g.
+        # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`.
         echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir"
         echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash
+        # if needed we could also add: `git reset --hard $nim_csourcesHash`
       fi
       _nimBuildCsourcesIfNeeded "$@"
     fi

config/build_config.txt

@@ -0,0 +1,4 @@
+nim_comment="key-value pairs for windows/posix bootstrapping build scripts"
+nim_csourcesDir=csources_v1
+nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git
+nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff

tools/ci_generate.nim

@@ -12,9 +12,10 @@ nim r tools/ci_generate.nim
 
 import std/[strformat, os]
 
+const doNotEdit = "DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`"
 proc genCiBsd(header: string, batch: int, num: int): string =
   result = fmt"""
-## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
+## {doNotEdit}
 
 {header}
 
@@ -43,6 +44,64 @@ triggers:
   to: Andreas Rumpf <rumpf_a@web.de>
 """
 
+proc genBuildExtras(echoRun, koch, nim: string): string =
+  result = fmt"""
+{echoRun} {nim} c --skipUserCfg --skipParentCfg --hints:off koch
+{echoRun} {koch} boot -d:release --skipUserCfg --skipParentCfg --hints:off
+{echoRun} {koch} tools --skipUserCfg --skipParentCfg --hints:off
+"""
+
+proc genWindowsScript(buildAll: bool): string =
+  result = fmt"""
+@echo off
+rem {doNotEdit}
+rem Build development version of the compiler; can be rerun safely
+rem bare bones version of ci/funs.sh adapted for windows.
+
+rem Read in some common shared variables (shared with other tools),
+rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
+for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
+SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
+echo "building from csources: %nim_csources%"
+
+if not exist %nim_csourcesDir% (
+  git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir%
+)
+
+if not exist %nim_csources% (
+  cd %nim_csourcesDir%
+  git checkout %nim_csourcesHash%
+  echo "%PROCESSOR_ARCHITECTURE%"
+  if "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
+    SET ARCH=64
+  )
+  CALL build.bat
+  cd ..
+  copy /y bin\nim.exe  %nim_csources%
+)
+"""
+
+  if buildAll:
+    result.add genBuildExtras("", "koch", r"bin\nim.exe")
+
+proc genPosixScript(): string =
+  result = fmt"""
+#! /bin/sh
+rem {doNotEdit}
+
+# build development version of the compiler; can be rerun safely.
+# arguments can be passed, e.g.:
+# CC=gcc ucpu=amd64 uos=darwin
+
+set -u # error on undefined variables
+set -e # exit on first error
+
+. ci/funs.sh
+nimBuildCsourcesIfNeeded "$@"
+
+{genBuildExtras("echo_run", "./koch", "bin/nim")}
+"""
+
 proc main()=
   let dir = ".builds"
   # not too large to be resource friendly, refs bug #17107
@@ -74,9 +133,13 @@ packages:
 - sdl2
 - libffi
 """
+
   for i in 0..<num:
     writeFile(dir / fmt"openbsd_{i}.yml", genCiBsd(headerOpenbsd, i, num))
   writeFile(dir / "freebsd.yml", genCiBsd(headerFreebsd, 0, 1))
+  writeFile("build_all.sh", genPosixScript())
+  writeFile("build_all.bat", genWindowsScript(buildAll = true))
+  writeFile("ci/build_autogen.bat", genWindowsScript(buildAll = false))
 
 when isMainModule:
   main()