From 1f1d85bb9c614d93ce38becdcc421dda00264a75 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 30 Apr 2021 22:24:41 -0700 Subject: [PATCH] reuse config/build_config.txt for all bootstrap scripts (posix + windows + ci); use build_all.bat in 1 CI, fix bug in build_all.bat (#17899) * reuse config/build_config.txt for all bootstrap scripts (posix + windows + ci) * ci_docs: use build_all.bat in CI (just in that pipeline) to ensure it keeps working * fixup * fix pre-existing bug in build_all.bat * fixup * cp => copy /y * auto-generate build_all.bat, build_all.sh * fixup --- .github/workflows/ci_docs.yml | 10 +++++- build_all.bat | 35 ++++++++++++------- build_all.sh | 9 ++--- ci/build_autogen.bat | 26 ++++++++++++++ ci/funs.sh | 8 ++--- config/build_config.txt | 4 +++ tools/ci_generate.nim | 65 ++++++++++++++++++++++++++++++++++- 7 files changed, 134 insertions(+), 23 deletions(-) create mode 100644 ci/build_autogen.bat create mode 100644 config/build_config.txt diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 1459d6a317..dfcff66e0e 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -70,13 +70,21 @@ jobs: shell: bash run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - name: 'Build csourcesAny' + - name: 'Build csourcesAny (posix)' + # this would work on windows and other CI use this on windows, + # but we ensure here that `ci/build_autogen.bat` keeps working on windows. + if: runner.os != 'Windows' shell: bash run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc # was previously using caching via `actions/cache@v1` but this wasn't # used in other CI pipelines and it's unclear the added complexity # was worth the saving; can be revisited if needed. + - name: 'Build csourcesAny (windows)' + if: runner.os == 'Windows' + shell: cmd + run: ci/build_autogen.bat + - name: 'Build koch' shell: bash run: nim c koch diff --git a/build_all.bat b/build_all.bat index 78a7a354a3..1bff761b34 100644 --- a/build_all.bat +++ b/build_all.bat @@ -1,20 +1,29 @@ @echo off -rem build development version of the compiler; can be rerun safely -rem TODO: call nimBuildCsourcesIfNeeded or auto-generate this file (from a nim script) -rem to avoid duplication. -if not exist csources_v1 ( - git clone --depth 1 https://github.com/nim-lang/csources_v1.git +rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% ) -if not exist bin\nim.exe ( - cd csources_v1 - git checkout a8a5241f9475099c823cfe1a5e0ca4022ac201ff - if PROCESSOR_ARCHITECTURE == AMD64 ( + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( SET ARCH=64 ) CALL build.bat cd .. + copy /y bin\nim.exe %nim_csources% ) -bin\nim.exe c --skipUserCfg --skipParentCfg koch -koch.exe boot -d:release --skipUserCfg --skipParentCfg -koch.exe tools --skipUserCfg --skipParentCfg - + bin\nim.exe c --skipUserCfg --skipParentCfg --hints:off koch + koch boot -d:release --skipUserCfg --skipParentCfg --hints:off + koch tools --skipUserCfg --skipParentCfg --hints:off diff --git a/build_all.sh b/build_all.sh index 49504c8fe4..90665672f1 100755 --- a/build_all.sh +++ b/build_all.sh @@ -1,4 +1,5 @@ #! /bin/sh +rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` # build development version of the compiler; can be rerun safely. # arguments can be passed, e.g.: @@ -10,7 +11,7 @@ set -e # exit on first error . ci/funs.sh nimBuildCsourcesIfNeeded "$@" -# Note: if fails, may need to update csourcesAny manually -echo_run bin/nim c --skipUserCfg --skipParentCfg koch -echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg -echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools. +echo_run bin/nim c --skipUserCfg --skipParentCfg --hints:off koch +echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off +echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off + diff --git a/ci/build_autogen.bat b/ci/build_autogen.bat new file mode 100644 index 0000000000..8b1113175c --- /dev/null +++ b/ci/build_autogen.bat @@ -0,0 +1,26 @@ +@echo off +rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% +) + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( + SET ARCH=64 + ) + CALL build.bat + cd .. + copy /y bin\nim.exe %nim_csources% +) diff --git a/ci/funs.sh b/ci/funs.sh index 7a6c66722f..5158f38d53 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -32,9 +32,7 @@ nimIsCiSkip(){ } nimDefineVars(){ - nim_csourcesDir=csources_v1 # where we clone - nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git - nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff + . config/build_config.txt nim_csources=bin/nim_csources_$nim_csourcesHash } @@ -88,9 +86,11 @@ nimBuildCsourcesIfNeeded(){ if test -d "$nim_csourcesDir"; then echo "$nim_csourcesDir exists." else - # depth 1: adjust as needed in case useful for `git bisect` + # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g. + # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`. echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir" echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash + # if needed we could also add: `git reset --hard $nim_csourcesHash` fi _nimBuildCsourcesIfNeeded "$@" fi diff --git a/config/build_config.txt b/config/build_config.txt new file mode 100644 index 0000000000..2484d7a525 --- /dev/null +++ b/config/build_config.txt @@ -0,0 +1,4 @@ +nim_comment="key-value pairs for windows/posix bootstrapping build scripts" +nim_csourcesDir=csources_v1 +nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git +nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index 6517df34a3..2e6b4b3dee 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -12,9 +12,10 @@ nim r tools/ci_generate.nim import std/[strformat, os] +const doNotEdit = "DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`" proc genCiBsd(header: string, batch: int, num: int): string = result = fmt""" -## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` +## {doNotEdit} {header} @@ -43,6 +44,64 @@ triggers: to: Andreas Rumpf """ +proc genBuildExtras(echoRun, koch, nim: string): string = + result = fmt""" +{echoRun} {nim} c --skipUserCfg --skipParentCfg --hints:off koch +{echoRun} {koch} boot -d:release --skipUserCfg --skipParentCfg --hints:off +{echoRun} {koch} tools --skipUserCfg --skipParentCfg --hints:off +""" + +proc genWindowsScript(buildAll: bool): string = + result = fmt""" +@echo off +rem {doNotEdit} +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% +) + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( + SET ARCH=64 + ) + CALL build.bat + cd .. + copy /y bin\nim.exe %nim_csources% +) +""" + + if buildAll: + result.add genBuildExtras("", "koch", r"bin\nim.exe") + +proc genPosixScript(): string = + result = fmt""" +#! /bin/sh +rem {doNotEdit} + +# build development version of the compiler; can be rerun safely. +# arguments can be passed, e.g.: +# CC=gcc ucpu=amd64 uos=darwin + +set -u # error on undefined variables +set -e # exit on first error + +. ci/funs.sh +nimBuildCsourcesIfNeeded "$@" + +{genBuildExtras("echo_run", "./koch", "bin/nim")} +""" + proc main()= let dir = ".builds" # not too large to be resource friendly, refs bug #17107 @@ -74,9 +133,13 @@ packages: - sdl2 - libffi """ + for i in 0..