From 036d7edda92c8d93c1295e4f2661152abe729bf1 Mon Sep 17 00:00:00 2001 From: A1029384756 Date: Thu, 5 Sep 2024 01:52:40 -0400 Subject: [PATCH] Changed Linux CI builds to static linking with Musl for better compatibility Changed to tarball distribution for mac and linux Updated upload-artifact to v4 --- .github/workflows/nightly.yml | 121 ++++++++++++++++++---------------- ci/build_linux_static.sh | 19 ++++++ ci/nightly.py | 39 ++++++----- 3 files changed, 106 insertions(+), 73 deletions(-) create mode 100755 ci/build_linux_static.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0c5526d0f..f6e50e60b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -36,43 +36,50 @@ jobs: cp -r bin dist cp -r examples dist - name: Upload artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: windows_artifacts path: dist - build_ubuntu: - name: Ubuntu Build + build_linux: + name: Linux Build if: github.repository == 'odin-lang/Odin' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: jirutka/setup-alpine@v1 + with: + branch: v3.20 - name: (Linux) Download LLVM run: | - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 18 - echo "/usr/lib/llvm-18/bin" >> $GITHUB_PATH + apk add --no-cache \ + musl-dev llvm18-dev clang18 git mold lz4 \ + libxml2-static llvm18-static zlib-static zstd-static \ + make + shell: alpine.sh --root {0} - name: build odin - run: make nightly + # NOTE: this build does slow compile times because of musl + run: ci/build_linux_static.sh + shell: alpine.sh {0} - name: Odin run run: ./odin run examples/demo - name: Copy artifacts run: | - mkdir dist - cp odin dist - cp LICENSE dist - cp -r shared dist - cp -r base dist - cp -r core dist - cp -r vendor dist - cp -r examples dist - # Zipping so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38 - zip -r dist.zip dist + FILE="odin-linux-amd64-nightly+$(date -I)" + mkdir $FILE + cp odin $FILE + cp LICENSE $FILE + cp -r shared $FILE + cp -r base $FILE + cp -r core $FILE + cp -r vendor $FILE + cp -r examples $FILE + # Creating a tarball so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38 + tar -czvf dist.tar.gz $FILE - name: Upload artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: - name: ubuntu_artifacts - path: dist.zip + name: linux_artifacts + path: dist.tar.gz build_macos: name: MacOS Build if: github.repository == 'odin-lang/Odin' @@ -89,24 +96,25 @@ jobs: run: CXXFLAGS="-L/usr/lib/system -L/usr/lib" make nightly - name: Bundle run: | - mkdir dist - cp odin dist - cp LICENSE dist - cp -r shared dist - cp -r base dist - cp -r core dist - cp -r vendor dist - cp -r examples dist - dylibbundler -b -x dist/odin -d dist/libs -od -p @executable_path/libs - # Zipping so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38 - zip -r dist.zip dist + FILE="odin-macos-amd64-nightly+$(date -I)" + mkdir $FILE + cp odin $FILE + cp LICENSE $FILE + cp -r shared $FILE + cp -r base $FILE + cp -r core $FILE + cp -r vendor $FILE + cp -r examples $FILE + dylibbundler -b -x $FILE/odin -d $FILE/libs -od -p @executable_path/libs + # Creating a tarball so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38 + tar -czvf dist.tar.gz $FILE - name: Odin run run: ./dist/odin run examples/demo - name: Upload artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: macos_artifacts - path: dist.zip + path: dist.tar.gz build_macos_arm: name: MacOS ARM Build if: github.repository == 'odin-lang/Odin' @@ -123,27 +131,28 @@ jobs: run: CXXFLAGS="-L/usr/lib/system -L/usr/lib" make nightly - name: Bundle run: | - mkdir dist - cp odin dist - cp LICENSE dist - cp -r shared dist - cp -r base dist - cp -r core dist - cp -r vendor dist - cp -r examples dist - dylibbundler -b -x dist/odin -d dist/libs -od -p @executable_path/libs - # Zipping so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38 - zip -r dist.zip dist + FILE="odin-macos-arm64-nightly+$(date -I)" + mkdir $FILE + cp odin $FILE + cp LICENSE $FILE + cp -r shared $FILE + cp -r base $FILE + cp -r core $FILE + cp -r vendor $FILE + cp -r examples $FILE + dylibbundler -b -x $FILE/odin -d $FILE/libs -od -p @executable_path/libs + # Creating a tarball so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38 + tar -czvf dist.tar.gz $FILE - name: Odin run run: ./dist/odin run examples/demo - name: Upload artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: macos_arm_artifacts - path: dist.zip + path: dist.tar.gz upload_b2: runs-on: [ubuntu-latest] - needs: [build_windows, build_macos, build_macos_arm, build_ubuntu] + needs: [build_windows, build_macos, build_macos_arm, build_linux] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v2 @@ -160,22 +169,22 @@ jobs: run: python -c "import sys; print(sys.version)" - name: Download Windows artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v4 with: name: windows_artifacts - name: Download Ubuntu artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v4 with: - name: ubuntu_artifacts + name: linux_artifacts - name: Download macOS artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v4 with: name: macos_artifacts - name: Download macOS arm artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v4 with: name: macos_arm_artifacts @@ -188,8 +197,8 @@ jobs: DAYS_TO_KEEP: ${{ secrets.B2_DAYS_TO_KEEP }} run: | python3 ci/nightly.py artifact windows-amd64 windows_artifacts/ - python3 ci/nightly.py artifact ubuntu-amd64 ubuntu_artifacts/dist.zip - python3 ci/nightly.py artifact macos-amd64 macos_artifacts/dist.zip - python3 ci/nightly.py artifact macos-arm64 macos_arm_artifacts/dist.zip + python3 ci/nightly.py artifact linux-amd64 linux_artifacts/dist.tar.gz + python3 ci/nightly.py artifact macos-amd64 macos_artifacts/dist.tar.gz + python3 ci/nightly.py artifact macos-arm64 macos_arm_artifacts/dist.tar.gz python3 ci/nightly.py prune python3 ci/nightly.py json diff --git a/ci/build_linux_static.sh b/ci/build_linux_static.sh new file mode 100755 index 000000000..f821cbb59 --- /dev/null +++ b/ci/build_linux_static.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +# Intended for use in Alpine containers, see the "nightly" Github action for a list of dependencies + +CXX="clang++-18" +LLVM_CONFIG="llvm-config-18" + +DISABLED_WARNINGS="-Wno-switch -Wno-macro-redefined -Wno-unused-value" + +CPPFLAGS="-DODIN_VERSION_RAW=\"dev-$(date +"%Y-%m")\"" +CXXFLAGS="-std=c++14 $($LLVM_CONFIG --cxxflags --ldflags)" + +LDFLAGS="-static -lm -lzstd -lz -lffi -pthread -ldl -fuse-ld=mold" +LDFLAGS="$LDFLAGS $($LLVM_CONFIG --link-static --ldflags --libs --system-libs --libfiles)" +LDFLAGS="$LDFLAGS -Wl,-rpath=\$ORIGIN" + +EXTRAFLAGS="-DNIGHTLY -O3" + +set -x +$CXX src/main.cpp src/libtommath.cpp $DISABLED_WARNINGS $CPPFLAGS $CXXFLAGS $EXTRAFLAGS $LDFLAGS -o odin diff --git a/ci/nightly.py b/ci/nightly.py index 7bd32899d..779a04d96 100644 --- a/ci/nightly.py +++ b/ci/nightly.py @@ -2,7 +2,7 @@ import os import sys from zipfile import ZipFile, ZIP_DEFLATED from b2sdk.v2 import InMemoryAccountInfo, B2Api -from datetime import datetime +from datetime import datetime, UTC import json UPLOAD_FOLDER = "nightly/" @@ -22,7 +22,7 @@ def auth() -> bool: pass # Not yet authenticated err = b2_api.authorize_account("production", application_key_id, application_key) - return err == None + return err is None def get_bucket(): if not auth(): sys.exit(1) @@ -32,30 +32,35 @@ def remove_prefix(text: str, prefix: str) -> str: return text[text.startswith(prefix) and len(prefix):] def create_and_upload_artifact_zip(platform: str, artifact: str) -> int: - now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) - destination_zip_name = "odin-{}-nightly+{}.zip".format(platform, now.strftime("%Y-%m-%d")) + now = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) - source_zip_name = artifact - if not artifact.endswith(".zip"): - print(f"Creating archive {destination_zip_name} from {artifact} and uploading to {bucket_name}") + source_archive: str + destination_name = f'odin-{platform}-nightly+{now.strftime("%Y-%m-%d")}' - source_zip_name = destination_zip_name - with ZipFile(source_zip_name, mode='w', compression=ZIP_DEFLATED, compresslevel=9) as z: + if platform.startswith("linux") or platform.startswith("macos"): + destination_name += ".tar.gz" + source_archive = artifact + else: + destination_name += ".zip" + source_archive = destination_name + + print(f"Creating archive {destination_name} from {artifact} and uploading to {bucket_name}") + with ZipFile(source_archive, mode='w', compression=ZIP_DEFLATED, compresslevel=9) as z: for root, directory, filenames in os.walk(artifact): for file in filenames: file_path = os.path.join(root, file) zip_path = os.path.join("dist", os.path.relpath(file_path, artifact)) z.write(file_path, zip_path) - if not os.path.exists(source_zip_name): - print(f"Error: Newly created ZIP archive {source_zip_name} not found.") + if not os.path.exists(source_archive): + print(f"Error: Newly created ZIP archive {source_archive} not found.") return 1 - print("Uploading {} to {}".format(source_zip_name, UPLOAD_FOLDER + destination_zip_name)) + print("Uploading {} to {}".format(source_archive, UPLOAD_FOLDER + destination_name)) bucket = get_bucket() res = bucket.upload_local_file( - source_zip_name, # Local file to upload - "nightly/" + destination_zip_name, # B2 destination path + source_archive, # Local file to upload + "nightly/" + destination_name, # B2 destination path ) return 0 @@ -66,7 +71,7 @@ def prune_artifacts(): for file, _ in bucket.ls(UPLOAD_FOLDER, latest_only=False): # Timestamp is in milliseconds date = datetime.fromtimestamp(file.upload_timestamp / 1_000.0).replace(hour=0, minute=0, second=0, microsecond=0) - now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + now = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) delta = now - date if delta.days > int(days_to_keep): @@ -100,7 +105,7 @@ def update_nightly_json(): 'sizeInBytes': size, }) - now = datetime.utcnow().isoformat() + now = datetime.now(UTC).isoformat() nightly = json.dumps({ 'last_updated' : now, @@ -137,4 +142,4 @@ if __name__ == "__main__": elif command == "json": res = update_nightly_json() - sys.exit(res) \ No newline at end of file + sys.exit(res)