Merge pull request #4204 from A1029384756/static_builds

Changed Linux CI builds to static linking with Musl for better compatibility

.github/workflows/nightly.yml

@@ -40,39 +40,46 @@ jobs:
         with:
           name: windows_artifacts
           path: dist
-  build_ubuntu:
-    name: Ubuntu Build
+  build_linux:
+    name: Linux Build
     if: github.repository == 'odin-lang/Odin'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: jirutka/setup-alpine@v1
+        with:
+          branch: v3.20
       - name: (Linux) Download LLVM
         run: |
-          wget https://apt.llvm.org/llvm.sh
-          chmod +x llvm.sh
-          sudo ./llvm.sh 18
-          echo "/usr/lib/llvm-18/bin" >> $GITHUB_PATH
+          apk add --no-cache \
+          musl-dev llvm18-dev clang18 git mold lz4 \
+          libxml2-static llvm18-static zlib-static zstd-static \
+          make
+        shell: alpine.sh --root {0}
       - name: build odin
-        run: make nightly
+        # NOTE: this build does slow compile times because of musl
+        run: ci/build_linux_static.sh
+        shell: alpine.sh {0}
       - name: Odin run
         run: ./odin run examples/demo
       - name: Copy artifacts
         run: |
-          mkdir dist
-          cp odin dist
-          cp LICENSE dist
-          cp -r shared dist
-          cp -r base dist
-          cp -r core dist
-          cp -r vendor dist
-          cp -r examples dist
-          # Zipping so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38
-          zip -r dist.zip dist
+          FILE="odin-linux-amd64-nightly+$(date -I)"
+          mkdir $FILE
+          cp odin $FILE
+          cp LICENSE $FILE
+          cp -r shared $FILE
+          cp -r base $FILE
+          cp -r core $FILE
+          cp -r vendor $FILE
+          cp -r examples $FILE
+          # Creating a tarball so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38
+          tar -czvf dist.tar.gz $FILE
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ubuntu_artifacts
-          path: dist.zip
+          name: linux_artifacts
+          path: dist.tar.gz
   build_macos:
     name: MacOS Build
     if: github.repository == 'odin-lang/Odin'
@@ -89,24 +96,25 @@ jobs:
         run: CXXFLAGS="-L/usr/lib/system -L/usr/lib" make nightly
       - name: Bundle
         run: |
-          mkdir dist
-          cp odin dist
-          cp LICENSE dist
-          cp -r shared dist
-          cp -r base dist
-          cp -r core dist
-          cp -r vendor dist
-          cp -r examples dist
-          dylibbundler -b -x dist/odin -d dist/libs -od -p @executable_path/libs
-          # Zipping so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38
-          zip -r dist.zip dist
+          FILE="odin-macos-amd64-nightly+$(date -I)"
+          mkdir $FILE
+          cp odin $FILE
+          cp LICENSE $FILE
+          cp -r shared $FILE
+          cp -r base $FILE
+          cp -r core $FILE
+          cp -r vendor $FILE
+          cp -r examples $FILE
+          dylibbundler -b -x $FILE/odin -d $FILE/libs -od -p @executable_path/libs
+          # Creating a tarball so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38
+          tar -czvf dist.tar.gz $FILE
       - name: Odin run
         run: ./dist/odin run examples/demo
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
           name: macos_artifacts
-          path: dist.zip
+          path: dist.tar.gz
   build_macos_arm:
     name: MacOS ARM Build
     if: github.repository == 'odin-lang/Odin'
@@ -123,27 +131,28 @@ jobs:
         run: CXXFLAGS="-L/usr/lib/system -L/usr/lib" make nightly
       - name: Bundle
         run: |
-          mkdir dist
-          cp odin dist
-          cp LICENSE dist
-          cp -r shared dist
-          cp -r base dist
-          cp -r core dist
-          cp -r vendor dist
-          cp -r examples dist
-          dylibbundler -b -x dist/odin -d dist/libs -od -p @executable_path/libs
-          # Zipping so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38
-          zip -r dist.zip dist
+          FILE="odin-macos-arm64-nightly+$(date -I)"
+          mkdir $FILE
+          cp odin $FILE
+          cp LICENSE $FILE
+          cp -r shared $FILE
+          cp -r base $FILE
+          cp -r core $FILE
+          cp -r vendor $FILE
+          cp -r examples $FILE
+          dylibbundler -b -x $FILE/odin -d $FILE/libs -od -p @executable_path/libs
+          # Creating a tarball so executable permissions are retained, see https://github.com/actions/upload-artifact/issues/38
+          tar -czvf dist.tar.gz $FILE
       - name: Odin run
         run: ./dist/odin run examples/demo
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
           name: macos_arm_artifacts
-          path: dist.zip
+          path: dist.tar.gz
   upload_b2:
     runs-on: [ubuntu-latest]
-    needs: [build_windows, build_macos, build_macos_arm, build_ubuntu]
+    needs: [build_windows, build_macos, build_macos_arm, build_linux]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v2
@@ -160,6 +169,7 @@ jobs:
         run: python -c "import sys; print(sys.version)"
 
       - name: Download Windows artifacts
+
         uses: actions/download-artifact@v4.1.7
         with:
           name: windows_artifacts
@@ -167,7 +177,7 @@ jobs:
       - name: Download Ubuntu artifacts
         uses: actions/download-artifact@v4.1.7
         with:
-          name: ubuntu_artifacts
+          name: linux_artifacts
 
       - name: Download macOS artifacts
         uses: actions/download-artifact@v4.1.7
@@ -188,8 +198,8 @@ jobs:
           DAYS_TO_KEEP: ${{ secrets.B2_DAYS_TO_KEEP }}
         run: |
           python3 ci/nightly.py artifact windows-amd64 windows_artifacts/
-          python3 ci/nightly.py artifact ubuntu-amd64 ubuntu_artifacts/dist.zip
-          python3 ci/nightly.py artifact macos-amd64 macos_artifacts/dist.zip
-          python3 ci/nightly.py artifact macos-arm64 macos_arm_artifacts/dist.zip
+          python3 ci/nightly.py artifact linux-amd64 linux_artifacts/dist.tar.gz
+          python3 ci/nightly.py artifact macos-amd64 macos_artifacts/dist.tar.gz
+          python3 ci/nightly.py artifact macos-arm64 macos_arm_artifacts/dist.tar.gz
           python3 ci/nightly.py prune
           python3 ci/nightly.py json

ci/build_linux_static.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env sh
+# Intended for use in Alpine containers, see the "nightly" Github action for a list of dependencies
+
+CXX="clang++-18"
+LLVM_CONFIG="llvm-config-18"
+
+DISABLED_WARNINGS="-Wno-switch -Wno-macro-redefined -Wno-unused-value"
+
+CPPFLAGS="-DODIN_VERSION_RAW=\"dev-$(date +"%Y-%m")\""
+CXXFLAGS="-std=c++14 $($LLVM_CONFIG --cxxflags --ldflags)"
+
+LDFLAGS="-static -lm -lzstd -lz -lffi -pthread -ldl -fuse-ld=mold"
+LDFLAGS="$LDFLAGS $($LLVM_CONFIG --link-static --ldflags --libs --system-libs --libfiles)"
+LDFLAGS="$LDFLAGS -Wl,-rpath=\$ORIGIN"
+
+EXTRAFLAGS="-DNIGHTLY -O3"
+
+set -x
+$CXX src/main.cpp src/libtommath.cpp $DISABLED_WARNINGS $CPPFLAGS $CXXFLAGS $EXTRAFLAGS $LDFLAGS -o odin

ci/nightly.py

@@ -2,7 +2,7 @@ import os
 import sys
 from zipfile  import ZipFile, ZIP_DEFLATED
 from b2sdk.v2 import InMemoryAccountInfo, B2Api
-from datetime import datetime
+from datetime import datetime, UTC
 import json
 
 UPLOAD_FOLDER = "nightly/"
@@ -22,7 +22,7 @@ def auth() -> bool:
 		pass        # Not yet authenticated
 
 	err = b2_api.authorize_account("production", application_key_id, application_key)
-	return err == None
+	return err is None
 
 def get_bucket():
 	if not auth(): sys.exit(1)
@@ -32,30 +32,35 @@ def remove_prefix(text: str, prefix: str) -> str:
 	return text[text.startswith(prefix) and len(prefix):]
 
 def create_and_upload_artifact_zip(platform: str, artifact: str) -> int:
-	now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
-	destination_zip_name = "odin-{}-nightly+{}.zip".format(platform, now.strftime("%Y-%m-%d"))
+	now = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
 
-	source_zip_name = artifact
-	if not artifact.endswith(".zip"):
-		print(f"Creating archive {destination_zip_name} from {artifact} and uploading to {bucket_name}")
+	source_archive: str
+	destination_name = f'odin-{platform}-nightly+{now.strftime("%Y-%m-%d")}'
 
-		source_zip_name = destination_zip_name
-		with ZipFile(source_zip_name, mode='w', compression=ZIP_DEFLATED, compresslevel=9) as z:
+	if platform.startswith("linux") or platform.startswith("macos"):
+		destination_name += ".tar.gz"
+		source_archive = artifact
+	else:
+		destination_name += ".zip"
+		source_archive = destination_name
+
+		print(f"Creating archive {destination_name} from {artifact} and uploading to {bucket_name}")
+		with ZipFile(source_archive, mode='w', compression=ZIP_DEFLATED, compresslevel=9) as z:
 			for root, directory, filenames in os.walk(artifact):
 				for file in filenames:
 					file_path = os.path.join(root, file)
 					zip_path  = os.path.join("dist", os.path.relpath(file_path, artifact))
 					z.write(file_path, zip_path)
 
-		if not os.path.exists(source_zip_name):
-			print(f"Error: Newly created ZIP archive {source_zip_name} not found.")
+		if not os.path.exists(source_archive):
+			print(f"Error: Newly created ZIP archive {source_archive} not found.")
 			return 1
 
-	print("Uploading {} to {}".format(source_zip_name, UPLOAD_FOLDER + destination_zip_name))
+	print("Uploading {} to {}".format(source_archive, UPLOAD_FOLDER + destination_name))
 	bucket = get_bucket()
 	res = bucket.upload_local_file(
-		source_zip_name,                   # Local file to upload
-		"nightly/" + destination_zip_name, # B2 destination path
+		source_archive,               # Local file to upload
+		"nightly/" + destination_name, # B2 destination path
 	)
 	return 0
 
@@ -66,7 +71,7 @@ def prune_artifacts():
 	for file, _ in bucket.ls(UPLOAD_FOLDER, latest_only=False):
 		# Timestamp is in milliseconds
 		date  = datetime.fromtimestamp(file.upload_timestamp / 1_000.0).replace(hour=0, minute=0, second=0, microsecond=0)
-		now   = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
+		now   = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
 		delta = now - date
 
 		if delta.days > int(days_to_keep):
@@ -100,7 +105,7 @@ def update_nightly_json():
 			'sizeInBytes': size,
 		})
 
-	now = datetime.utcnow().isoformat()
+	now = datetime.now(UTC).isoformat()
 
 	nightly = json.dumps({
 		'last_updated' : now,
@@ -137,4 +142,4 @@ if __name__ == "__main__":
 
 		elif command == "json":
 			res = update_nightly_json()
-			sys.exit(res)
+			sys.exit(res)