mirror of
https://github.com/odin-lang/Odin.git
synced 2026-04-19 13:00:28 +00:00
Merge pull request #3688 from Kelimion/nightly
Rewrite upload_b2 nightly action against B2 SDK
This commit is contained in:
Jeroen van Rijn
31
.github/workflows/nightly.yml
vendored
31
.github/workflows/nightly.yml
vendored
@@ -151,11 +151,11 @@ jobs:
|
||||
with:
|
||||
python-version: '3.8.x'
|
||||
|
||||
- name: Install B2 CLI
|
||||
- name: Install B2 SDK
|
||||
shell: bash
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install --upgrade b2
|
||||
pip install --upgrade b2sdk
|
||||
|
||||
- name: Display Python version
|
||||
run: python -c "import sys; print(sys.version)"
|
||||
@@ -188,24 +188,9 @@ jobs:
|
||||
BUCKET: ${{ secrets.B2_BUCKET }}
|
||||
DAYS_TO_KEEP: ${{ secrets.B2_DAYS_TO_KEEP }}
|
||||
run: |
|
||||
echo Authorizing B2 account
|
||||
b2 account authorize "$APPID" "$APPKEY"
|
||||
|
||||
echo Uploading artifcates to B2
|
||||
chmod +x ./ci/upload_create_nightly.sh
|
||||
./ci/upload_create_nightly.sh "$BUCKET" windows-amd64 windows_artifacts/
|
||||
./ci/upload_create_nightly.sh "$BUCKET" ubuntu-amd64 ubuntu_artifacts/dist.zip
|
||||
./ci/upload_create_nightly.sh "$BUCKET" macos-amd64 macos_artifacts/dist.zip
|
||||
./ci/upload_create_nightly.sh "$BUCKET" macos-arm64 macos_arm_artifacts/dist.zip
|
||||
|
||||
echo Deleting old artifacts in B2
|
||||
python3 ci/delete_old_binaries.py "$BUCKET" "$DAYS_TO_KEEP"
|
||||
|
||||
echo Creating nightly.json
|
||||
python3 ci/create_nightly_json.py "$BUCKET" > nightly.json
|
||||
|
||||
echo Uploading nightly.json
|
||||
b2 upload-file "$BUCKET" nightly.json nightly.json
|
||||
|
||||
echo Clear B2 account info
|
||||
b2 clear-account
|
||||
python3 ci/nightly.py artifact windows-amd64 windows_artifacts/
|
||||
python3 ci/nightly.py artifact ubuntu-amd64 ubuntu_artifacts/dist.zip
|
||||
python3 ci/nightly.py artifact macos-amd64 macos_artifacts/dist.zip
|
||||
python3 ci/nightly.py artifact macos-arm64 macos_arm_artifacts/dist.zip
|
||||
python3 ci/nightly.py prune
|
||||
python3 ci/nightly.py json
|
||||
@@ -1,51 +0,0 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import json
|
||||
import datetime
|
||||
import urllib.parse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
files_by_date = {}
|
||||
bucket = sys.argv[1]
|
||||
|
||||
files_lines = execute_cli(f"b2 ls --long b2://{bucket}/nightly/").split("\n")
|
||||
for x in files_lines:
|
||||
parts = x.split(" ", 1)
|
||||
if parts[0]:
|
||||
print(f"Parts[0]: {parts[0]}", flush=True)
|
||||
json_str = execute_cli(f"b2 file info b2://{bucket}/{parts[0]}")
|
||||
data = json.loads(json_str)
|
||||
name = remove_prefix(data['fileName'], "nightly/")
|
||||
url = f"https://f001.backblazeb2.com/file/{bucket}/nightly/{urllib.parse.quote_plus(name)}"
|
||||
sha1 = data['contentSha1']
|
||||
size = int(data['size'])
|
||||
ts = int(data['fileInfo']['src_last_modified_millis'])
|
||||
date = datetime.datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d')
|
||||
|
||||
if date not in files_by_date.keys():
|
||||
files_by_date[date] = []
|
||||
|
||||
files_by_date[date].append({
|
||||
'name': name,
|
||||
'url': url,
|
||||
'sha1': sha1,
|
||||
'sizeInBytes': size,
|
||||
})
|
||||
|
||||
now = datetime.datetime.utcnow().isoformat()
|
||||
|
||||
print(json.dumps({
|
||||
'last_updated' : now,
|
||||
'files': files_by_date
|
||||
}, sort_keys=True, indent=4))
|
||||
|
||||
def remove_prefix(text, prefix):
|
||||
return text[text.startswith(prefix) and len(prefix):]
|
||||
|
||||
def execute_cli(command):
|
||||
sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
|
||||
return sb.stdout.read().decode("utf-8");
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,33 +0,0 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import json
|
||||
import datetime
|
||||
import urllib.parse
|
||||
import sys
|
||||
|
||||
def main():
|
||||
files_by_date = {}
|
||||
bucket = sys.argv[1]
|
||||
days_to_keep = int(sys.argv[2])
|
||||
print(f"Looking for binaries to delete older than {days_to_keep} days")
|
||||
|
||||
files_lines = execute_cli(f"b2 ls --long --versions b2://{bucket}/nightly/").split("\n")
|
||||
for x in files_lines:
|
||||
parts = [y for y in x.split(' ') if y]
|
||||
|
||||
if parts and parts[0]:
|
||||
date = datetime.datetime.strptime(parts[2], '%Y-%m-%d').replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
now = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
delta = now - date
|
||||
|
||||
if delta.days > days_to_keep:
|
||||
print(f'Deleting b2://{bucket}/{parts[5]}')
|
||||
execute_cli(f'b2 rm b2://{bucket}/{parts[5]}')
|
||||
|
||||
|
||||
def execute_cli(command):
|
||||
sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
|
||||
return sb.stdout.read().decode("utf-8");
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
140
ci/nightly.py
Normal file
140
ci/nightly.py
Normal file
@@ -0,0 +1,140 @@
|
||||
import os
|
||||
import sys
|
||||
from zipfile import ZipFile
|
||||
from b2sdk.v2 import InMemoryAccountInfo, B2Api
|
||||
from datetime import datetime
|
||||
import json
|
||||
|
||||
UPLOAD_FOLDER = "nightly/"
|
||||
|
||||
info = InMemoryAccountInfo()
|
||||
b2_api = B2Api(info)
|
||||
application_key_id = os.environ['APPID']
|
||||
application_key = os.environ['APPKEY']
|
||||
bucket_name = os.environ['BUCKET']
|
||||
days_to_keep = os.environ['DAYS_TO_KEEP']
|
||||
|
||||
def auth() -> bool:
|
||||
try:
|
||||
realm = b2_api.account_info.get_realm()
|
||||
return True # Already authenticated
|
||||
except:
|
||||
pass # Not yet authenticated
|
||||
|
||||
err = b2_api.authorize_account("production", application_key_id, application_key)
|
||||
return err == None
|
||||
|
||||
def get_bucket():
|
||||
if not auth(): sys.exit(1)
|
||||
return b2_api.get_bucket_by_name(bucket_name)
|
||||
|
||||
def remove_prefix(text: str, prefix: str) -> str:
|
||||
return text[text.startswith(prefix) and len(prefix):]
|
||||
|
||||
def create_and_upload_artifact_zip(platform: str, artifact: str) -> int:
|
||||
now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
destination_zip_name = "odin-{}-nightly+{}.zip".format(platform, now.strftime("%Y-%m-%d"))
|
||||
|
||||
source_zip_name = artifact
|
||||
if not artifact.endswith(".zip"):
|
||||
print(f"Creating archive {destination_zip_name} from {artifact} and uploading to {bucket_name}")
|
||||
|
||||
source_zip_name = destination_zip_name
|
||||
with ZipFile(source_zip_name, 'w') as z:
|
||||
for root, directory, filenames in os.walk(artifact):
|
||||
for file in filenames:
|
||||
file_path = os.path.join(root, file)
|
||||
zip_path = os.path.join("dist", os.path.relpath(file_path, artifact))
|
||||
z.write(file_path, zip_path)
|
||||
|
||||
if not os.path.exists(source_zip_name):
|
||||
print(f"Error: Newly created ZIP archive {source_zip_name} not found.")
|
||||
return 1
|
||||
|
||||
print("Uploading {} to {}".format(source_zip_name, UPLOAD_FOLDER + destination_zip_name))
|
||||
bucket = get_bucket()
|
||||
res = bucket.upload_local_file(
|
||||
source_zip_name, # Local file to upload
|
||||
"nightly/" + destination_zip_name, # B2 destination path
|
||||
)
|
||||
return 0
|
||||
|
||||
def prune_artifacts():
|
||||
print(f"Looking for binaries to delete older than {days_to_keep} days")
|
||||
|
||||
bucket = get_bucket()
|
||||
for file, _ in bucket.ls(UPLOAD_FOLDER, latest_only=False):
|
||||
# Timestamp is in milliseconds
|
||||
date = datetime.fromtimestamp(file.upload_timestamp / 1_000.0).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
delta = now - date
|
||||
|
||||
if delta.days > int(days_to_keep):
|
||||
print("Deleting {}".format(file.file_name))
|
||||
file.delete()
|
||||
|
||||
return 0
|
||||
|
||||
def update_nightly_json():
|
||||
print(f"Updating nightly.json with files {days_to_keep} days or newer")
|
||||
|
||||
files_by_date = {}
|
||||
|
||||
bucket = get_bucket()
|
||||
|
||||
for file, _ in bucket.ls(UPLOAD_FOLDER, latest_only=True):
|
||||
# Timestamp is in milliseconds
|
||||
date = datetime.fromtimestamp(file.upload_timestamp / 1_000.0).replace(hour=0, minute=0, second=0, microsecond=0).strftime('%Y-%m-%d')
|
||||
name = remove_prefix(file.file_name, UPLOAD_FOLDER)
|
||||
sha1 = file.content_sha1
|
||||
size = file.size
|
||||
url = bucket.get_download_url(file.file_name)
|
||||
|
||||
if date not in files_by_date.keys():
|
||||
files_by_date[date] = []
|
||||
|
||||
files_by_date[date].append({
|
||||
'name': name,
|
||||
'url': url,
|
||||
'sha1': sha1,
|
||||
'sizeInBytes': size,
|
||||
})
|
||||
|
||||
now = datetime.utcnow().isoformat()
|
||||
|
||||
nightly = json.dumps({
|
||||
'last_updated' : now,
|
||||
'files': files_by_date
|
||||
}, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8')
|
||||
|
||||
res = bucket.upload_bytes(
|
||||
nightly, # JSON bytes
|
||||
"nightly.json", # B2 destination path
|
||||
)
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 1:
|
||||
print("Usage: {} <verb> [arguments]".format(sys.argv[0]))
|
||||
print("\tartifact <platform prefix> <artifact path>\n\t\tCreates and uploads a platform artifact zip.")
|
||||
print("\tprune\n\t\tDeletes old artifacts from bucket")
|
||||
print("\tjson\n\t\tUpdate and upload nightly.json")
|
||||
sys.exit(1)
|
||||
else:
|
||||
command = sys.argv[1].lower()
|
||||
if command == "artifact":
|
||||
if len(sys.argv) != 4:
|
||||
print("Usage: {} artifact <platform prefix> <artifact path>".format(sys.argv[0]))
|
||||
print("Error: Expected artifact command to be given platform prefix and artifact path.\n")
|
||||
sys.exit(1)
|
||||
|
||||
res = create_and_upload_artifact_zip(sys.argv[2], sys.argv[3])
|
||||
sys.exit(res)
|
||||
|
||||
elif command == "prune":
|
||||
res = prune_artifacts()
|
||||
sys.exit(res)
|
||||
|
||||
elif command == "json":
|
||||
res = update_nightly_json()
|
||||
sys.exit(res)
|
||||
@@ -1,25 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
bucket=$1
|
||||
platform=$2
|
||||
artifact=$3
|
||||
|
||||
now=$(date +'%Y-%m-%d')
|
||||
filename="odin-$platform-nightly+$now.zip"
|
||||
|
||||
echo "Creating archive $filename from $artifact and uploading to $bucket"
|
||||
|
||||
# If this is already zipped up (done before artifact upload to keep permissions in tact), just move it.
|
||||
if [ "${artifact: -4}" == ".zip" ]
|
||||
then
|
||||
echo "Artifact already a zip"
|
||||
mkdir -p "output"
|
||||
mv "$artifact" "output/$filename"
|
||||
else
|
||||
echo "Artifact needs to be zipped"
|
||||
7z a -bd "output/$filename" -r "$artifact"
|
||||
fi
|
||||
|
||||
b2 file upload "$bucket" "output/$filename" "nightly/$filename"
|
||||
Reference in New Issue
Block a user