release: build aarch64 libraries with 16kiB page size

(cherry picked from commit f201b64ffe)
This commit is contained in:
Anonymous Maarten
2025-08-09 05:25:31 +02:00
parent 9464f334b5
commit 0aef2c3612

View File

@@ -37,6 +37,8 @@ logger = logging.getLogger(__name__)
GIT_HASH_FILENAME = ".git-hash"
REVISION_TXT = "REVISION.txt"
RE_ILLEGAL_MINGW_LIBRARIES = re.compile(r"(?:lib)?(?:gcc|(?:std)?c[+][+]|(?:win)?pthread).*", flags=re.I)
def safe_isotime_to_datetime(str_isotime: str) -> datetime.datetime:
try:
@@ -339,6 +341,12 @@ class ArchiveFileTree:
def add_file(self, file: NodeInArchive):
self._tree[file.arcpath] = file
def __iter__(self) -> typing.Iterable[NodeInArchive]:
yield from self._tree.values()
def __contains__(self, value: str) -> bool:
return value in self._tree
def get_latest_mod_time(self) -> datetime.datetime:
return max(item.time for item in self._tree.values() if item.time)
@@ -526,6 +534,18 @@ class SourceCollector:
return path_times
class AndroidApiVersion:
def __init__(self, name: str, ints: tuple[int, ...]):
self.name = name
self.ints = ints
def __repr__(self) -> str:
return f"<{self.name} ({'.'.join(str(v) for v in self.ints)})>"
ANDROID_ABI_EXTRA_LINK_OPTIONS = {
"arm64-v8a": "-Wl,-z,max-page-size=16384 -Wl,-z,common-page-size=16384",
}
class Releaser:
def __init__(self, release_info: dict, commit: str, revision: str, root: Path, dist_path: Path, section_printer: SectionPrinter, executer: Executer, cmake_generator: str, deps_path: Path, overwrite: bool, github: bool, fast: bool):
self.release_info = release_info
@@ -586,12 +606,22 @@ class Releaser:
def create_source_archives(self) -> None:
source_collector = SourceCollector(root=self.root, commit=self.commit, executer=self.executer, filter=self._path_filter)
print(f"Collecting sources of {self.project}...")
archive_tree = source_collector.get_archive_file_tree()
archive_tree: ArchiveFileTree = source_collector.get_archive_file_tree()
latest_mod_time = archive_tree.get_latest_mod_time()
archive_tree.add_file(NodeInArchive.from_text(arcpath=REVISION_TXT, text=f"{self.revision}\n", time=latest_mod_time))
archive_tree.add_file(NodeInArchive.from_text(arcpath=f"{GIT_HASH_FILENAME}", text=f"{self.commit}\n", time=latest_mod_time))
archive_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["source"].get("files", {}), file_mapping_root=self.root, context=self.get_context(), time=latest_mod_time)
if "Makefile.am" in archive_tree:
patched_time = latest_mod_time + datetime.timedelta(minutes=1)
print(f"Makefile.am detected -> touching aclocal.m4, */Makefile.in, configure")
for node_data in archive_tree:
arc_name = os.path.basename(node_data.arcpath)
arc_name_we, arc_name_ext = os.path.splitext(arc_name)
if arc_name in ("aclocal.m4", "configure", "Makefile.in"):
print(f"Bumping time of {node_data.arcpath}")
node_data.time = patched_time
archive_base = f"{self.project}-{self.version}"
zip_path = self.dist_path / f"{archive_base}.zip"
tgz_path = self.dist_path / f"{archive_base}.tar.gz"
@@ -668,6 +698,15 @@ class Releaser:
def git_hash_data(self) -> bytes:
return f"{self.commit}\n".encode()
def verify_mingw_library(self, triplet: str, path: Path):
objdump_output = self.executer.check_output([f"{triplet}-objdump", "-p", str(path)])
libraries = re.findall(r"DLL Name: ([^\n]+)", objdump_output)
logger.info("%s (%s) libraries: %r", path, triplet, libraries)
illegal_libraries = list(filter(RE_ILLEGAL_MINGW_LIBRARIES.match, libraries))
logger.error("Detected 'illegal' libraries: %r", illegal_libraries)
if illegal_libraries:
raise Exception(f"{path} links to illegal libraries: {illegal_libraries}")
def create_mingw_archives(self) -> None:
build_type = "Release"
build_parent_dir = self.root / "build-mingw"
@@ -764,7 +803,6 @@ class Releaser:
f"--includedir=${{prefix}}/include",
f"--libdir=${{prefix}}/lib",
f"--bindir=${{prefix}}/bin",
f"--exec-prefix=${{prefix}}/bin",
f"--host={triplet}",
f"--build=x86_64-none-linux-gnu",
"CFLAGS=-O2",
@@ -772,9 +810,10 @@ class Releaser:
"LDFLAGS=-Wl,-s",
] + extra_args, cwd=build_path, env=new_env)
with self.section_printer.group(f"Build MinGW {triplet} (autotools)"):
self.executer.run(["make", "V=1", f"-j{self.cpu_count}"], cwd=build_path, env=new_env)
self.executer.run(["make", f"-j{self.cpu_count}"], cwd=build_path, env=new_env)
with self.section_printer.group(f"Install MinGW {triplet} (autotools)"):
self.executer.run(["make", "install"], cwd=build_path, env=new_env)
self.verify_mingw_library(triplet=ARCH_TO_TRIPLET[arch], path=install_path / "bin" / f"{self.project}.dll")
archive_file_tree.add_directory_tree(arc_dir=arc_join(arc_root, triplet), path=install_path, time=self.arc_time)
print("Recording arch-dependent extra files for MinGW development archive ...")
@@ -830,6 +869,7 @@ class Releaser:
self.executer.run(["cmake", "--build", str(build_path), "--verbose", "--config", build_type], cwd=build_path, env=new_env)
with self.section_printer.group(f"Install MinGW {triplet} (CMake)"):
self.executer.run(["cmake", "--install", str(build_path)], cwd=build_path, env=new_env)
self.verify_mingw_library(triplet=ARCH_TO_TRIPLET[arch], path=install_path / "bin" / f"{self.project}.dll")
archive_file_tree.add_directory_tree(arc_dir=arc_join(arc_root, triplet), path=install_path, time=self.arc_time)
print("Recording arch-dependent extra files for MinGW development archive ...")
@@ -857,22 +897,25 @@ class Releaser:
self.artifacts["mingw-devel-tar-gz"] = tgz_path
self.artifacts["mingw-devel-tar-xz"] = txz_path
def _detect_android_api(self, android_home: str) -> typing.Optional[int]:
def _detect_android_api(self, android_home: str) -> typing.Optional[AndroidApiVersion]:
platform_dirs = list(Path(p) for p in glob.glob(f"{android_home}/platforms/android-*"))
re_platform = re.compile("android-([0-9]+)")
platform_versions = []
re_platform = re.compile("^android-([0-9]+)(?:-ext([0-9]+))?$")
platform_versions: list[AndroidApiVersion] = []
for platform_dir in platform_dirs:
logger.debug("Found Android Platform SDK: %s", platform_dir)
if not (platform_dir / "android.jar").is_file():
logger.debug("Skipping SDK, missing android.jar")
continue
if m:= re_platform.match(platform_dir.name):
platform_versions.append(int(m.group(1)))
platform_versions.sort()
platform_versions.append(AndroidApiVersion(name=platform_dir.name, ints=(int(m.group(1)), int(m.group(2) or 0))))
platform_versions.sort(key=lambda v: v.ints)
logger.info("Available platform versions: %s", platform_versions)
platform_versions = list(filter(lambda v: v >= self._android_api_minimum, platform_versions))
logger.info("Valid platform versions (>=%d): %s", self._android_api_minimum, platform_versions)
platform_versions = list(filter(lambda v: v.ints >= self._android_api_minimum.ints, platform_versions))
logger.info("Valid platform versions (>=%s): %s", self._android_api_minimum.ints, platform_versions)
if not platform_versions:
return None
android_api = platform_versions[0]
logger.info("Selected API version %d", android_api)
logger.info("Selected API version %s", android_api)
return android_api
def _get_prefab_json_text(self) -> str:
@@ -896,8 +939,19 @@ class Releaser:
return json.dumps(module_json_dict, indent=4)
@property
def _android_api_minimum(self):
return self.release_info["android"]["api-minimum"]
def _android_api_minimum(self) -> AndroidApiVersion:
value = self.release_info["android"]["api-minimum"]
if isinstance(value, int):
ints = (value, )
elif isinstance(value, str):
ints = tuple(split("."))
else:
raise ValueError("Invalid android.api-minimum: must be X or X.Y")
match len(ints):
case 1: name = f"android-{ints[0]}"
case 2: name = f"android-{ints[0]}-ext-{ints[1]}"
case _: raise ValueError("Invalid android.api-minimum: must be X or X.Y")
return AndroidApiVersion(name=name, ints=ints)
@property
def _android_api_target(self):
@@ -910,7 +964,7 @@ class Releaser:
def _get_prefab_abi_json_text(self, abi: str, cpp: bool, shared: bool) -> str:
abi_json_dict = {
"abi": abi,
"api": self._android_api_minimum,
"api": self._android_api_minimum.ints[0],
"ndk": self._android_ndk_minimum,
"stl": "c++_shared" if cpp else "none",
"static": not shared,
@@ -923,7 +977,7 @@ class Releaser:
xmlns:android="http://schemas.android.com/apk/res/android"
package="org.libsdl.android.{self.project}" android:versionCode="1"
android:versionName="1.0">
<uses-sdk android:minSdkVersion="{self._android_api_minimum}"
<uses-sdk android:minSdkVersion="{self._android_api_minimum.ints[0]}"
android:targetSdkVersion="{self._android_api_target}" />
</manifest>
""")
@@ -933,7 +987,8 @@ class Releaser:
if not cmake_toolchain_file.exists():
logger.error("CMake toolchain file does not exist (%s)", cmake_toolchain_file)
raise SystemExit(1)
aar_path = self.dist_path / f"{self.project}-{self.version}.aar"
aar_path = self.root / "build-android" / f"{self.project}-{self.version}.aar"
android_dist_path = self.dist_path / f"{self.project}-devel-{self.version}-android.zip"
android_abis = self.release_info["android"]["abis"]
java_jars_added = False
module_data_added = False
@@ -941,16 +996,27 @@ class Releaser:
shutil.rmtree(android_deps_path, ignore_errors=True)
for dep, depinfo in self.release_info["android"].get("dependencies", {}).items():
android_aar = self.deps_path / glob.glob(depinfo["artifact"], root_dir=self.deps_path)[0]
with self.section_printer.group(f"Extracting Android dependency {dep} ({android_aar.name})"):
self.executer.run([sys.executable, str(android_aar), "-o", str(android_deps_path)])
dep_devel_zip = self.deps_path / glob.glob(depinfo["artifact"], root_dir=self.deps_path)[0]
dep_extract_path = self.deps_path / f"extract/android/{dep}"
shutil.rmtree(dep_extract_path, ignore_errors=True)
dep_extract_path.mkdir(parents=True, exist_ok=True)
with self.section_printer.group(f"Extracting Android dependency {dep} ({dep_devel_zip})"):
with zipfile.ZipFile(dep_devel_zip, "r") as zf:
zf.extractall(dep_extract_path)
dep_devel_aar = dep_extract_path / glob.glob("*.aar", root_dir=dep_extract_path)[0]
self.executer.run([sys.executable, str(dep_devel_aar), "-o", str(android_deps_path)])
for module_name, module_info in self.release_info["android"]["modules"].items():
assert "type" in module_info and module_info["type"] in ("interface", "library"), f"module {module_name} must have a valid type"
archive_file_tree = ArchiveFileTree()
aar_file_tree = ArchiveFileTree()
android_devel_file_tree = ArchiveFileTree()
for android_abi in android_abis:
extra_link_options = ANDROID_ABI_EXTRA_LINK_OPTIONS.get(android_abi, "")
with self.section_printer.group(f"Building for Android {android_api} {android_abi}"):
build_dir = self.root / "build-android" / f"{android_abi}-build"
install_dir = self.root / "install-android" / f"{android_abi}-install"
@@ -961,8 +1027,11 @@ class Releaser:
"cmake",
"-S", str(self.root),
"-B", str(build_dir),
f'''-DCMAKE_C_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''',
f'''-DCMAKE_CXX_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''',
# NDK 21e does not support -ffile-prefix-map
# f'''-DCMAKE_C_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''',
# f'''-DCMAKE_CXX_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''',
f"-DCMAKE_EXE_LINKER_FLAGS={extra_link_options}",
f"-DCMAKE_SHARED_LINKER_FLAGS={extra_link_options}",
f"-DCMAKE_TOOLCHAIN_FILE={cmake_toolchain_file}",
f"-DCMAKE_PREFIX_PATH={str(android_deps_path)}",
f"-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH",
@@ -999,20 +1068,20 @@ class Releaser:
assert library.suffix in (".so", ".a")
assert library.is_file(), f"CMake should have built library '{library}' for module {module_name}"
arcdir_prefab_libs = f"{arcdir_prefab_module}/libs/android.{android_abi}"
archive_file_tree.add_file(NodeInArchive.from_fs(arcpath=f"{arcdir_prefab_libs}/{library.name}", path=library, time=self.arc_time))
archive_file_tree.add_file(NodeInArchive.from_text(arcpath=f"{arcdir_prefab_libs}/abi.json", text=self._get_prefab_abi_json_text(abi=android_abi, cpp=False, shared=library.suffix == ".so"), time=self.arc_time))
aar_file_tree.add_file(NodeInArchive.from_fs(arcpath=f"{arcdir_prefab_libs}/{library.name}", path=library, time=self.arc_time))
aar_file_tree.add_file(NodeInArchive.from_text(arcpath=f"{arcdir_prefab_libs}/abi.json", text=self._get_prefab_abi_json_text(abi=android_abi, cpp=False, shared=library.suffix == ".so"), time=self.arc_time))
if not module_data_added:
library_name = None
if module_info["type"] == "library":
library_name = Path(module_info["library"]).stem.removeprefix("lib")
export_libraries = module_info.get("export-libraries", [])
archive_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_module, "module.json"), text=self._get_prefab_module_json_text(library_name=library_name, export_libraries=export_libraries), time=self.arc_time))
aar_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_module, "module.json"), text=self._get_prefab_module_json_text(library_name=library_name, export_libraries=export_libraries), time=self.arc_time))
arcdir_prefab_include = f"prefab/modules/{module_name}/include"
if "includes" in module_info:
archive_file_tree.add_file_mapping(arc_dir=arcdir_prefab_include, file_mapping=module_info["includes"], file_mapping_root=install_dir, context=self.get_context(), time=self.arc_time)
aar_file_tree.add_file_mapping(arc_dir=arcdir_prefab_include, file_mapping=module_info["includes"], file_mapping_root=install_dir, context=self.get_context(), time=self.arc_time)
else:
archive_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_include, ".keep"), text="\n", time=self.arc_time))
aar_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_include, ".keep"), text="\n", time=self.arc_time))
module_data_added = True
if not java_jars_added:
@@ -1025,21 +1094,28 @@ class Releaser:
assert sources_jar_path.is_file(), f"CMake should have archived the java sources into a JAR ({sources_jar_path})"
assert doc_jar_path.is_file(), f"CMake should have archived javadoc into a JAR ({doc_jar_path})"
archive_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes.jar", path=classes_jar_path, time=self.arc_time))
archive_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-sources.jar", path=sources_jar_path, time=self.arc_time))
archive_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-doc.jar", path=doc_jar_path, time=self.arc_time))
aar_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes.jar", path=classes_jar_path, time=self.arc_time))
aar_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-sources.jar", path=sources_jar_path, time=self.arc_time))
aar_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-doc.jar", path=doc_jar_path, time=self.arc_time))
assert ("jars" in self.release_info["android"] and java_jars_added) or "jars" not in self.release_info["android"], "Must have archived java JAR archives"
archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["android"].get("files", {}), file_mapping_root=self.root, context=self.get_context(), time=self.arc_time)
aar_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["android"]["aar-files"], file_mapping_root=self.root, context=self.get_context(), time=self.arc_time)
archive_file_tree.add_file(NodeInArchive.from_text(arcpath="prefab/prefab.json", text=self._get_prefab_json_text(), time=self.arc_time))
archive_file_tree.add_file(NodeInArchive.from_text(arcpath="AndroidManifest.xml", text=self._get_android_manifest_text(), time=self.arc_time))
aar_file_tree.add_file(NodeInArchive.from_text(arcpath="prefab/prefab.json", text=self._get_prefab_json_text(), time=self.arc_time))
aar_file_tree.add_file(NodeInArchive.from_text(arcpath="AndroidManifest.xml", text=self._get_android_manifest_text(), time=self.arc_time))
with Archiver(zip_path=aar_path) as archiver:
archive_file_tree.add_to_archiver(archive_base="", archiver=archiver)
aar_file_tree.add_to_archiver(archive_base="", archiver=archiver)
archiver.add_git_hash(arcdir="", commit=self.commit, time=self.arc_time)
self.artifacts[f"android-aar"] = aar_path
android_devel_file_tree.add_file(NodeInArchive.from_fs(arcpath=aar_path.name, path=aar_path))
android_devel_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["android"]["files"], file_mapping_root=self.root, context=self.get_context(), time=self.arc_time)
with Archiver(zip_path=android_dist_path) as archiver:
android_devel_file_tree.add_to_archiver(archive_base="", archiver=archiver)
archiver.add_git_hash(arcdir="", commit=self.commit, time=self.arc_time)
self.artifacts[f"android-aar"] = android_dist_path
def download_dependencies(self):
shutil.rmtree(self.deps_path, ignore_errors=True)
@@ -1076,7 +1152,7 @@ class Releaser:
assert len(msvc_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {msvc_matches}"
if "android" in self.release_info:
android_matches = glob.glob(self.release_info["android"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)
assert len(android_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {msvc_matches}"
assert len(android_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {android_matches}"
@staticmethod
def _arch_to_vs_platform(arch: str, configuration: str="Release") -> VsArchPlatformConfig:
@@ -1202,6 +1278,10 @@ class Releaser:
platform_context = self.get_context(extra_context=arch_platform.extra_context())
build_type = "Release"
extra_context = {
"ARCH": arch_platform.arch,
"PLATFORM": arch_platform.platform,
}
built_paths = set(install_path / configure_text(f, context=platform_context) for file_mapping in (self.release_info["msvc"]["cmake"]["files-lib"], self.release_info["msvc"]["cmake"]["files-devel"]) for files_list in file_mapping.values() for f in files_list)
logger.info("CMake builds these files, to be included in the package: %s", built_paths)
@@ -1252,7 +1332,7 @@ class Releaser:
logger.info("Collecting files...")
archive_file_tree = ArchiveFileTree()
archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["cmake"]["files-lib"], file_mapping_root=install_path, context=platform_context, time=self.arc_time)
archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["files-lib"], file_mapping_root=self.root, context=self.get_context(), time=self.arc_time)
archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["files-lib"], file_mapping_root=self.root, context=self.get_context(extra_context=extra_context), time=self.arc_time)
logger.info("Creating %s", zip_path)
with Archiver(zip_path=zip_path) as archiver:
@@ -1316,7 +1396,7 @@ def main(argv=None) -> int:
parser.add_argument("--actions", choices=["download", "source", "android", "mingw", "msvc", "dmg"], required=True, nargs="+", dest="actions", help="What to do?")
parser.set_defaults(loglevel=logging.INFO)
parser.add_argument('--vs-year', dest="vs_year", help="Visual Studio year")
parser.add_argument('--android-api', type=int, dest="android_api", help="Android API version")
parser.add_argument('--android-api', dest="android_api", help="Android API version")
parser.add_argument('--android-home', dest="android_home", default=os.environ.get("ANDROID_HOME"), help="Android Home folder")
parser.add_argument('--android-ndk-home', dest="android_ndk_home", default=os.environ.get("ANDROID_NDK_HOME"), help="Android NDK Home folder")
parser.add_argument('--cmake-generator', dest="cmake_generator", default="Ninja", help="CMake Generator")
@@ -1443,14 +1523,27 @@ def main(argv=None) -> int:
if args.android_api is None:
with section_printer.group("Detect Android APIS"):
args.android_api = releaser._detect_android_api(android_home=args.android_home)
if args.android_api is None or not (Path(args.android_home) / f"platforms/android-{args.android_api}").is_dir():
else:
try:
android_api_ints = tuple(int(v) for v in args.android_api.split("."))
match len(android_api_ints):
case 1: android_api_name = f"android-{android_api_ints[0]}"
case 2: android_api_name = f"android-{android_api_ints[0]}-ext-{android_api_ints[1]}"
case _: raise ValueError
except ValueError:
logger.error("Invalid --android-api, must be a 'X' or 'X.Y' version")
args.android_api = AndroidApiVersion(ints=android_api_ints, name=android_api_name)
if args.android_api is None:
parser.error("Invalid --android-api, and/or could not be detected")
android_api_path = Path(args.android_home) / f"platforms/{args.android_api.name}"
if not android_api_path.is_dir():
logger.warning(f"Android API directory does not exist ({android_api_path})")
with section_printer.group("Android arguments"):
print(f"android_home = {args.android_home}")
print(f"android_ndk_home = {args.android_ndk_home}")
print(f"android_api = {args.android_api}")
releaser.create_android_archives(
android_api=args.android_api,
android_api=args.android_api.ints[0],
android_home=args.android_home,
android_ndk_home=args.android_ndk_home,
)