diff --git a/build-scripts/build-release.py b/build-scripts/build-release.py index d3f44a3b6f..b256e4d54c 100755 --- a/build-scripts/build-release.py +++ b/build-scripts/build-release.py @@ -37,6 +37,8 @@ logger = logging.getLogger(__name__) GIT_HASH_FILENAME = ".git-hash" REVISION_TXT = "REVISION.txt" +RE_ILLEGAL_MINGW_LIBRARIES = re.compile(r"(?:lib)?(?:gcc|(?:std)?c[+][+]|(?:win)?pthread).*", flags=re.I) + def safe_isotime_to_datetime(str_isotime: str) -> datetime.datetime: try: @@ -339,6 +341,12 @@ class ArchiveFileTree: def add_file(self, file: NodeInArchive): self._tree[file.arcpath] = file + def __iter__(self) -> typing.Iterable[NodeInArchive]: + yield from self._tree.values() + + def __contains__(self, value: str) -> bool: + return value in self._tree + def get_latest_mod_time(self) -> datetime.datetime: return max(item.time for item in self._tree.values() if item.time) @@ -526,6 +534,18 @@ class SourceCollector: return path_times +class AndroidApiVersion: + def __init__(self, name: str, ints: tuple[int, ...]): + self.name = name + self.ints = ints + + def __repr__(self) -> str: + return f"<{self.name} ({'.'.join(str(v) for v in self.ints)})>" + +ANDROID_ABI_EXTRA_LINK_OPTIONS = { + "arm64-v8a": "-Wl,-z,max-page-size=16384 -Wl,-z,common-page-size=16384", +} + class Releaser: def __init__(self, release_info: dict, commit: str, revision: str, root: Path, dist_path: Path, section_printer: SectionPrinter, executer: Executer, cmake_generator: str, deps_path: Path, overwrite: bool, github: bool, fast: bool): self.release_info = release_info @@ -586,12 +606,22 @@ class Releaser: def create_source_archives(self) -> None: source_collector = SourceCollector(root=self.root, commit=self.commit, executer=self.executer, filter=self._path_filter) print(f"Collecting sources of {self.project}...") - archive_tree = source_collector.get_archive_file_tree() + archive_tree: ArchiveFileTree = source_collector.get_archive_file_tree() latest_mod_time = archive_tree.get_latest_mod_time() archive_tree.add_file(NodeInArchive.from_text(arcpath=REVISION_TXT, text=f"{self.revision}\n", time=latest_mod_time)) archive_tree.add_file(NodeInArchive.from_text(arcpath=f"{GIT_HASH_FILENAME}", text=f"{self.commit}\n", time=latest_mod_time)) archive_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["source"].get("files", {}), file_mapping_root=self.root, context=self.get_context(), time=latest_mod_time) + if "Makefile.am" in archive_tree: + patched_time = latest_mod_time + datetime.timedelta(minutes=1) + print(f"Makefile.am detected -> touching aclocal.m4, */Makefile.in, configure") + for node_data in archive_tree: + arc_name = os.path.basename(node_data.arcpath) + arc_name_we, arc_name_ext = os.path.splitext(arc_name) + if arc_name in ("aclocal.m4", "configure", "Makefile.in"): + print(f"Bumping time of {node_data.arcpath}") + node_data.time = patched_time + archive_base = f"{self.project}-{self.version}" zip_path = self.dist_path / f"{archive_base}.zip" tgz_path = self.dist_path / f"{archive_base}.tar.gz" @@ -668,6 +698,15 @@ class Releaser: def git_hash_data(self) -> bytes: return f"{self.commit}\n".encode() + def verify_mingw_library(self, triplet: str, path: Path): + objdump_output = self.executer.check_output([f"{triplet}-objdump", "-p", str(path)]) + libraries = re.findall(r"DLL Name: ([^\n]+)", objdump_output) + logger.info("%s (%s) libraries: %r", path, triplet, libraries) + illegal_libraries = list(filter(RE_ILLEGAL_MINGW_LIBRARIES.match, libraries)) + logger.error("Detected 'illegal' libraries: %r", illegal_libraries) + if illegal_libraries: + raise Exception(f"{path} links to illegal libraries: {illegal_libraries}") + def create_mingw_archives(self) -> None: build_type = "Release" build_parent_dir = self.root / "build-mingw" @@ -764,7 +803,6 @@ class Releaser: f"--includedir=${{prefix}}/include", f"--libdir=${{prefix}}/lib", f"--bindir=${{prefix}}/bin", - f"--exec-prefix=${{prefix}}/bin", f"--host={triplet}", f"--build=x86_64-none-linux-gnu", "CFLAGS=-O2", @@ -772,9 +810,10 @@ class Releaser: "LDFLAGS=-Wl,-s", ] + extra_args, cwd=build_path, env=new_env) with self.section_printer.group(f"Build MinGW {triplet} (autotools)"): - self.executer.run(["make", "V=1", f"-j{self.cpu_count}"], cwd=build_path, env=new_env) + self.executer.run(["make", f"-j{self.cpu_count}"], cwd=build_path, env=new_env) with self.section_printer.group(f"Install MinGW {triplet} (autotools)"): self.executer.run(["make", "install"], cwd=build_path, env=new_env) + self.verify_mingw_library(triplet=ARCH_TO_TRIPLET[arch], path=install_path / "bin" / f"{self.project}.dll") archive_file_tree.add_directory_tree(arc_dir=arc_join(arc_root, triplet), path=install_path, time=self.arc_time) print("Recording arch-dependent extra files for MinGW development archive ...") @@ -830,6 +869,7 @@ class Releaser: self.executer.run(["cmake", "--build", str(build_path), "--verbose", "--config", build_type], cwd=build_path, env=new_env) with self.section_printer.group(f"Install MinGW {triplet} (CMake)"): self.executer.run(["cmake", "--install", str(build_path)], cwd=build_path, env=new_env) + self.verify_mingw_library(triplet=ARCH_TO_TRIPLET[arch], path=install_path / "bin" / f"{self.project}.dll") archive_file_tree.add_directory_tree(arc_dir=arc_join(arc_root, triplet), path=install_path, time=self.arc_time) print("Recording arch-dependent extra files for MinGW development archive ...") @@ -857,22 +897,25 @@ class Releaser: self.artifacts["mingw-devel-tar-gz"] = tgz_path self.artifacts["mingw-devel-tar-xz"] = txz_path - def _detect_android_api(self, android_home: str) -> typing.Optional[int]: + def _detect_android_api(self, android_home: str) -> typing.Optional[AndroidApiVersion]: platform_dirs = list(Path(p) for p in glob.glob(f"{android_home}/platforms/android-*")) - re_platform = re.compile("android-([0-9]+)") - platform_versions = [] + re_platform = re.compile("^android-([0-9]+)(?:-ext([0-9]+))?$") + platform_versions: list[AndroidApiVersion] = [] for platform_dir in platform_dirs: logger.debug("Found Android Platform SDK: %s", platform_dir) + if not (platform_dir / "android.jar").is_file(): + logger.debug("Skipping SDK, missing android.jar") + continue if m:= re_platform.match(platform_dir.name): - platform_versions.append(int(m.group(1))) - platform_versions.sort() + platform_versions.append(AndroidApiVersion(name=platform_dir.name, ints=(int(m.group(1)), int(m.group(2) or 0)))) + platform_versions.sort(key=lambda v: v.ints) logger.info("Available platform versions: %s", platform_versions) - platform_versions = list(filter(lambda v: v >= self._android_api_minimum, platform_versions)) - logger.info("Valid platform versions (>=%d): %s", self._android_api_minimum, platform_versions) + platform_versions = list(filter(lambda v: v.ints >= self._android_api_minimum.ints, platform_versions)) + logger.info("Valid platform versions (>=%s): %s", self._android_api_minimum.ints, platform_versions) if not platform_versions: return None android_api = platform_versions[0] - logger.info("Selected API version %d", android_api) + logger.info("Selected API version %s", android_api) return android_api def _get_prefab_json_text(self) -> str: @@ -896,8 +939,19 @@ class Releaser: return json.dumps(module_json_dict, indent=4) @property - def _android_api_minimum(self): - return self.release_info["android"]["api-minimum"] + def _android_api_minimum(self) -> AndroidApiVersion: + value = self.release_info["android"]["api-minimum"] + if isinstance(value, int): + ints = (value, ) + elif isinstance(value, str): + ints = tuple(split(".")) + else: + raise ValueError("Invalid android.api-minimum: must be X or X.Y") + match len(ints): + case 1: name = f"android-{ints[0]}" + case 2: name = f"android-{ints[0]}-ext-{ints[1]}" + case _: raise ValueError("Invalid android.api-minimum: must be X or X.Y") + return AndroidApiVersion(name=name, ints=ints) @property def _android_api_target(self): @@ -910,7 +964,7 @@ class Releaser: def _get_prefab_abi_json_text(self, abi: str, cpp: bool, shared: bool) -> str: abi_json_dict = { "abi": abi, - "api": self._android_api_minimum, + "api": self._android_api_minimum.ints[0], "ndk": self._android_ndk_minimum, "stl": "c++_shared" if cpp else "none", "static": not shared, @@ -923,7 +977,7 @@ class Releaser: xmlns:android="http://schemas.android.com/apk/res/android" package="org.libsdl.android.{self.project}" android:versionCode="1" android:versionName="1.0"> - """) @@ -933,7 +987,8 @@ class Releaser: if not cmake_toolchain_file.exists(): logger.error("CMake toolchain file does not exist (%s)", cmake_toolchain_file) raise SystemExit(1) - aar_path = self.dist_path / f"{self.project}-{self.version}.aar" + aar_path = self.root / "build-android" / f"{self.project}-{self.version}.aar" + android_dist_path = self.dist_path / f"{self.project}-devel-{self.version}-android.zip" android_abis = self.release_info["android"]["abis"] java_jars_added = False module_data_added = False @@ -941,16 +996,27 @@ class Releaser: shutil.rmtree(android_deps_path, ignore_errors=True) for dep, depinfo in self.release_info["android"].get("dependencies", {}).items(): - android_aar = self.deps_path / glob.glob(depinfo["artifact"], root_dir=self.deps_path)[0] - with self.section_printer.group(f"Extracting Android dependency {dep} ({android_aar.name})"): - self.executer.run([sys.executable, str(android_aar), "-o", str(android_deps_path)]) + dep_devel_zip = self.deps_path / glob.glob(depinfo["artifact"], root_dir=self.deps_path)[0] + + dep_extract_path = self.deps_path / f"extract/android/{dep}" + shutil.rmtree(dep_extract_path, ignore_errors=True) + dep_extract_path.mkdir(parents=True, exist_ok=True) + + with self.section_printer.group(f"Extracting Android dependency {dep} ({dep_devel_zip})"): + with zipfile.ZipFile(dep_devel_zip, "r") as zf: + zf.extractall(dep_extract_path) + + dep_devel_aar = dep_extract_path / glob.glob("*.aar", root_dir=dep_extract_path)[0] + self.executer.run([sys.executable, str(dep_devel_aar), "-o", str(android_deps_path)]) for module_name, module_info in self.release_info["android"]["modules"].items(): assert "type" in module_info and module_info["type"] in ("interface", "library"), f"module {module_name} must have a valid type" - archive_file_tree = ArchiveFileTree() + aar_file_tree = ArchiveFileTree() + android_devel_file_tree = ArchiveFileTree() for android_abi in android_abis: + extra_link_options = ANDROID_ABI_EXTRA_LINK_OPTIONS.get(android_abi, "") with self.section_printer.group(f"Building for Android {android_api} {android_abi}"): build_dir = self.root / "build-android" / f"{android_abi}-build" install_dir = self.root / "install-android" / f"{android_abi}-install" @@ -961,8 +1027,11 @@ class Releaser: "cmake", "-S", str(self.root), "-B", str(build_dir), - f'''-DCMAKE_C_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''', - f'''-DCMAKE_CXX_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''', + # NDK 21e does not support -ffile-prefix-map + # f'''-DCMAKE_C_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''', + # f'''-DCMAKE_CXX_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''', + f"-DCMAKE_EXE_LINKER_FLAGS={extra_link_options}", + f"-DCMAKE_SHARED_LINKER_FLAGS={extra_link_options}", f"-DCMAKE_TOOLCHAIN_FILE={cmake_toolchain_file}", f"-DCMAKE_PREFIX_PATH={str(android_deps_path)}", f"-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH", @@ -999,20 +1068,20 @@ class Releaser: assert library.suffix in (".so", ".a") assert library.is_file(), f"CMake should have built library '{library}' for module {module_name}" arcdir_prefab_libs = f"{arcdir_prefab_module}/libs/android.{android_abi}" - archive_file_tree.add_file(NodeInArchive.from_fs(arcpath=f"{arcdir_prefab_libs}/{library.name}", path=library, time=self.arc_time)) - archive_file_tree.add_file(NodeInArchive.from_text(arcpath=f"{arcdir_prefab_libs}/abi.json", text=self._get_prefab_abi_json_text(abi=android_abi, cpp=False, shared=library.suffix == ".so"), time=self.arc_time)) + aar_file_tree.add_file(NodeInArchive.from_fs(arcpath=f"{arcdir_prefab_libs}/{library.name}", path=library, time=self.arc_time)) + aar_file_tree.add_file(NodeInArchive.from_text(arcpath=f"{arcdir_prefab_libs}/abi.json", text=self._get_prefab_abi_json_text(abi=android_abi, cpp=False, shared=library.suffix == ".so"), time=self.arc_time)) if not module_data_added: library_name = None if module_info["type"] == "library": library_name = Path(module_info["library"]).stem.removeprefix("lib") export_libraries = module_info.get("export-libraries", []) - archive_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_module, "module.json"), text=self._get_prefab_module_json_text(library_name=library_name, export_libraries=export_libraries), time=self.arc_time)) + aar_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_module, "module.json"), text=self._get_prefab_module_json_text(library_name=library_name, export_libraries=export_libraries), time=self.arc_time)) arcdir_prefab_include = f"prefab/modules/{module_name}/include" if "includes" in module_info: - archive_file_tree.add_file_mapping(arc_dir=arcdir_prefab_include, file_mapping=module_info["includes"], file_mapping_root=install_dir, context=self.get_context(), time=self.arc_time) + aar_file_tree.add_file_mapping(arc_dir=arcdir_prefab_include, file_mapping=module_info["includes"], file_mapping_root=install_dir, context=self.get_context(), time=self.arc_time) else: - archive_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_include, ".keep"), text="\n", time=self.arc_time)) + aar_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_include, ".keep"), text="\n", time=self.arc_time)) module_data_added = True if not java_jars_added: @@ -1025,21 +1094,28 @@ class Releaser: assert sources_jar_path.is_file(), f"CMake should have archived the java sources into a JAR ({sources_jar_path})" assert doc_jar_path.is_file(), f"CMake should have archived javadoc into a JAR ({doc_jar_path})" - archive_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes.jar", path=classes_jar_path, time=self.arc_time)) - archive_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-sources.jar", path=sources_jar_path, time=self.arc_time)) - archive_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-doc.jar", path=doc_jar_path, time=self.arc_time)) + aar_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes.jar", path=classes_jar_path, time=self.arc_time)) + aar_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-sources.jar", path=sources_jar_path, time=self.arc_time)) + aar_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-doc.jar", path=doc_jar_path, time=self.arc_time)) assert ("jars" in self.release_info["android"] and java_jars_added) or "jars" not in self.release_info["android"], "Must have archived java JAR archives" - archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["android"].get("files", {}), file_mapping_root=self.root, context=self.get_context(), time=self.arc_time) + aar_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["android"]["aar-files"], file_mapping_root=self.root, context=self.get_context(), time=self.arc_time) - archive_file_tree.add_file(NodeInArchive.from_text(arcpath="prefab/prefab.json", text=self._get_prefab_json_text(), time=self.arc_time)) - archive_file_tree.add_file(NodeInArchive.from_text(arcpath="AndroidManifest.xml", text=self._get_android_manifest_text(), time=self.arc_time)) + aar_file_tree.add_file(NodeInArchive.from_text(arcpath="prefab/prefab.json", text=self._get_prefab_json_text(), time=self.arc_time)) + aar_file_tree.add_file(NodeInArchive.from_text(arcpath="AndroidManifest.xml", text=self._get_android_manifest_text(), time=self.arc_time)) with Archiver(zip_path=aar_path) as archiver: - archive_file_tree.add_to_archiver(archive_base="", archiver=archiver) + aar_file_tree.add_to_archiver(archive_base="", archiver=archiver) archiver.add_git_hash(arcdir="", commit=self.commit, time=self.arc_time) - self.artifacts[f"android-aar"] = aar_path + + android_devel_file_tree.add_file(NodeInArchive.from_fs(arcpath=aar_path.name, path=aar_path)) + android_devel_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["android"]["files"], file_mapping_root=self.root, context=self.get_context(), time=self.arc_time) + with Archiver(zip_path=android_dist_path) as archiver: + android_devel_file_tree.add_to_archiver(archive_base="", archiver=archiver) + archiver.add_git_hash(arcdir="", commit=self.commit, time=self.arc_time) + + self.artifacts[f"android-aar"] = android_dist_path def download_dependencies(self): shutil.rmtree(self.deps_path, ignore_errors=True) @@ -1076,7 +1152,7 @@ class Releaser: assert len(msvc_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {msvc_matches}" if "android" in self.release_info: android_matches = glob.glob(self.release_info["android"]["dependencies"][dep]["artifact"], root_dir=self.deps_path) - assert len(android_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {msvc_matches}" + assert len(android_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {android_matches}" @staticmethod def _arch_to_vs_platform(arch: str, configuration: str="Release") -> VsArchPlatformConfig: @@ -1202,6 +1278,10 @@ class Releaser: platform_context = self.get_context(extra_context=arch_platform.extra_context()) build_type = "Release" + extra_context = { + "ARCH": arch_platform.arch, + "PLATFORM": arch_platform.platform, + } built_paths = set(install_path / configure_text(f, context=platform_context) for file_mapping in (self.release_info["msvc"]["cmake"]["files-lib"], self.release_info["msvc"]["cmake"]["files-devel"]) for files_list in file_mapping.values() for f in files_list) logger.info("CMake builds these files, to be included in the package: %s", built_paths) @@ -1252,7 +1332,7 @@ class Releaser: logger.info("Collecting files...") archive_file_tree = ArchiveFileTree() archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["cmake"]["files-lib"], file_mapping_root=install_path, context=platform_context, time=self.arc_time) - archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["files-lib"], file_mapping_root=self.root, context=self.get_context(), time=self.arc_time) + archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["files-lib"], file_mapping_root=self.root, context=self.get_context(extra_context=extra_context), time=self.arc_time) logger.info("Creating %s", zip_path) with Archiver(zip_path=zip_path) as archiver: @@ -1316,7 +1396,7 @@ def main(argv=None) -> int: parser.add_argument("--actions", choices=["download", "source", "android", "mingw", "msvc", "dmg"], required=True, nargs="+", dest="actions", help="What to do?") parser.set_defaults(loglevel=logging.INFO) parser.add_argument('--vs-year', dest="vs_year", help="Visual Studio year") - parser.add_argument('--android-api', type=int, dest="android_api", help="Android API version") + parser.add_argument('--android-api', dest="android_api", help="Android API version") parser.add_argument('--android-home', dest="android_home", default=os.environ.get("ANDROID_HOME"), help="Android Home folder") parser.add_argument('--android-ndk-home', dest="android_ndk_home", default=os.environ.get("ANDROID_NDK_HOME"), help="Android NDK Home folder") parser.add_argument('--cmake-generator', dest="cmake_generator", default="Ninja", help="CMake Generator") @@ -1443,14 +1523,27 @@ def main(argv=None) -> int: if args.android_api is None: with section_printer.group("Detect Android APIS"): args.android_api = releaser._detect_android_api(android_home=args.android_home) - if args.android_api is None or not (Path(args.android_home) / f"platforms/android-{args.android_api}").is_dir(): + else: + try: + android_api_ints = tuple(int(v) for v in args.android_api.split(".")) + match len(android_api_ints): + case 1: android_api_name = f"android-{android_api_ints[0]}" + case 2: android_api_name = f"android-{android_api_ints[0]}-ext-{android_api_ints[1]}" + case _: raise ValueError + except ValueError: + logger.error("Invalid --android-api, must be a 'X' or 'X.Y' version") + args.android_api = AndroidApiVersion(ints=android_api_ints, name=android_api_name) + if args.android_api is None: parser.error("Invalid --android-api, and/or could not be detected") + android_api_path = Path(args.android_home) / f"platforms/{args.android_api.name}" + if not android_api_path.is_dir(): + logger.warning(f"Android API directory does not exist ({android_api_path})") with section_printer.group("Android arguments"): print(f"android_home = {args.android_home}") print(f"android_ndk_home = {args.android_ndk_home}") print(f"android_api = {args.android_api}") releaser.create_android_archives( - android_api=args.android_api, + android_api=args.android_api.ints[0], android_home=args.android_home, android_ndk_home=args.android_ndk_home, )