Refactor gendynapi.py with the final goal to make it re-usable

This commit is contained in:
Anonymous Maarten
2024-10-04 18:51:05 +02:00
committed by Anonymous Maarten
parent 173c168ff4
commit b641c2a0db

View File

@@ -24,60 +24,68 @@
# output looks sane (git diff, it adds to existing files), and commit it.
# It keeps the dynamic API jump table operating correctly.
#
# OS-specific API:
# Platform-specific API:
# After running the script, you have to manually add #ifdef SDL_PLATFORM_WIN32
# or similar around the function in 'SDL_dynapi_procs.h'
# or similar around the function in 'SDL_dynapi_procs.h'.
#
import argparse
import dataclasses
import json
import logging
import os
import pathlib
from pathlib import Path
import pprint
import re
SDL_ROOT = pathlib.Path(__file__).resolve().parents[2]
SDL_ROOT = Path(__file__).resolve().parents[2]
SDL_INCLUDE_DIR = SDL_ROOT / "include/SDL3"
SDL_DYNAPI_PROCS_H = SDL_ROOT / "src/dynapi/SDL_dynapi_procs.h"
SDL_DYNAPI_OVERRIDES_H = SDL_ROOT / "src/dynapi/SDL_dynapi_overrides.h"
SDL_DYNAPI_SYM = SDL_ROOT / "src/dynapi/SDL_dynapi.sym"
full_API = []
RE_EXTERN_C = re.compile(r'.*extern[ "]*C[ "].*')
RE_COMMENT_REMOVE_CONTENT = re.compile(r'\/\*.*\*/')
RE_PARSING_FUNCTION = re.compile(r'(.*SDLCALL[^\(\)]*) ([a-zA-Z0-9_]+) *\((.*)\) *;.*')
#eg:
# void (SDLCALL *callback)(void*, int)
# \1(\2)\3
RE_PARSING_CALLBACK = re.compile(r'([^\(\)]*)\(([^\(\)]+)\)(.*)')
def main():
logger = logging.getLogger(__name__)
# Parse 'sdl_dynapi_procs_h' file to find existing functions
existing_procs = find_existing_procs()
# Get list of SDL headers
sdl_list_includes = get_header_list()
@dataclasses.dataclass(frozen=True)
class SdlProcedure:
retval: str
name: str
parameter: list[str]
parameter_name: list[str]
header: str
comment: str
reg_externC = re.compile(r'.*extern[ "]*C[ "].*')
reg_comment_remove_content = re.compile(r'\/\*.*\*/')
reg_parsing_function = re.compile(r'(.*SDLCALL[^\(\)]*) ([a-zA-Z0-9_]+) *\((.*)\) *;.*')
@property
def variadic(self) -> bool:
return "..." in self.parameter
#eg:
# void (SDLCALL *callback)(void*, int)
# \1(\2)\3
reg_parsing_callback = re.compile(r'([^\(\)]*)\(([^\(\)]+)\)(.*)')
for filename in sdl_list_includes:
if args.debug:
print("Parse header: %s" % filename)
def parse_header(header_path: Path) -> list[SdlProcedure]:
logger.debug("Parse header: %s", header_path)
input = open(filename)
header_procedures = []
parsing_function = False
current_func = ""
parsing_comment = False
current_comment = ""
parsing_function = False
current_func = ""
parsing_comment = False
current_comment = ""
ignore_wiki_documentation = False
ignore_wiki_documentation = False
for line in input:
with header_path.open() as f:
for line in f:
# Skip lines if we're in a wiki documentation block.
if ignore_wiki_documentation:
@@ -95,13 +103,13 @@ def main():
continue
# Discard "extern C" line
match = reg_externC.match(line)
match = RE_EXTERN_C.match(line)
if match:
continue
# Remove one line comment // ...
# eg: extern SDL_DECLSPEC SDL_hid_device * SDLCALL SDL_hid_open_path(const char *path, int bExclusive /* = false */);
line = reg_comment_remove_content.sub('', line)
# eg: extern SDL_DECLSPEC SDL_hid_device * SDLCALL SDL_hid_open_path(const char *path, int bExclusive /* = false */)
line = RE_COMMENT_REMOVE_CONTENT.sub('', line)
# Get the comment block /* ... */ across several lines
match_start = "/*" in line
@@ -131,14 +139,14 @@ def main():
continue
# Start grabbing the new function
current_func = line.strip()
parsing_function = True;
parsing_function = True
# If it contains ';', then the function is complete
if ";" not in current_func:
continue
# Got function/comment, reset vars
parsing_function = False;
parsing_function = False
func = current_func
comment = current_comment
current_func = ""
@@ -146,47 +154,48 @@ def main():
# Discard if it doesn't contain 'SDLCALL'
if "SDLCALL" not in func:
if args.debug:
print(" Discard, doesn't have SDLCALL: " + func)
logger.debug(" Discard, doesn't have SDLCALL: %r", func)
continue
# Discard if it contains 'SDLMAIN_DECLSPEC' (these are not SDL symbols).
if "SDLMAIN_DECLSPEC" in func:
if args.debug:
print(" Discard, has SDLMAIN_DECLSPEC: " + func)
logger.debug(" Discard, has SDLMAIN_DECLSPEC: %r", func)
continue
if args.debug:
print(" Raw data: " + func);
logger.debug("Raw data: %r", func)
# Replace unusual stuff...
func = func.replace(" SDL_PRINTF_VARARG_FUNC(1)", "");
func = func.replace(" SDL_PRINTF_VARARG_FUNC(2)", "");
func = func.replace(" SDL_PRINTF_VARARG_FUNC(3)", "");
func = func.replace(" SDL_PRINTF_VARARG_FUNCV(1)", "");
func = func.replace(" SDL_PRINTF_VARARG_FUNCV(2)", "");
func = func.replace(" SDL_PRINTF_VARARG_FUNCV(3)", "");
func = func.replace(" SDL_WPRINTF_VARARG_FUNC(3)", "");
func = func.replace(" SDL_WPRINTF_VARARG_FUNCV(3)", "");
func = func.replace(" SDL_SCANF_VARARG_FUNC(2)", "");
func = func.replace(" SDL_SCANF_VARARG_FUNCV(2)", "");
func = func.replace(" SDL_ANALYZER_NORETURN", "");
func = func.replace(" SDL_MALLOC", "");
func = func.replace(" SDL_ALLOC_SIZE2(1, 2)", "");
func = func.replace(" SDL_ALLOC_SIZE(2)", "");
func = re.sub(r" SDL_ACQUIRE\(.*\)", "", func);
func = re.sub(r" SDL_ACQUIRE_SHARED\(.*\)", "", func);
func = re.sub(r" SDL_TRY_ACQUIRE\(.*\)", "", func);
func = re.sub(r" SDL_TRY_ACQUIRE_SHARED\(.*\)", "", func);
func = re.sub(r" SDL_RELEASE\(.*\)", "", func);
func = re.sub(r" SDL_RELEASE_SHARED\(.*\)", "", func);
func = re.sub(r" SDL_RELEASE_GENERIC\(.*\)", "", func);
func = func.replace(" SDL_PRINTF_VARARG_FUNC(1)", "")
func = func.replace(" SDL_PRINTF_VARARG_FUNC(2)", "")
func = func.replace(" SDL_PRINTF_VARARG_FUNC(3)", "")
func = func.replace(" SDL_PRINTF_VARARG_FUNCV(1)", "")
func = func.replace(" SDL_PRINTF_VARARG_FUNCV(2)", "")
func = func.replace(" SDL_PRINTF_VARARG_FUNCV(3)", "")
func = func.replace(" SDL_WPRINTF_VARARG_FUNC(3)", "")
func = func.replace(" SDL_WPRINTF_VARARG_FUNCV(3)", "")
func = func.replace(" SDL_SCANF_VARARG_FUNC(2)", "")
func = func.replace(" SDL_SCANF_VARARG_FUNCV(2)", "")
func = func.replace(" SDL_ANALYZER_NORETURN", "")
func = func.replace(" SDL_MALLOC", "")
func = func.replace(" SDL_ALLOC_SIZE2(1, 2)", "")
func = func.replace(" SDL_ALLOC_SIZE(2)", "")
func = re.sub(r" SDL_ACQUIRE\(.*\)", "", func)
func = re.sub(r" SDL_ACQUIRE_SHARED\(.*\)", "", func)
func = re.sub(r" SDL_TRY_ACQUIRE\(.*\)", "", func)
func = re.sub(r" SDL_TRY_ACQUIRE_SHARED\(.*\)", "", func)
func = re.sub(r" SDL_RELEASE\(.*\)", "", func)
func = re.sub(r" SDL_RELEASE_SHARED\(.*\)", "", func)
func = re.sub(r" SDL_RELEASE_GENERIC\(.*\)", "", func)
func = re.sub(r"([ (),])(SDL_IN_BYTECAP\([^)]*\))", r"\1", func)
func = re.sub(r"([ (),])(SDL_OUT_BYTECAP\([^)]*\))", r"\1", func)
func = re.sub(r"([ (),])(SDL_INOUT_Z_CAP\([^)]*\))", r"\1", func)
func = re.sub(r"([ (),])(SDL_OUT_Z_CAP\([^)]*\))", r"\1", func)
# Should be a valid function here
match = reg_parsing_function.match(func)
match = RE_PARSING_FUNCTION.match(func)
if not match:
print("Cannot parse: "+ func)
exit(-1)
logger.error("Cannot parse: %s", func)
raise ValueError(func)
func_ret = match.group(1)
func_name = match.group(2)
@@ -198,11 +207,8 @@ def main():
func_ret = func_ret.replace('extern', ' ')
func_ret = func_ret.replace('SDLCALL', ' ')
func_ret = func_ret.replace('SDL_DECLSPEC', ' ')
func_ret, _ = re.subn('([ ]{2,})', ' ', func_ret)
# Remove trailing spaces in front of '*'
tmp = ""
while func_ret != tmp:
tmp = func_ret
func_ret = func_ret.replace(' ', ' ')
func_ret = func_ret.replace(' *', '*')
func_ret = func_ret.strip()
@@ -246,10 +252,10 @@ def main():
# parameter is a callback
if '(' in t:
match = reg_parsing_callback.match(t)
match = RE_PARSING_CALLBACK.match(t)
if not match:
print("cannot parse callback: " + t);
exit(-1)
logger.error("cannot parse callback: %s", t)
raise ValueError(t)
a = match.group(1).strip()
b = match.group(2).strip()
c = match.group(3).strip()
@@ -257,7 +263,7 @@ def main():
try:
(param_type, param_name) = b.rsplit('*', 1)
except:
param_type = t;
param_type = t
param_name = "param_name_not_specified"
# bug rsplit ??
@@ -281,7 +287,7 @@ def main():
try:
(param_type, param_name) = t.rsplit('*', 1)
except:
param_type = t;
param_type = t
param_name = "param_name_not_specified"
# bug rsplit ??
@@ -305,7 +311,7 @@ def main():
try:
(param_type, param_name) = t.rsplit(' ', 1)
except:
param_type = t;
param_type = t
param_name = "param_name_not_specified"
val = param_type.strip() + " REWRITE_NAME"
@@ -317,268 +323,220 @@ def main():
func_param_type.append(val)
func_param_name.append(param_name.strip())
new_proc = {}
# Return value type
new_proc['retval'] = func_ret
# List of parameters (type + anonymized param name 'REWRITE_NAME')
new_proc['parameter'] = func_param_type
# Real parameter name, or 'param_name_not_specified'
new_proc['parameter_name'] = func_param_name
# Function name
new_proc['name'] = func_name
# Header file
new_proc['header'] = os.path.basename(filename)
# Function comment
new_proc['comment'] = comment
new_proc = SdlProcedure(
retval=func_ret, # Return value type
name=func_name, # Function name
comment=comment, # Function comment
header=header_path.name, # Header file
parameter=func_param_type, # List of parameters (type + anonymized param name 'REWRITE_NAME')
parameter_name=func_param_name, # Real parameter name, or 'param_name_not_specified'
)
full_API.append(new_proc)
header_procedures.append(new_proc)
if args.debug:
pprint.pprint(new_proc);
print("\n")
if logger.getEffectiveLevel() <= logging.DEBUG:
logger.debug("%s", pprint.pformat(new_proc))
if func_name not in existing_procs:
print("NEW " + func)
add_dyn_api(new_proc)
return header_procedures
# For-End line in input
input.close()
# For-End parsing all files of sdl_list_includes
# Dump API into a json file
full_API_json()
# Check comment formatting
check_comment();
# Dump API into a json file
def full_API_json():
if args.dump:
filename = 'sdl.json'
with open(filename, 'w', newline='') as f:
json.dump(full_API, f, indent=4, sort_keys=True)
print("dump API to '%s'" % filename);
def full_API_json(path: Path, procedures: list[SdlProcedure]):
with path.open('w', newline='') as f:
json.dump([dataclasses.asdict(proc) for proc in procedures], f, indent=4, sort_keys=True)
logger.info("dump API to '%s'", path)
class CallOnce:
def __init__(self, cb):
self._cb = cb
self._called = False
def __call__(self, *args, **kwargs):
if self._called:
return
self._called = True
self._cb(*args, **kwargs)
# Check public function comments are correct
def check_comment_header():
if not check_comment_header.done:
check_comment_header.done = True
print("")
print("Please fix following warning(s):")
print("-------------------------------")
def print_check_comment_header():
logger.warning("")
logger.warning("Please fix following warning(s):")
logger.warning("--------------------------------")
def check_comment():
def check_documentations(procedures: list[SdlProcedure]) -> None:
check_comment_header.done = False
check_comment_header = CallOnce(print_check_comment_header)
warning_header_printed = False
# Check \param
for i in full_API:
comment = i['comment']
name = i['name']
retval = i['retval']
header = i['header']
expected = len(i['parameter'])
for proc in procedures:
expected = len(proc.parameter)
if expected == 1:
if i['parameter'][0] == 'void':
expected = 0;
count = comment.count("\\param")
if proc.parameter[0] == 'void':
expected = 0
count = proc.comment.count("\\param")
if count != expected:
# skip SDL_stdinc.h
if header != 'SDL_stdinc.h':
if proc.header != 'SDL_stdinc.h':
# Warning mismatch \param and function prototype
check_comment_header()
print(" In file %s: function %s() has %d '\\param' but expected %d" % (header, name, count, expected));
logger.warning(" In file %s: function %s() has %d '\\param' but expected %d", proc.header, proc.name, count, expected)
# Warning check \param uses the correct parameter name
# skip SDL_stdinc.h
if header != 'SDL_stdinc.h':
parameter_name = i['parameter_name']
for n in parameter_name:
if n != "" and "\\param " + n not in comment and "\\param[out] " + n not in comment:
if proc.header != 'SDL_stdinc.h':
for n in proc.parameter_name:
if n != "" and "\\param " + n not in proc.comment and "\\param[out] " + n not in proc.comment:
check_comment_header()
print(" In file %s: function %s() missing '\\param %s'" % (header, name, n));
logger.warning(" In file %s: function %s() missing '\\param %s'", proc.header, proc.name, n)
# Check \returns
for i in full_API:
comment = i['comment']
name = i['name']
retval = i['retval']
header = i['header']
for proc in procedures:
expected = 1
if retval == 'void':
expected = 0;
if proc.retval == 'void':
expected = 0
count = comment.count("\\returns")
count = proc.comment.count("\\returns")
if count != expected:
# skip SDL_stdinc.h
if header != 'SDL_stdinc.h':
if proc.header != 'SDL_stdinc.h':
# Warning mismatch \param and function prototype
check_comment_header()
print(" In file %s: function %s() has %d '\\returns' but expected %d" % (header, name, count, expected));
logger.warning(" In file %s: function %s() has %d '\\returns' but expected %d" % (proc.header, proc.name, count, expected))
# Check \since
for i in full_API:
comment = i['comment']
name = i['name']
retval = i['retval']
header = i['header']
for proc in procedures:
expected = 1
count = comment.count("\\since")
count = proc.comment.count("\\since")
if count != expected:
# skip SDL_stdinc.h
if header != 'SDL_stdinc.h':
if proc.header != 'SDL_stdinc.h':
# Warning mismatch \param and function prototype
check_comment_header()
print(" In file %s: function %s() has %d '\\since' but expected %d" % (header, name, count, expected));
logger.warning(" In file %s: function %s() has %d '\\since' but expected %d" % (proc.header, proc.name, count, expected))
# Parse 'sdl_dynapi_procs_h' file to find existing functions
def find_existing_procs():
def find_existing_proc_names() -> list[str]:
reg = re.compile(r'SDL_DYNAPI_PROC\([^,]*,([^,]*),.*\)')
ret = []
input = open(SDL_DYNAPI_PROCS_H)
for line in input:
match = reg.match(line)
if not match:
continue
existing_func = match.group(1)
ret.append(existing_func);
# print(existing_func)
input.close()
with SDL_DYNAPI_PROCS_H.open() as f:
for line in f:
match = reg.match(line)
if not match:
continue
existing_func = match.group(1)
ret.append(existing_func)
return ret
# Get list of SDL headers
def get_header_list():
reg = re.compile(r'^.*\.h$')
def get_header_list() -> list[Path]:
ret = []
tmp = os.listdir(SDL_INCLUDE_DIR)
for f in tmp:
for f in SDL_INCLUDE_DIR.iterdir():
# Only *.h files
match = reg.match(f)
if not match:
if args.debug:
print("Skip %s" % f)
continue
ret.append(SDL_INCLUDE_DIR / f)
if f.is_file() and f.suffix == ".h":
ret.append(f)
else:
logger.debug("Skip %s", f)
return ret
# Write the new API in files: _procs.h _overrivides.h and .sym
def add_dyn_api(proc):
func_name = proc['name']
func_ret = proc['retval']
func_argtype = proc['parameter']
def add_dyn_api(proc: SdlProcedure) -> None:
decl_args: list[str] = []
call_args = []
for i, argtype in enumerate(proc.parameter):
# Special case, void has no parameter name
if argtype == "void":
assert len(decl_args) == 0
assert len(proc.parameter) == 1
decl_args.append("void")
continue
# Var name: a, b, c, ...
varname = chr(ord('a') + i)
decl_args.append(argtype.replace("REWRITE_NAME", varname))
if argtype != "...":
call_args.append(varname)
macro_args = (
proc.retval,
proc.name,
"({})".format(",".join(decl_args)),
"({})".format(",".join(call_args)),
"" if proc.retval == "void" else "return",
)
# File: SDL_dynapi_procs.h
#
# Add at last
# SDL_DYNAPI_PROC(SDL_EGLConfig,SDL_EGL_GetCurrentConfig,(void),(),return)
f = open(SDL_DYNAPI_PROCS_H, "a", newline="")
dyn_proc = "SDL_DYNAPI_PROC(" + func_ret + "," + func_name + ",("
i = ord('a')
remove_last = False
for argtype in func_argtype:
# Special case, void has no parameter name
if argtype == "void":
dyn_proc += "void"
continue
# Var name: a, b, c, ...
varname = chr(i)
i += 1
tmp = argtype.replace("REWRITE_NAME", varname)
dyn_proc += tmp + ", "
remove_last = True
# remove last 2 char ', '
if remove_last:
dyn_proc = dyn_proc[:-1]
dyn_proc = dyn_proc[:-1]
dyn_proc += "),("
i = ord('a')
remove_last = False
for argtype in func_argtype:
# Special case, void has no parameter name
if argtype == "void":
continue
# Special case, '...' has no parameter name
if argtype == "...":
continue
# Var name: a, b, c, ...
varname = chr(i)
i += 1
dyn_proc += varname + ","
remove_last = True
# remove last char ','
if remove_last:
dyn_proc = dyn_proc[:-1]
dyn_proc += "),"
if func_ret != "void":
dyn_proc += "return"
dyn_proc += ")"
f.write(dyn_proc + "\n")
f.close()
with SDL_DYNAPI_PROCS_H.open("a", newline="") as f:
if proc.variadic:
f.write("#ifndef SDL_DYNAPI_PROC_NO_VARARGS\n")
f.write(f"SDL_DYNAPI_PROC({','.join(macro_args)})\n")
if proc.variadic:
f.write("#endif\n")
# File: SDL_dynapi_overrides.h
#
# Add at last
# "#define SDL_DelayNS SDL_DelayNS_REAL
f = open(SDL_DYNAPI_OVERRIDES_H, "a", newline="")
f.write("#define " + func_name + " " + func_name + "_REAL\n")
f.write(f"#define {proc.name} {proc.name}_REAL\n")
f.close()
# File: SDL_dynapi.sym
#
# Add before "extra symbols go here" line
input = open(SDL_DYNAPI_SYM)
new_input = []
for line in input:
if "extra symbols go here" in line:
new_input.append(" " + func_name + ";\n")
new_input.append(line)
input.close()
f = open(SDL_DYNAPI_SYM, 'w', newline='')
for line in new_input:
f.write(line)
f.close()
with SDL_DYNAPI_SYM.open() as f:
new_input = []
for line in f:
if "extra symbols go here" in line:
new_input.append(f" {proc.name};\n")
new_input.append(line)
with SDL_DYNAPI_SYM.open('w', newline='') as f:
for line in new_input:
f.write(line)
def main():
parser = argparse.ArgumentParser()
parser.set_defaults(loglevel=logging.INFO)
parser.add_argument('--dump', nargs='?', default=None, const="sdl.json", metavar="JSON", help='output all SDL API into a .json file')
parser.add_argument('--debug', action='store_const', const=logging.DEBUG, dest="loglevel", help='add debug traces')
args = parser.parse_args()
logging.basicConfig(level=args.loglevel, format='[%(levelname)s] %(message)s')
# Get list of SDL headers
sdl_list_includes = get_header_list()
procedures = []
for filename in sdl_list_includes:
header_procedures = parse_header(filename)
procedures.extend(header_procedures)
# Parse 'sdl_dynapi_procs_h' file to find existing functions
existing_proc_names = find_existing_proc_names()
for procedure in procedures:
if procedure.name not in existing_proc_names:
logger.info("NEW %s", procedure.name)
add_dyn_api(procedure)
if args.dump:
# Dump API into a json file
full_API_json(path=Path(args.dump), procedures=procedures)
# Check comment formatting
check_documentations(procedures)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--dump', help='output all SDL API into a .json file', action='store_true')
parser.add_argument('--debug', help='add debug traces', action='store_true')
args = parser.parse_args()
try:
main()
except Exception as e:
print(e)
exit(-1)
print("done!")
exit(0)
raise SystemExit(main())