From bd9aac8cba0c55cece0f937722188097008a7dc1 Mon Sep 17 00:00:00 2001 From: cytopia Date: Sun, 27 Nov 2022 16:04:32 +0100 Subject: [PATCH] Nicyfy get-modules.py --- .ansible/bin/Makefile | 27 ++++++ .ansible/bin/get-modules.py | 167 ++++++++++++++++++++++++------------ 2 files changed, 141 insertions(+), 53 deletions(-) create mode 100644 .ansible/bin/Makefile diff --git a/.ansible/bin/Makefile b/.ansible/bin/Makefile new file mode 100644 index 0000000..74c1a89 --- /dev/null +++ b/.ansible/bin/Makefile @@ -0,0 +1,27 @@ +ifneq (,) +.error This Makefile requires GNU Make. +endif + +# Ensure additional Makefiles are present +MAKEFILES = Makefile.python +$(MAKEFILES): URL=https://raw.githubusercontent.com/devilbox/makefiles/master/$(@) +$(MAKEFILES): + @if ! (curl --fail -sS -o $(@) $(URL) || wget -O $(@) $(URL)); then \ + echo "Error, curl or wget required."; \ + echo "Exiting."; \ + false; \ + fi +include $(MAKEFILES) + + +# ------------------------------------------------------------------------------------------------- +# Default configuration +# ------------------------------------------------------------------------------------------------- +MYPY_ARGS = --strict --disable-error-code no-any-return + +PYLINT_DIR = get-modules.py +PYLINT_PIP_PKGS = yamllint +PYLINT_ARGS = --disable=invalid-name get-modules.py + +BLACK_LINT_ARGS = -l 100 --check --diff +BLACK_FIX_ARGS = -l 100 diff --git a/.ansible/bin/get-modules.py b/.ansible/bin/get-modules.py index f8f08d9..94d33b3 100755 --- a/.ansible/bin/get-modules.py +++ b/.ansible/bin/get-modules.py @@ -1,65 +1,92 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Generate Ansible group_vars from module definition.""" import os -import yaml -import copy +import sys from collections import OrderedDict +from typing import Dict, List, Any +import yaml -SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) -REPOSITORY_PATH = os.path.dirname(os.path.dirname(SCRIPT_PATH)) -PHP_MODULE_PATH = os.path.join(REPOSITORY_PATH, "php_modules") -GROUP_VARS_PATH = os.path.join(REPOSITORY_PATH, ".ansible", "group_vars", "all") - # -------------------------------------------------------------------------------------------------- -# PATH FUNCTIONS +# GLOBALS # -------------------------------------------------------------------------------------------------- -def get_repository_path() -> str: - """Returns the absolute repository directory path.""" - script_path = os.path.dirname(os.path.realpath(__file__)) - return os.path.dirname(os.path.dirname(script_path)) +SCRIPT_PATH = str(os.path.dirname(os.path.realpath(__file__))) +REPOSITORY_PATH = str(os.path.dirname(os.path.dirname(SCRIPT_PATH))) +PHP_MODULE_PATH = str(os.path.join(REPOSITORY_PATH, "php_modules")) +GROUP_VARS_PATH = str(os.path.join(REPOSITORY_PATH, ".ansible", "group_vars", "all")) -def get_module_path(repository_path: str) -> str: - """Returns the absolute PHP module directory path.""" - return os.path.join(repository_path, "php_modules") +# -------------------------------------------------------------------------------------------------- +# HELPER FUNCTIONS +# -------------------------------------------------------------------------------------------------- -def get_group_vars_path(repository_path: str) -> str: - """Returns the absolute mods group_vars directory path.""" - return os.path.join(repository_path, ".ansible", "group_vars", "all") +def get_el_by_name(items: List[Dict[str, Any]], name: str) -> Dict[str, Any]: + """Returns an element from a dict list by its 'name' key with given value.""" + for item in items: + if item["name"] == name: + return item + print("error, key name not found by value", name, "in list: ", items) + sys.exit(1) # -------------------------------------------------------------------------------------------------- # MODULE FUNCTIONS # -------------------------------------------------------------------------------------------------- -def get_module_options(module_dirname: str) -> dict[str, str]: + +def get_module_options(module_dirname: str) -> Dict[str, Any]: """Returns yaml dict options of a PHP module given by its absolute file path.""" - with open(os.path.join(PHP_MODULE_PATH, module_dirname, "options.yml")) as fp: + with open(os.path.join(PHP_MODULE_PATH, module_dirname, "options.yml"), encoding="utf8") as fp: data = yaml.safe_load(fp) return data -def get_module_dir_names() -> list[str]: +def get_module_build(module_dirname: str) -> Dict[str, Any]: + """Returns yaml dict build configuration of a PHP module given by its absolute file path.""" + with open(os.path.join(PHP_MODULE_PATH, module_dirname, "build.yml"), encoding="utf8") as fp: + data = yaml.safe_load(fp) + return data + + +def get_module_test(module_dirname: str) -> Dict[str, Any]: + """Returns yaml dict test configuration of a PHP module given by its absolute file path.""" + with open(os.path.join(PHP_MODULE_PATH, module_dirname, "test.yml"), encoding="utf8") as fp: + data = yaml.safe_load(fp) + return data + + +def get_modules() -> List[Dict[str, Any]]: """Returns a list of PHP module directory names.""" - directories = [] + modules = [] with os.scandir(PHP_MODULE_PATH) as it: for item in it: - if not item.name.startswith('.') and item.is_dir(): - directories.append(item.name) - return sorted(directories, key=str.lower) + if not item.name.startswith(".") and item.is_dir(): + data = get_module_options(item.name) + modules.append({"dir": item.name, "name": data["name"], "deps": data["requires"]}) + # Convert list of deps into dict(dir, name, deps) + items = [] + for module in modules: + if module["deps"]: + deps = [] + for dep in module["deps"]: + deps.append(get_el_by_name(modules, dep)) + module["deps"] = deps + items.append(module) + else: + items.append(module) + return sorted(items, key=lambda item: item["dir"]) -def get_module_dependency_tree(names: list[str]) -> dict(): +def get_module_dependency_tree(modules: List[Dict[str, Any]]) -> OrderedDict[str, Any]: """Returns dictionary of module dependency tree.""" - module_tree = OrderedDict() + module_tree = OrderedDict() # type: OrderedDict[str, Any] - for name in names: - data = get_module_options(name) - - mod_name = data["name"] - mod_deps = data["requires"] + for module in modules: + mod_name = module["name"] + mod_deps = module["deps"] module_tree[mod_name] = {} @@ -69,10 +96,10 @@ def get_module_dependency_tree(names: list[str]) -> dict(): return module_tree -def resolve_module_dependency_tree(tree): +def resolve_module_dependency_tree(tree: OrderedDict[str, Any]) -> List[str]: """Returns sorted list of resolved dependencies.""" resolved = [] - for key, value in tree.items(): + for key, _ in tree.items(): # Has dependenies if tree[key]: childs = resolve_module_dependency_tree(tree[key]) @@ -89,49 +116,83 @@ def resolve_module_dependency_tree(tree): # PRINT FUNCTIONS # -------------------------------------------------------------------------------------------------- -def print_dependency_tree(tree, lvl=0): + +def print_modules(modules: List[Dict[str, Any]]) -> None: + """Print directory modules.""" + for module in modules: + print(module["dir"] + "/") + print(" name:", module["name"]) + print(" deps:", end=" ") + for dep in module["deps"]: + print(dep["name"], end=", ") + print() + + +def print_dependency_tree(tree: Dict[str, Any], lvl: int = 0) -> None: + """Print dependency tree of modules.""" for key, value in tree.items(): - print(" "*lvl, "-", key) + print(" " * lvl, "-", key) if value: - print_dependency_tree(tree[key], lvl+2) + print_dependency_tree(tree[key], lvl + 2) + + +# -------------------------------------------------------------------------------------------------- +# WRITE ANSIBLE GROUP_VARS FUNCTIONS +# -------------------------------------------------------------------------------------------------- + + +def write_group_vars(module_names: List[str]) -> None: + """Write mods.yml group_vars for ansible.""" + group_vars = os.path.join(GROUP_VARS_PATH, "mods.yml") + with open(group_vars, "w", encoding="utf8") as fp: + fp.write("---\n\n") + fp.write("# Do not alter this file, it is autogenerated\n\n") + fp.write("modules:\n") + for name in module_names: + fp.write(" - " + name + "\n") # -------------------------------------------------------------------------------------------------- # MAIN FUNCTION # -------------------------------------------------------------------------------------------------- -def main(): + +def main() -> None: + """Main entrypoint.""" # Module directory names - directory_names = get_module_dir_names() + modules = get_modules() # Get modules in order of dependencies - module_tree = get_module_dependency_tree(directory_names) - modules = resolve_module_dependency_tree(module_tree) + module_tree = get_module_dependency_tree(modules) + names = resolve_module_dependency_tree(module_tree) - print("#", "-"*78) + print("#", "-" * 78) print("# Paths") - print("#", "-"*78) + print("#", "-" * 78) print("Repository: ", REPOSITORY_PATH) print("PHP Module: ", PHP_MODULE_PATH) print("Group Vars: ", GROUP_VARS_PATH) print() - print("#", "-"*78) + print("#", "-" * 78) print("# Module directories") - print("#", "-"*78) - print(directory_names) + print("#", "-" * 78) + print_modules(modules) print() - print("#", "-"*78) - print("# Dependency Tree") - print("#", "-"*78) + print("#", "-" * 78) + print("# Build Dependency Tree") + print("#", "-" * 78) print_dependency_tree(module_tree) print() - print("#", "-"*78) - print("# Sorted PHP modules") - print("#", "-"*78) - print(modules) + print("#", "-" * 78) + print("# Build order") + print("#", "-" * 78) + print(names) + + # Create group_vars file mods.yml + write_group_vars(names) if __name__ == "__main__":