diff --git a/src/gardenlinux/apt/__init__.py b/src/gardenlinux/apt/__init__.py index 3d2dfb7c..5697b29c 100644 --- a/src/gardenlinux/apt/__init__.py +++ b/src/gardenlinux/apt/__init__.py @@ -4,7 +4,31 @@ APT module """ +from .changelog_file import ChangelogFile +from .control_file import ControlFile +from .copyright_file import CopyrightFile from .debsource import Debsrc, DebsrcFile +from .docs_file import DocsFile +from .install_file import InstallFile from .package_repo_info import GardenLinuxRepo +from .preinst_file import PreinstFile +from .prerm_file import PrermFile +from .postinst_file import PostinstFile +from .postrm_file import PostrmFile +from .rules_file import RulesFile -__all__ = ["Debsrc", "DebsrcFile", "GardenLinuxRepo"] +__all__ = [ + "ChangelogFile", + "ControlFile", + "CopyrightFile", + "Debsrc", + "DebsrcFile", + "DocsFile", + "GardenLinuxRepo", + "InstallFile", + "PreinstFile", + "PrermFile", + "PostinstFile", + "PostrmFile", + "RulesFile", +] diff --git a/src/gardenlinux/apt/changelog_file.py b/src/gardenlinux/apt/changelog_file.py new file mode 100644 index 00000000..4f8a5de7 --- /dev/null +++ b/src/gardenlinux/apt/changelog_file.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +from time import gmtime, strftime +import textwrap + +from ..constants import GL_AUTHORS_EMAIL, GL_AUTHORS_NAME +from .debian_file_mixin import DebianFileMixin + + +class ChangelogFile(DebianFileMixin): + def __init__(self, package_name, maintainer=None, maintainer_email=None): + DebianFileMixin.__init__(self) + + self._entries = [] + self._package_name = package_name + + if maintainer is None: + maintainer = GL_AUTHORS_NAME + if maintainer_email is None: + maintainer_email = GL_AUTHORS_EMAIL + + self._maintainer = maintainer + self._maintainer_email = maintainer_email + + @property + def content(self): + content = "" + + for entry in self._entries: + content += f"{entry}\n\n" + + return content.strip() + "\n" + + def add_entry( + self, + package_version, + package_timestamp, + changes, + maintainer=None, + maintainer_email=None, + ): + if maintainer is None: + maintainer = self._maintainer + if maintainer_email is None: + maintainer_email = self._maintainer_email + + changes = textwrap.indent(changes.strip(), " ") + + package_timestamp_string = strftime( + "%a, %d %b %Y %H:%M:%S +0000", gmtime(package_timestamp) + ) + + content = f""" +{self._package_name} ({package_version}) universal; urgency=low + +{changes} + + -- {maintainer} <{maintainer_email}> {package_timestamp_string} + """ + + self._entries.append(content.strip()) + + def generate(self, target_dir): + self._generate(target_dir, "changelog", self.content) diff --git a/src/gardenlinux/apt/code_file_mixin.py b/src/gardenlinux/apt/code_file_mixin.py new file mode 100644 index 00000000..3cd82503 --- /dev/null +++ b/src/gardenlinux/apt/code_file_mixin.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + + +class CodeFileMixin(object): + def __init__(self, executable=None, header_code=None, footer_code=None): + self._code = [] + self._executable = executable + self._footer_code = footer_code + self._header_code = header_code + + @property + def content(self): + content = "" + + if len(self._code) > 0: + if self._executable is not None: + content += f"#!{self._executable}\n\n" + + if self._header_code is not None: + content += self._header_code + "\n" + + for code in self._code: + content += f"\n{code}\n" + + if self._footer_code is not None: + content += "\n" + self._footer_code + + return content.strip() + "\n" + + @property + def empty(self): + return len(self._code) < 1 + + def add_code(self, content): + self._code.append(content.strip()) diff --git a/src/gardenlinux/apt/control_file.py b/src/gardenlinux/apt/control_file.py new file mode 100644 index 00000000..2ce9abe5 --- /dev/null +++ b/src/gardenlinux/apt/control_file.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +from os import PathLike +from pathlib import Path +import textwrap + +from ..constants import GL_AUTHORS_EMAIL, GL_AUTHORS_NAME, GL_HOME_URL +from .debian_file_mixin import DebianFileMixin + + +class ControlFile(dict, DebianFileMixin): + def __init__(self, package_name, *args, **kwargs): + dict.__init__(self, *args, **kwargs) + DebianFileMixin.__init__(self) + + self._conflicts = [] + self._breaking_packages = [] + self._dependencies = [] + self["package_name"] = package_name + self._packages = [] + + @property + def content(self): + homepage = self.get("homepage", GL_HOME_URL) + maintainer = self.get("maintainer", GL_AUTHORS_NAME) + maintainer_email = self.get("maintainer_email", GL_AUTHORS_EMAIL) + + content = f""" +Source: {self["package_name"]} +Standards-Version: 4.0.0 +Section: universe +Priority: optional +Maintainer: {maintainer} <{maintainer_email}> +Build-Depends: debhelper-compat (=12) +Homepage: {homepage} + """.strip() + + for package in self._packages: + content += f"\n\n{package}" + + if len(self._dependencies) > 0: + dependencies = textwrap.indent(", ".join(self._dependencies), " ") + content += f"\nDepends:{dependencies}" + + if len(self._conflicts) > 0: + dependencies = textwrap.indent(", ".join(self._conflicts), " ") + content += f"\nConflicts:{dependencies}" + + if len(self._breaking_packages) > 0: + dependencies = textwrap.indent(", ".join(self._breaking_packages), " ") + content += f"\nBreaks:{dependencies}" + + content += "\n" + + return content + + def add_breaking_package(self, package_name): + if ( + package_name not in self._breaking_packages + and package_name not in self._conflicts + ): + self._breaking_packages.append(package_name) + + def add_conflict(self, package_name): + if ( + package_name not in self._breaking_packages + and package_name not in self._conflicts + ): + self._conflicts.append(package_name) + + def add_dependency(self, package_name): + if package_name not in self._dependencies: + self._dependencies.append(package_name) + + def add_package(self, package_name, description, architecture=None): + if architecture is None: + architecture = "any" + + description = textwrap.indent(description.strip(), " ") + + content = f""" +Package: {package_name} +Architecture: {architecture} +Description:{description} + """ + + self._packages.append(content.strip()) + + def generate(self, target_dir): + if not isinstance(target_dir, PathLike): + target_dir = Path(target_dir) + + if not target_dir.is_dir(): + raise ValueError("Target directory given is invalid") + + source_dir_path = target_dir.joinpath("source") + format_file_path = source_dir_path.joinpath("format") + + if format_file_path.is_file(): + raise RuntimeError( + "Target directory already contains a 'source/format' file" + ) + + if not source_dir_path.is_dir(): + source_dir_path.mkdir() + + self._generate(target_dir, "control", self.content) + self._generate(source_dir_path, "format", "3.0 (native)") diff --git a/src/gardenlinux/apt/copyright_file.py b/src/gardenlinux/apt/copyright_file.py new file mode 100644 index 00000000..def6306e --- /dev/null +++ b/src/gardenlinux/apt/copyright_file.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +from collections import OrderedDict +import textwrap + +from ..constants import GL_AUTHORS_EMAIL, GL_AUTHORS_NAME, GL_REPOSITORY_URL +from .debian_file_mixin import DebianFileMixin + + +class CopyrightFile(dict, DebianFileMixin): + def __init__(self, package_name, *args, **kwargs): + dict.__init__(self, *args, **kwargs) + DebianFileMixin.__init__(self) + + self["package_name"] = package_name + + self._licensed_files_entries = OrderedDict() + self._license_text_entries = {} + + def add_licensed_files_declaration( + self, + files_definition_string, + copyright_note, + license_id=None, + license_text=None, + ): + if files_definition_string in self._licensed_files_entries: + raise ValueError( + "A license has already been declared for the files definition given" + ) + + if license_id is None and license_text is None: + license_id = "MIT" + + license_text = f""" +{copyright_note} + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + """ + + if license_id is None or license_text is None: + raise ValueError("Invalid input for license ID or text") + + if ( + license_id in self._license_text_entries + and self._license_text_entries[license_id] != license_text + ): + raise ValueError("Given license text for already used license ID differs") + + self._license_text_entries[license_id] = license_text.strip() + + files_definition_string = textwrap.indent(files_definition_string.strip(), " ") + copyright_note = textwrap.indent(copyright_note.strip(), " ") + + content = f""" +Files:{files_definition_string} +Copyright:{copyright_note} +License: {license_id} + """.strip() + + self._licensed_files_entries[files_definition_string] = content + + @property + def content(self): + if len(self._licensed_files_entries) < 1: + self.add_licensed_files_declaration("*", self._generate_copyright_note()) + + maintainer = self.get("maintainer", GL_AUTHORS_NAME) + maintainer_email = self.get("maintainer_email", GL_AUTHORS_EMAIL) + package_name = self["package_name"] + source_url = self.get("source_url", GL_REPOSITORY_URL) + + content = f""" +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Source: {source_url} +Upstream-Name: {package_name} +Upstream-Contact: {maintainer} <{maintainer_email}> + """.strip() + + for entry in self._licensed_files_entries.values(): + content += f"\n\n{entry}" + + for entry_id, entry_text in self._license_text_entries.items(): + entry_text = textwrap.indent(entry_text, " ") + content += f"\n\nLicense: {entry_id}\n{entry_text}" + + content += "\n" + + return content + + def generate(self, target_dir): + self._generate(target_dir, "copyright", self.content) + + def _generate_copyright_note(self): + return self.get("copyright", f"Copyright (c) {GL_AUTHORS_NAME}") diff --git a/src/gardenlinux/apt/debian_file_mixin.py b/src/gardenlinux/apt/debian_file_mixin.py new file mode 100644 index 00000000..a9f6b429 --- /dev/null +++ b/src/gardenlinux/apt/debian_file_mixin.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +from os import PathLike +from pathlib import Path + + +class DebianFileMixin(object): + def _generate(self, target_dir, file_name, content, chmod_octet=0o644): + if not isinstance(target_dir, PathLike): + target_dir = Path(target_dir) + + if not target_dir.is_dir(): + raise ValueError("Target directory given is invalid") + + file_path_name = target_dir.joinpath(file_name) + + if file_path_name.is_file(): + raise RuntimeError( + f"Target directory already contains a '{file_path_name}' file" + ) + + file_path_name.write_text(content) + file_path_name.chmod(chmod_octet) diff --git a/src/gardenlinux/apt/docs_file.py b/src/gardenlinux/apt/docs_file.py new file mode 100644 index 00000000..5b5322f8 --- /dev/null +++ b/src/gardenlinux/apt/docs_file.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +from os import PathLike +from pathlib import Path +import os + +from .debian_file_mixin import DebianFileMixin + + +class DocsFile(DebianFileMixin): + def __init__(self): + DebianFileMixin.__init__(self) + self._files = [] + + def add_file(self, file_path_name): + if not isinstance(file_path_name, PathLike): + file_path_name = Path(file_path_name) + + if not file_path_name.is_file(): + raise ValueError("File given is invalid") + + if not os.access(file_path_name, os.R_OK): + raise ValueError("File given is not readable") + + self._files.append(file_path_name) + + @property + def content(self): + content = "" + + if len(self._files) > 0: + for file_path_name in self._files: + content += f"{file_path_name.name}\n" + + return content + + def generate(self, debian_dir): + target_dir = debian_dir.parent + + for file_path_name in self._files: + self._generate(target_dir, file_path_name.name, file_path_name.read_text()) + + self._generate(debian_dir, "docs", self.content) diff --git a/src/gardenlinux/apt/install_file.py b/src/gardenlinux/apt/install_file.py new file mode 100644 index 00000000..b3cd2c30 --- /dev/null +++ b/src/gardenlinux/apt/install_file.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +from collections import OrderedDict +from os import PathLike +from pathlib import Path +import os + +from .debian_file_mixin import DebianFileMixin + + +class InstallFile(DebianFileMixin): + def __init__(self, dir_path, package_name): + DebianFileMixin.__init__(self) + + if not isinstance(dir_path, PathLike): + dir_path = Path(dir_path) + + self._install_definitions = OrderedDict() + self._install_dir_path = dir_path + self._package_name = package_name + + def add_directory(self, dir_path, target_dir): + if not isinstance(dir_path, PathLike): + dir_path = Path(dir_path) + + if not dir_path.is_dir(): + raise ValueError("Directory given is invalid") + + self._install_definitions[target_dir] = str( + dir_path.joinpath("*").relative_to(self._install_dir_path) + ) + + def add_entry(self, path_name, target_path_name): + if not isinstance(path_name, PathLike): + path_name = Path(path_name) + + if not os.access(path_name, os.R_OK): + raise ValueError("Install entry given is not readable") + + self._install_definitions[target_path_name] = str( + path_name.relative_to(self._install_dir_path) + ) + + @property + def content(self): + content = "" + + if len(self._install_definitions) > 0: + for target_path_name, source_path_name in self._install_definitions.items(): + content += f"{source_path_name} {target_path_name}\n" + + return content + + def generate(self, target_dir): + self._generate(target_dir, f"{self._package_name}.install", self.content) diff --git a/src/gardenlinux/apt/postinst_file.py b/src/gardenlinux/apt/postinst_file.py new file mode 100644 index 00000000..797e1a4f --- /dev/null +++ b/src/gardenlinux/apt/postinst_file.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from .code_file_mixin import CodeFileMixin +from .debian_file_mixin import DebianFileMixin + + +class PostinstFile(CodeFileMixin, DebianFileMixin): + def __init__( + self, + executable="/usr/bin/env bash", + header_code="set -euo pipefail", + footer_code="exit 0", + ): + CodeFileMixin.__init__(self, executable, header_code, footer_code) + DebianFileMixin.__init__(self) + + def generate(self, target_dir): + self._generate(target_dir, "postinst", self.content, 0o755) diff --git a/src/gardenlinux/apt/postrm_file.py b/src/gardenlinux/apt/postrm_file.py new file mode 100644 index 00000000..8740dde3 --- /dev/null +++ b/src/gardenlinux/apt/postrm_file.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from .code_file_mixin import CodeFileMixin +from .debian_file_mixin import DebianFileMixin + + +class PostrmFile(CodeFileMixin, DebianFileMixin): + def __init__( + self, + executable="/usr/bin/env bash", + header_code="set -euo pipefail", + footer_code="exit 0", + ): + CodeFileMixin.__init__(self, executable, header_code, footer_code) + DebianFileMixin.__init__(self) + + def generate(self, target_dir): + self._generate(target_dir, "postrm", self.content, 0o755) diff --git a/src/gardenlinux/apt/preinst_file.py b/src/gardenlinux/apt/preinst_file.py new file mode 100644 index 00000000..52c5e9e3 --- /dev/null +++ b/src/gardenlinux/apt/preinst_file.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from .code_file_mixin import CodeFileMixin +from .debian_file_mixin import DebianFileMixin + + +class PreinstFile(CodeFileMixin, DebianFileMixin): + def __init__( + self, + executable="/usr/bin/env bash", + header_code="set -euo pipefail", + footer_code="exit 0", + ): + CodeFileMixin.__init__(self, executable, header_code, footer_code) + DebianFileMixin.__init__(self) + + def generate(self, target_dir): + self._generate(target_dir, "preinst", self.content, 0o755) diff --git a/src/gardenlinux/apt/prerm_file.py b/src/gardenlinux/apt/prerm_file.py new file mode 100644 index 00000000..9def7aac --- /dev/null +++ b/src/gardenlinux/apt/prerm_file.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from .code_file_mixin import CodeFileMixin +from .debian_file_mixin import DebianFileMixin + + +class PrermFile(CodeFileMixin, DebianFileMixin): + def __init__( + self, + executable="/usr/bin/env bash", + header_code="set -euo pipefail", + footer_code="exit 0", + ): + CodeFileMixin.__init__(self, executable, header_code, footer_code) + DebianFileMixin.__init__(self) + + def generate(self, target_dir): + self._generate(target_dir, "prerm", self.content, 0o755) diff --git a/src/gardenlinux/apt/rules_file.py b/src/gardenlinux/apt/rules_file.py new file mode 100644 index 00000000..8373d9f8 --- /dev/null +++ b/src/gardenlinux/apt/rules_file.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +from collections import OrderedDict +import textwrap + +from .debian_file_mixin import DebianFileMixin + + +class RulesFile(DebianFileMixin): + def __init__(self): + DebianFileMixin.__init__(self) + + self._hooks = OrderedDict() + + def add_hook(self, hook, content): + hook = hook.lower() + + if hook in self._hooks: + self._hooks[hook] += "\n\n" + else: + self._hooks[hook] = "" + + self._hooks[hook] += content.strip() + + @property + def content(self): + additional_hooks = "\n" + + for hook, content in self._hooks.items(): + content = textwrap.indent(content, "\t") + additional_hooks += f"{hook}:\n{content}\n" + + content = f""" +#!/usr/bin/make -f +{additional_hooks} +%: + dh $@ + """ + + return content.strip() + "\n" + + def generate(self, target_dir): + self._generate(target_dir, "rules", self.content, 0o777) diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index 293086d8..2f8cc436 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -146,6 +146,8 @@ GL_BUG_REPORT_URL = "https://github.com/gardenlinux/gardenlinux/issues" GL_COMMIT_SPECIAL_VALUES = ("local",) GL_DEB_REPO_BASE_URL = "https://packages.gardenlinux.io/gardenlinux" +GL_AUTHORS_EMAIL = "contact@gardenlinux.io" +GL_AUTHORS_NAME = "Garden Linux Maintainers" GL_DISTRIBUTION_NAME = "Garden Linux" GL_HOME_URL = "https://gardenlinux.io" GL_RELEASE_ID = "gardenlinux" diff --git a/src/gardenlinux/features/__main__.py b/src/gardenlinux/features/__main__.py index 76224da2..e5a1346d 100644 --- a/src/gardenlinux/features/__main__.py +++ b/src/gardenlinux/features/__main__.py @@ -33,11 +33,6 @@ "graph", ] -RE_CAMEL_CASE_SPLITTER = re.compile("([A-Z]+|[a-z0-9])([A-Z])(?!$)") -""" -CamelCase splitter RegExp -""" - def main() -> None: """ @@ -293,7 +288,7 @@ def additional_filter_func(node: str) -> bool: print(cname) elif output_type == "container_name": - print(RE_CAMEL_CASE_SPLITTER.sub("\\1_\\2", cname_base).lower()) + print(CName.get_camel_case_name_for_feature(cname_base)) elif output_type == "graph": print(graph_as_mermaid_markup(flavor, graph)) diff --git a/src/gardenlinux/features/cname.py b/src/gardenlinux/features/cname.py index 3822d793..6caae1ad 100644 --- a/src/gardenlinux/features/cname.py +++ b/src/gardenlinux/features/cname.py @@ -20,6 +20,11 @@ ) from .parser import Parser +RE_CAMEL_CASE_SPLITTER = re.compile("([A-Z]+|[a-z0-9])([A-Z])(?!$)") +""" +CamelCase splitter RegExp +""" + class CName(object): """ @@ -504,3 +509,7 @@ def save_to_release_file( with release_file.open("w") as fp: # type: ignore[attr-defined] fp.write(self.release_metadata_string) + + @staticmethod + def get_camel_case_name_for_feature(feature, char = "_"): + return RE_CAMEL_CASE_SPLITTER.sub(f"\\1{char}\\2", feature).lower() diff --git a/src/gardenlinux/features/packer/__init__.py b/src/gardenlinux/features/packer/__init__.py new file mode 100644 index 00000000..c91bcd2b --- /dev/null +++ b/src/gardenlinux/features/packer/__init__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- + +""" +Packer module +""" + +from .deb_packer import DebPacker + +__all__ = ["DebPacker"] diff --git a/src/gardenlinux/features/packer/__main__.py b/src/gardenlinux/features/packer/__main__.py new file mode 100644 index 00000000..0f3b288e --- /dev/null +++ b/src/gardenlinux/features/packer/__main__.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +gl-packer main entrypoint +""" + +import json +from argparse import ArgumentParser, Namespace +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, List, Tuple + +from ..constants import GL_REPOSITORY_URL +from ..git import Repository +from .parser import Parser + + +def _get_flavors_file_data(flavors_file: Path) -> str: + if not flavors_file.exists(): + raise RuntimeError(f"Error: {flavors_file} does not exist.") + + # Load and validate the flavors.yaml + with flavors_file.open("r") as fp: + return fp.read() + + +def generate_markdown_table(combinations: List[Tuple[Any, str]]) -> str: + """ + Generate a markdown table of platforms and their flavors. + + :param combinations: List of tuples of architectures and flavors + + :return: (str) Markdown table + :since: 0.7.0 + """ + + table = "| Platform | Architecture | Flavor |\n" + table += "|------------|--------------------|------------------------------------------|\n" + + for arch, combination in combinations: + platform = combination.split("-")[0] + table += ( + f"| {platform:<10} | {arch:<18} | `{combination}` |\n" + ) + + return table + + +def parse_args() -> Namespace: + """ + Parses arguments used for main() + + :return: (object) Parsed argparse.ArgumentParser namespace + :since: 0.7.0 + """ + + parser = ArgumentParser(description="Parse flavors.yaml and generate combinations.") + + parser.add_argument( + "--commit", + default=None, + help="Commit hash to fetch flavors.yaml from GitHub. An existing 'flavors.yaml' file will be preferred.", + ) + parser.add_argument( + "--no-arch", + action="store_true", + help="Exclude architecture from the flavor output.", + ) + parser.add_argument( + "--include-only", + action="append", + default=[], + help="Restrict combinations to those matching wildcard patterns (can be specified multiple times).", + ) + parser.add_argument( + "--exclude", + action="append", + default=[], + help="Exclude combinations based on wildcard patterns (can be specified multiple times).", + ) + parser.add_argument( + "--build", + action="store_true", + help="Filter combinations to include only those with build enabled.", + ) + parser.add_argument( + "--publish", + action="store_true", + help="Filter combinations to include only those with publish enabled.", + ) + parser.add_argument( + "--test", + action="store_true", + help="Filter combinations to include only those with test enabled.", + ) + parser.add_argument( + "--test-platform", + action="store_true", + help="Filter combinations to include only platforms with test-platform: true.", + ) + parser.add_argument( + "--category", + action="append", + default=[], + help="Filter combinations to include only platforms belonging to the specified categories (can be specified multiple times).", + ) + parser.add_argument( + "--exclude-category", + action="append", + default=[], + help="Exclude platforms belonging to the specified categories (can be specified multiple times).", + ) + parser.add_argument( + "--json-by-arch", + action="store_true", + help="Output a JSON dictionary where keys are architectures and values are lists of flavors.", + ) + parser.add_argument( + "--markdown-table-by-platform", + action="store_true", + help="Generate a markdown table by platform.", + ) + + return parser.parse_args() + + +def main() -> None: + """ + gl-flavors-parse main() + + :since: 0.7.0 + """ + + args = parse_args() + + try: + flavors_data = _get_flavors_file_data(Path(Repository().root, "flavors.yaml")) + except RuntimeError: + with TemporaryDirectory() as git_directory: + repo = Repository.checkout_repo_sparse( + git_directory, + ["flavors.yaml"], + repo_url=GL_REPOSITORY_URL, + commit=args.commit, + ) + + flavors_data = _get_flavors_file_data(Path(repo.root, "flavors.yaml")) + + combinations = Parser(flavors_data).filter( + include_only_patterns=args.include_only, + wildcard_excludes=args.exclude, + only_build=args.build, + only_test=args.test, + only_test_platform=args.test_platform, + only_publish=args.publish, + filter_categories=args.category, + exclude_categories=args.exclude_category, + ) + + if args.json_by_arch: + grouped_combinations = Parser.group_by_arch(combinations) + + # If --no-arch, strip architectures from the grouped output + if args.no_arch: + grouped_combinations = { + arch: sorted(set(item.replace(f"-{arch}", "") for item in items)) + for arch, items in grouped_combinations.items() + } + + print(json.dumps(grouped_combinations, indent=2)) + elif args.markdown_table_by_platform: + print(generate_markdown_table(combinations)) + else: + if args.no_arch: + printable_combinations = sorted(set(Parser.remove_arch(combinations))) + else: + printable_combinations = sorted(set(comb[1] for comb in combinations)) + + print("\n".join(sorted(set(printable_combinations)))) + + +if __name__ == "__main__": + # Create a null logger as default + main() diff --git a/src/gardenlinux/features/packer/changelog_file.py b/src/gardenlinux/features/packer/changelog_file.py new file mode 100644 index 00000000..f5d83059 --- /dev/null +++ b/src/gardenlinux/features/packer/changelog_file.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +from time import time + +from ...apt import ChangelogFile as _ChangelogFile +from ..cname import CName + + +class ChangelogFile(_ChangelogFile): + def __init__( + self, + feature_package_name, + package_version, + maintainer=None, + maintainer_email=None, + ): + package_name = "gardenlinux-" + CName.get_camel_case_name_for_feature( + feature_package_name, "-" + ) + + _ChangelogFile.__init__(self, package_name, maintainer, maintainer_email) + + self.add_entry( + package_version, time(), f"* Automated build for {package_version}" + ) diff --git a/src/gardenlinux/features/packer/control_file.py b/src/gardenlinux/features/packer/control_file.py new file mode 100644 index 00000000..3b1ae022 --- /dev/null +++ b/src/gardenlinux/features/packer/control_file.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +from ...apt import ControlFile as _ControlFile +from ..cname import CName + + +class ControlFile(_ControlFile): + def __init__(self, feature_package_name, feature, *args, **kwargs): + self._architecture = "all" + self._feature = feature + + self._package_name = "gardenlinux-" + CName.get_camel_case_name_for_feature( + feature_package_name, "-" + ) + + _ControlFile.__init__(self, self._package_name, *args, **kwargs) + + @_ControlFile.content.getter + def content(self): + self.add_package( + self._package_name, + f"Provides GL/features/{self._feature}", + self._architecture, + ) + return _ControlFile.content.fget(self) + + def add_dependency(self, package_name): + if "${Arch}" in package_name or ("[" in package_name and "]" in package_name): + self._architecture = "any" + + _ControlFile.add_dependency(self, package_name) diff --git a/src/gardenlinux/features/packer/copyright_file.py b/src/gardenlinux/features/packer/copyright_file.py new file mode 100644 index 00000000..90b6ee5d --- /dev/null +++ b/src/gardenlinux/features/packer/copyright_file.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +from ...apt import CopyrightFile as _CopyrightFile +from ..cname import CName + + +class CopyrightFile(_CopyrightFile): + def __init__(self, feature_package_name, *args, **kwargs): + package_name = "gardenlinux-" + CName.get_camel_case_name_for_feature( + feature_package_name, "-" + ) + + _CopyrightFile.__init__(self, package_name, *args, **kwargs) diff --git a/src/gardenlinux/features/packer/deb_packer.py b/src/gardenlinux/features/packer/deb_packer.py new file mode 100644 index 00000000..d9682347 --- /dev/null +++ b/src/gardenlinux/features/packer/deb_packer.py @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- + +from glob import glob +from os import PathLike +from pathlib import Path +from tempfile import TemporaryDirectory +import re +import tarfile + +from ..cname import CName +from ..parser import Parser +from .changelog_file import ChangelogFile +from .control_file import ControlFile +from .copyright_file import CopyrightFile +from .docs_file import DocsFile +from .install_file import InstallFile +from .postinst_file import PostinstFile +from .prerm_file import PrermFile +from .rules_file import RulesFile + + +class DebPacker(object): + def __init__(self, version, target_dir): + if not isinstance(target_dir, PathLike): + target_dir = Path(target_dir) + + if not target_dir.is_dir(): + raise ValueError("Target directory given is invalid") + + self._target_dir = target_dir + self._version = version + + def pack(self): + feature_parser = Parser() + features_dir_path = feature_parser.features_dir_path + + for feature_dir_entry in features_dir_path.iterdir(): + if feature_dir_entry.is_dir(): + self.pack_feature(feature_dir_entry.name, feature_parser) + + def pack_feature(self, feature, feature_parser=None): + if feature_parser is None: + feature_parser = Parser() + + feature_graph = feature_parser.filter(feature, ignore_excludes=True) + feature_dir_path = feature_parser.features_dir_path.joinpath(feature) + + feature_package_name = self._get_feature_package_name_from_content( + feature, feature_graph.nodes[feature]["content"] + ) + + package_name = f"gardenlinux-{feature_package_name}" + + tarfile_path_name = self._target_dir.joinpath( + f"{package_name}-{self._version}.tar.gz" + ) + + if tarfile_path_name.is_file(): + raise RuntimeError(f"tar archive '{tarfile_path_name}' already exists") + + with TemporaryDirectory(dir=self._target_dir) as tmp: + tmp_path = Path(tmp) + debian_path = tmp_path.joinpath("debian") + + debian_path.mkdir() + + postinst_file = PostinstFile() + prerm_file = PrermFile() + + ChangelogFile(feature_package_name, self._version).generate(debian_path) + + self._generate_control_file( + feature_graph.nodes, feature, feature_dir_path, debian_path + ) + + CopyrightFile(feature_package_name).generate(debian_path) + + self._generate_docs_file(feature_dir_path, debian_path) + + source_path_names = self._generate_sources( + feature_dir_path, + feature_package_name, + tmp_path, + postinst_file, + prerm_file, + ) + + if not postinst_file.empty: + postinst_file.generate(debian_path) + + if not prerm_file.empty: + prerm_file.generate(debian_path) + + self._generate_install_file( + feature_package_name, tmp_path, debian_path, source_path_names + ) + + RulesFile().generate(debian_path) + + package_name = f"gardenlinux-{feature_package_name}" + + tarfile_path_name = self._target_dir.joinpath( + f"{package_name}-{self._version}.tar.gz" + ) + + with tarfile.open(tarfile_path_name, "w:gz") as tar_archive: + tar_archive.add(tmp_path, ".") + + def _generate_control_file(self, features, feature, feature_dir_path, debian_path): + feature_content = features[feature]["content"] + + feature_package_name = self._get_feature_package_name_from_content( + feature, + feature_content, + ) + + control_file = ControlFile(feature_package_name, feature) + + for feature in feature_content.get("features", {}).get("include", []): + parent_feature_content = features[feature]["content"] + + parent_feature_package_name = self._get_feature_package_name_from_content( + feature, + parent_feature_content, + ) + + control_file.add_dependency( + f"gardenlinux-{parent_feature_package_name} (= {self._version})" + ) + + pkg_include_file = feature_dir_path.joinpath("pkg.include") + + if pkg_include_file.is_file(): + for pkg in pkg_include_file.read_text().splitlines(): + pkg = pkg.strip() + + if len(pkg) < 1 or pkg.startswith("#"): + continue + + control_file.add_dependency(pkg) + + for feature in feature_content.get("features", {}).get("exclude", []): + if feature in features: + parent_feature_content = features[feature]["content"] + else: + feature_graph = Parser().filter(feature, ignore_excludes=True) + parent_feature_content = feature_graph.nodes[feature]["content"] + + parent_feature_package_name = self._get_feature_package_name_from_content( + feature, parent_feature_content + ) + + control_file.add_conflict(f"gardenlinux-{parent_feature_package_name}") + + pkg_exclude_file = feature_dir_path.joinpath("pkg.exclude") + + if pkg_exclude_file.is_file(): + for pkg in pkg_exclude_file.read_text().splitlines(): + pkg = pkg.strip() + + if len(pkg) < 1 or pkg.startswith("#"): + continue + + control_file.add_breaking_package(pkg) + + control_file.generate(debian_path) + + def _generate_docs_file(self, feature_dir_path, debian_path): + readme_file = feature_dir_path.joinpath("README.md") + + if readme_file.is_file(): + docs_file = DocsFile() + docs_file.add_file(readme_file) + docs_file.generate(debian_path) + + def _generate_install_file( + self, feature_package_name, tmp_path, debian_path, source_path_names + ): + if len(source_path_names) > 0: + package_name = f"gardenlinux-{feature_package_name}" + source_dir_path = tmp_path.joinpath(f"{package_name}-{self._version}") + + install_file = InstallFile(tmp_path, feature_package_name) + + for source_path_name in source_path_names: + source_path = source_dir_path.joinpath(source_path_name) + + if source_path.is_dir(): + install_file.add_directory(source_path, str(source_path_name)) + else: + install_file.add_entry(source_path, str(source_path_name)) + + install_file.generate(debian_path) + + def _generate_sources( + self, + feature_dir_path, + feature_package_name, + tmp_path, + postinst_file, + prerm_file, + ): + source_path_names = [] + + package_name = f"gardenlinux-{feature_package_name}" + source_dir_path = tmp_path.joinpath(f"{package_name}-{self._version}") + source_dir_path.mkdir() + + file_include_dir_path = feature_dir_path.joinpath("file.include") + + if file_include_dir_path.is_dir(): + for root_path, _, files in file_include_dir_path.walk(): + relative_path = root_path.relative_to(file_include_dir_path) + + if relative_path.is_relative_to("usr/local"): + # self._logger.warn(f"Moving '{relative_path}' to '/usr' as Debian packages conformance requirements") + + target_dir_path = source_dir_path.joinpath("usr") + + if relative_path != Path("usr", "local"): + target_dir_path = target_dir_path.joinpath( + *relative_path.parts[2:] + ) + else: + target_dir_path = source_dir_path.joinpath(relative_path) + + hidden_files_count = 0 + + for file_name in files: + if file_name.startswith("."): + hidden_files_count += 1 + + only_hidden_files = hidden_files_count == len(files) + + for file_name in files: + if not target_dir_path.is_dir(): + if not only_hidden_files: + source_path_names.append(str(target_dir_path.relative_to(source_dir_path))) + + target_dir_path.mkdir(exist_ok=True, parents=True) + + file_path = target_dir_path.joinpath(file_name) + + root_path.joinpath(file_name).copy( + file_path, + preserve_metadata=True, + follow_symlinks=False, + ) + + if ( + file_path.name.startswith(".") + or root_path == file_include_dir_path + ): + source_path_names.append( + str(file_path.relative_to(source_dir_path)) + ) + + file_stat_file = feature_dir_path.joinpath("file.include.stat") + + if file_stat_file.is_file(): + re_object = re.compile("\\s+") + for file_line in file_stat_file.read_text().splitlines(): + file_line = file_line.strip() + + if len(file_line) < 1 or file_line.startswith("#"): + continue + + file_data = re_object.split(file_line, 3) + + if len(file_data) != 4: + # self._logger.warn(f"{file_stat_file} contains invalid stat definition lines") + continue + + file_glob = file_data[3] + + if file_glob.startswith("/"): + file_glob_list = glob(file_data[3][1:], root_dir=source_dir_path) + else: + file_glob_list = glob(file_data[3], root_dir=source_dir_path) + + if len(file_glob_list) < 1: + # self._logger.warn(f"{file_stat_file} contains stat definition lines not matching any files") + pass + + for file_name in file_glob_list: + postinst_file.add_code( + f'dpkg-statoverride --add {file_data[0]} {file_data[1]} {file_data[2]} "/{file_name}"' + ) + + prerm_file.add_code(f'dpkg-statoverride --remove "/{file_name}"') + + return source_path_names + + def _get_feature_package_name_from_content(self, feature, feature_content): + if feature.startswith("_"): + feature = feature[1:] + + feature = CName.get_camel_case_name_for_feature(feature, "-") + + return f"{feature_content['type']}-{feature}" diff --git a/src/gardenlinux/features/packer/docs_file.py b/src/gardenlinux/features/packer/docs_file.py new file mode 100644 index 00000000..c778c22c --- /dev/null +++ b/src/gardenlinux/features/packer/docs_file.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +from ...apt import DocsFile as _DocsFile + + +class DocsFile(_DocsFile): + pass diff --git a/src/gardenlinux/features/packer/install_file.py b/src/gardenlinux/features/packer/install_file.py new file mode 100644 index 00000000..7a87243c --- /dev/null +++ b/src/gardenlinux/features/packer/install_file.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +from ...apt import InstallFile as _InstallFile +from ..cname import CName + + +class InstallFile(_InstallFile): + def __init__(self, dir_path, feature_package_name): + package_name = "gardenlinux-" + CName.get_camel_case_name_for_feature( + feature_package_name, "-" + ) + + _InstallFile.__init__(self, dir_path, package_name) diff --git a/src/gardenlinux/features/packer/postinst_file.py b/src/gardenlinux/features/packer/postinst_file.py new file mode 100644 index 00000000..214685d2 --- /dev/null +++ b/src/gardenlinux/features/packer/postinst_file.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +from ...apt import PostinstFile as _PostinstFile + + +class PostinstFile(_PostinstFile): + def __init__(self): + _PostinstFile.__init__(self) diff --git a/src/gardenlinux/features/packer/postrm_file.py b/src/gardenlinux/features/packer/postrm_file.py new file mode 100644 index 00000000..13b94fa3 --- /dev/null +++ b/src/gardenlinux/features/packer/postrm_file.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +from ...apt import PostrmFile as _PostrmFile + + +class PostrmFile(_PostrmFile): + def __init__(self): + _PostrmFile.__init__(self) diff --git a/src/gardenlinux/features/packer/preinst_file.py b/src/gardenlinux/features/packer/preinst_file.py new file mode 100644 index 00000000..07406c5f --- /dev/null +++ b/src/gardenlinux/features/packer/preinst_file.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +from ...apt import PreinstFile as _PreinstFile + + +class PreinstFile(_PreinstFile): + def __init__(self): + _PreinstFile.__init__(self) diff --git a/src/gardenlinux/features/packer/prerm_file.py b/src/gardenlinux/features/packer/prerm_file.py new file mode 100644 index 00000000..d8509c0b --- /dev/null +++ b/src/gardenlinux/features/packer/prerm_file.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +from ...apt import PrermFile as _PrermFile + + +class PrermFile(_PrermFile): + def __init__(self): + _PrermFile.__init__(self) diff --git a/src/gardenlinux/features/packer/rules_file.py b/src/gardenlinux/features/packer/rules_file.py new file mode 100644 index 00000000..930c9a8e --- /dev/null +++ b/src/gardenlinux/features/packer/rules_file.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +from ...apt import RulesFile as _RulesFile + + +class RulesFile(_RulesFile): + pass diff --git a/src/gardenlinux/features/parser.py b/src/gardenlinux/features/parser.py index 497e66fb..6686192c 100644 --- a/src/gardenlinux/features/parser.py +++ b/src/gardenlinux/features/parser.py @@ -73,6 +73,17 @@ def __init__( "features.Parser initialized for directory: {0}".format(feature_base_dir) ) + @property + def features_dir_path(self) -> Path[str]: + """ + Returns the GardenLinux features directory. + + :return: (Path) Features directory path + :since: 1.0.0 + """ + + return self._feature_base_dir + @property def graph(self) -> networkx.Graph: """ @@ -284,7 +295,8 @@ def filter_based_on_feature_set( for feature in feature_set: for node in networkx.descendants( - Parser._get_graph_view_for_attr(self.graph, "include"), feature + Parser._get_graph_view_for_attr(self.graph, "include"), + feature, ): if node not in filter_set: filter_set.append(node) @@ -395,6 +407,9 @@ def get_flavor_as_feature_set(cname: str) -> List[str]: flags = [] for feature in cname.split("-"): + if len(feature) < 1: + continue + if platform is None: platform = feature continue