From 13e0e5d37ba00271c326fc3723c4140d17cfb6c0 Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Fri, 4 Sep 2026 14:49:05 +0200 Subject: [PATCH 1/5] test(database/csv): Cover missing input file Verify that CSV input convert missing-file errors into ConnectionFailed exceptions. --- tests/unit/database/csv/test_driver.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/database/csv/test_driver.py b/tests/unit/database/csv/test_driver.py index 06673157..6d823a61 100644 --- a/tests/unit/database/csv/test_driver.py +++ b/tests/unit/database/csv/test_driver.py @@ -32,6 +32,7 @@ import pytest from powerapi.database.csv.driver import CSVInput, CSVInputFactory, CSVOutput, CSVOutputFactory +from powerapi.database.exceptions import ConnectionFailed from powerapi.report import FormulaReport, HWPCReport, PowerReport, Report @@ -66,6 +67,16 @@ def test_csv_input_factory_is_picklable() -> None: pickle.dumps(factory) +def test_csv_input_connect_with_missing_file_raise_connection_failed(tmp_path) -> None: + """ + CSV input should report a controlled connection failure when an input file cannot be opened. + """ + csv_input = CSVInput(HWPCReport, [str(tmp_path / 'missing.csv')]) + + with pytest.raises(ConnectionFailed): + csv_input.connect() + + @pytest.mark.parametrize('report_type', [PowerReport, FormulaReport]) def test_create_csv_output(report_type: type[Report]) -> None: """ From 02898f342222ddb52b58afee7bc1546815f6c492 Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Fri, 4 Sep 2026 14:51:24 +0200 Subject: [PATCH 2/5] test(database/json): Cover missing input file Verify that JSON inputs convert missing-file errors into ConnectionFailed exception. --- tests/unit/database/json/test_driver.py | 43 +++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/unit/database/json/test_driver.py diff --git a/tests/unit/database/json/test_driver.py b/tests/unit/database/json/test_driver.py new file mode 100644 index 00000000..49e2f17c --- /dev/null +++ b/tests/unit/database/json/test_driver.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest + +from powerapi.database.exceptions import ConnectionFailed +from powerapi.database.json.driver import JsonInput +from powerapi.report import HWPCReport + + +def test_json_input_connect_with_missing_file_raise_connection_failed(tmp_path) -> None: + """ + JSON input should report a controlled connection failure when its input file cannot be opened. + """ + json_input = JsonInput(HWPCReport, str(tmp_path / 'missing.jsonl'), 'auto') + + with pytest.raises(ConnectionFailed): + json_input.connect() From e257808cd4fb32a232ddc36e2e9b60494dc8b357 Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Fri, 4 Sep 2026 15:00:20 +0200 Subject: [PATCH 3/5] feat(cli)!: Use explicit configuration assignments Replace the contextual CLI parser with argparse and the repeatable `-C/--set-config PATH=VALUE` option. Introduce schemas for root properties, component groups and fixed component settings. Load JSON files, environment variables and CLI assignments independencly, merge them with CLI precedence, and validate the resulting configuration once. BREAKING CHANGE: Contextual componenet arguments are replaced by explicit configuration assignments. Schema aliases, ConfigValidator, and legacy parser APIs and exceptions are removed. --- src/powerapi/cli/__init__.py | 2 - src/powerapi/cli/_utils.py | 79 +- src/powerapi/cli/cli_parser.py | 451 ++++++ .../cli/common_cli_parsing_manager.py | 350 ++--- src/powerapi/cli/config_loader.py | 170 +++ src/powerapi/cli/config_parser.py | 927 ++++-------- src/powerapi/cli/config_validator.py | 139 -- src/powerapi/cli/generator.py | 2 +- src/powerapi/cli/parsing_manager.py | 324 ++-- src/powerapi/exception.py | 21 +- tests/unit/cli/conftest.py | 378 +---- tests/unit/cli/test_cli_parser.py | 318 ++++ .../cli/test_common_cli_parsing_manager.py | 244 +-- tests/unit/cli/test_config_parser.py | 1111 +++++--------- tests/unit/cli/test_config_validator.py | 162 -- tests/unit/cli/test_generator_k8s.py | 2 +- tests/unit/cli/test_parsing_manager.py | 1323 ++--------------- tests/unit/cli/test_utils.py | 98 ++ tests/utils/cli/base_config_parser.py | 31 +- ...put_stream_mode_enabled_configuration.json | 8 +- ..._pre_processor_complete_configuration.json | 2 +- ...ith_non_existing_puller_configuration.json | 2 +- ...uts_stream_mode_enabled_configuration.json | 6 +- 23 files changed, 2289 insertions(+), 3861 deletions(-) create mode 100644 src/powerapi/cli/cli_parser.py create mode 100644 src/powerapi/cli/config_loader.py delete mode 100644 src/powerapi/cli/config_validator.py create mode 100644 tests/unit/cli/test_cli_parser.py delete mode 100644 tests/unit/cli/test_config_validator.py create mode 100644 tests/unit/cli/test_utils.py diff --git a/src/powerapi/cli/__init__.py b/src/powerapi/cli/__init__.py index 34334ce1..0ff4604d 100644 --- a/src/powerapi/cli/__init__.py +++ b/src/powerapi/cli/__init__.py @@ -26,5 +26,3 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from .config_validator import ConfigValidator diff --git a/src/powerapi/cli/_utils.py b/src/powerapi/cli/_utils.py index 20d528eb..5206829f 100644 --- a/src/powerapi/cli/_utils.py +++ b/src/powerapi/cli/_utils.py @@ -27,25 +27,27 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -def find_longest_string_in_list(strings: list[str]) -> str: - """ - Find the longest string from a given list of string. - :param strings: List of strings - """ - return max(strings, key=len) +from copy import deepcopy +from typing import Any def string_to_bool(value: str) -> bool: """ - Transforms a string to a boolean according to its content. - :param value: The string to be converted - :return: Boolean value + Convert a textual boolean value. + :param value: Textual boolean value. + :return: Converted boolean. + :raises ValueError: If the value is not a recognized boolean. """ - return value.casefold() in ("yes", "y", "true", "t", "1") + normalized_value = value.strip().casefold() + if normalized_value in ('yes', 'y', 'true', 't', '1'): + return True + if normalized_value in ('no', 'n', 'false', 'f', '0'): + return False + raise ValueError(f'Invalid boolean value: {value}') -def string_to_list(value: str) -> list: + +def string_to_list(value: str) -> list[str]: """ Transforms a comma separated list to a list of strings. :param value: The string to be converted @@ -57,48 +59,21 @@ def string_to_list(value: str) -> list: return [v.strip() for v in value.split(',')] -def merge_dictionaries(source: dict, destination: dict) -> dict: - """ - Recursively merge the source dictionary into destination. - :param source: Dictionary to be merged - :param destination: Dictionary where the source will be merged to +def merge_dictionaries(*configurations: dict[str, Any]) -> dict[str, Any]: """ - for key, value in source.items(): - if isinstance(value, dict) and key in destination and isinstance(destination[key], dict): - destination[key] = merge_dictionaries(value, destination[key]) - else: - destination[key] = value - - return destination + Recursively merge configurations from lowest to highest precedence. - -def get_longest_related_suffix(var: str, suffixes: list) -> str: - """ - Search for the longest suffix of a string variable in a provided list. It returns None if a suffix is not found - :param var: A string for looking its longest suffix - :param suffixes: A list of suffixes + Later configurations override earlier ones. Inputs are not modified. + :param configurations: Configuration dictionaries ordered from lowest to highest precedence. + :return: A new recursively merged configuration dictionary. """ - suffix = None - - for current_suffix in suffixes: - if var.endswith(current_suffix): - if suffix is None: - suffix = current_suffix - elif len(current_suffix) > len(suffix): - suffix = current_suffix - - return suffix + merged = {} + for configuration in configurations: + for key, value in configuration.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = merge_dictionaries(merged[key], value) + else: + merged[key] = deepcopy(value) -def to_lower_case_and_replace_separators(strings: list, old_separator: str, new_separator: str) -> list: - """ - Transform all strings in a provided list to lower case and replace a given separator for a new one - :param strings: List of string to be converted to lower case - :param old_separator: The old separator in the list of strings - :param new_separator: The new separator in the list of strings - """ - new_strings = [] - for current_string in strings: - new_strings.append(current_string.lower().replace(old_separator, new_separator)) - - return new_strings + return merged diff --git a/src/powerapi/cli/cli_parser.py b/src/powerapi/cli/cli_parser.py new file mode 100644 index 00000000..8567d348 --- /dev/null +++ b/src/powerapi/cli/cli_parser.py @@ -0,0 +1,451 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +from dataclasses import dataclass + +from powerapi.cli.config_parser import ( + ArgumentDefinition, + ComponentGroupSchema, + ComponentSchema, + ConfigurationSchema, + ConfigurationSectionSchema, +) +from powerapi.exception import PowerAPIExceptionWithMessage + +_CONFIG_ASSIGNMENTS_DEST = '_powerapi_config_assignments' +_CONFIG_FILE_DEST = '_powerapi_config_file' + + +class CLIParseException(PowerAPIExceptionWithMessage): + """ + Exception raised when command-line arguments cannot be parsed. + """ + + +@dataclass(frozen=True) +class ConfigAssignment: + """ + Raw value associated with a hierarchical configuration path. + """ + path: tuple[str, ...] + value: str + + +@dataclass(frozen=True) +class CLIParseResult: + """ + Configuration values and configuration file selected on the CLI. + """ + configuration: dict + config_file: str | None + + +@dataclass(frozen=True) +class _HelpTheme: + """ + Argparse colors assigned to configuration help elements. + """ + heading_color: str = '' + selector_color: str = '' + path_color: str = '' + supported_type_color: str = '' + component_type_color: str = '' + details_color: str = '' + reset: str = '' + + def _colorize(self, text: str, color: str) -> str: + """ + Apply one theme color to text. + :param text: Text to colorize. + :param color: ANSI color sequence, or an empty string when colors are disabled. + :return: Colorized or unchanged text. + """ + return f'{color}{text}{self.reset}' + + def heading(self, text: str) -> str: + """ + Colorize a configuration help heading. + :param text: Heading to colorize. + :return: Colorized or unchanged heading. + """ + return self._colorize(text, self.heading_color) + + def selector(self, text: str) -> str: + """ + Colorize a configuration block selector. + :param text: Selector to colorize. + :return: Colorized or unchanged selector. + """ + return self._colorize(text, self.selector_color) + + def path(self, text: str) -> str: + """ + Colorize a nested configuration path. + :param text: Path to colorize. + :return: Colorized or unchanged path. + """ + return self._colorize(text, self.path_color) + + def supported_type(self, text: str) -> str: + """ + Colorize a supported component type. + :param text: Component type to colorize. + :return: Colorized or unchanged component type. + """ + return self._colorize(text, self.supported_type_color) + + def component_type(self, text: str) -> str: + """ + Colorize a selected component type. + :param text: Component type to colorize. + :return: Colorized or unchanged component type. + """ + return self._colorize(text, self.component_type_color) + + def details(self, text: str) -> str: + """ + Colorize configuration argument details. + :param text: Details to colorize. + :return: Colorized or unchanged details. + """ + return self._colorize(text, self.details_color) + + +def _parse_assignment(expression: str) -> ConfigAssignment: + """ + Parse a dotted ``PATH=VALUE`` configuration assignment. + :param expression: Assignment expression to parse. + :return: Parsed configuration path and raw value. + :raises CLIParseException: If the expression does not contain a valid supported path. + """ + path_expression, separator, value = expression.partition('=') + if separator == '': + raise CLIParseException(f'Invalid configuration assignment "{expression}": expected PATH=VALUE') + + path = tuple(path_expression.split('.')) + if any(segment == '' for segment in path): + raise CLIParseException(f'Invalid configuration assignment "{expression}": path contains an empty segment') + if len(path) not in (1, 3): + raise CLIParseException(f'Invalid configuration assignment "{expression}": expected PROPERTY or GROUP.COMPONENT.PROPERTY') + + return ConfigAssignment(path=path, value=value) + + +def _apply_assignment(configuration: dict, assignment: ConfigAssignment) -> None: + """ + Apply an assignment to a nested configuration. + :param configuration: Configuration to update. + :param assignment: Parsed assignment to apply. + :raises CLIParseException: If the assignment conflicts with an existing path. + """ + target = configuration + dotted_path = '.'.join(assignment.path) + + for segment in assignment.path[:-1]: + target = target.setdefault(segment, {}) + if not isinstance(target, dict): + raise CLIParseException(f'Conflicting configuration path: "{dotted_path}"') + + property_name = assignment.path[-1] + if isinstance(target.get(property_name), dict): + raise CLIParseException(f'Conflicting configuration path: "{dotted_path}"') + + target[property_name] = assignment.value + + +def _build_configuration(expressions: list[str]) -> dict: + """ + Build a nested configuration dictionary from dotted assignments. + :param expressions: Assignment expressions ordered as they appeared on the command line. + :return: Nested configuration containing the assigned raw values. + :raises CLIParseException: If an expression is invalid or conflicts with another path. + """ + configuration = {} + + for expression in expressions: + _apply_assignment(configuration, _parse_assignment(expression)) + + return configuration + + +def _get_argument_definitions(schema: ConfigurationSectionSchema) -> list[ArgumentDefinition]: + """ + Return each argument definition once in registration order. + :param schema: Configuration section containing the arguments. + :return: Canonical argument definitions. + """ + return list(schema.arguments.values()) + + +def _get_argparse_theme() -> _HelpTheme: + """ + Return argparse's active color theme when its expected layout is available. + :return: Active argparse theme, or a plain-text fallback. + """ + formatter = argparse.RawDescriptionHelpFormatter('') + try: + theme = getattr(formatter, '_theme') # noqa: B009 + return _HelpTheme( + heading_color=theme.heading, + selector_color=theme.long_option, + path_color=theme.summary_long_option, + supported_type_color=theme.summary_action, + component_type_color=theme.action, + details_color=theme.summary_label, + reset=theme.reset, + ) + except AttributeError: + return _HelpTheme() + + +def _format_argument_help(argument: ArgumentDefinition, theme: _HelpTheme) -> str: + """ + Format the description and constraints of a configuration argument. + :param argument: Argument definition to describe. + :param theme: Active argparse color theme. + :return: Human-readable argument help. + """ + details = [argument.argument_type.__name__] + if argument.is_mandatory: + details.append('required') + elif argument.default_value is not None: + details.append(f"default: {argument.default_value!r}") + + details_text = theme.details(f'({", ".join(details)})') + return f'{argument.help_text} {details_text}' if argument.help_text else details_text + + +def _format_assignment_help(theme: _HelpTheme) -> str: + """ + Format the supported explicit-assignment syntax. + :param theme: Active argparse color theme. + :return: Assignment syntax and examples. + """ + return '\n'.join([ + theme.heading('configuration assignments:'), + f' {theme.path('PROPERTY=VALUE')}', + f' {theme.path('GROUP.NAME.PROPERTY=VALUE')}', + ' Example: -C stream=false -C input.sensor.type=socket -C input.sensor.port=9080', + ]) + + +def _format_root_arguments_help(schema: ConfigurationSchema, theme: _HelpTheme) -> str: + """ + Format root configuration paths. + :param schema: Configuration schema containing the root arguments. + :param theme: Active argparse color theme. + :return: Root configuration help, or an empty string when no root arguments are registered. + """ + arguments = _get_argument_definitions(schema) + if not arguments: + return '' + + lines = [theme.heading('root properties:')] + for argument in arguments: + lines.append(f' {theme.path(argument.name)}') + lines.append(f' {_format_argument_help(argument, theme)}') + + return '\n'.join(lines) + + +def _format_component_help(group_name: str, component: ComponentSchema, theme: _HelpTheme) -> str: + """ + Format configuration paths for one component type. + :param group_name: Name of the component group. + :param component: Component schema to describe. + :param theme: Active argparse color theme. + :return: Component type assignment and its property paths. + """ + selector = theme.selector(f'{group_name}.NAME.type') + component_type = theme.component_type(component.name) + lines = [f' {selector}={component_type}'] + for argument in _get_argument_definitions(component): + path = theme.path(f'{group_name}.NAME.{argument.name}') + lines.append(f' {path}') + lines.append(f' {_format_argument_help(argument, theme)}') + + return '\n'.join(lines) + + +def _format_section_help(group_name: str, section_name: str, section: ConfigurationSectionSchema, theme: _HelpTheme) -> str: + """ + Format configuration paths for one fixed section. + :param group_name: Name of the configuration group. + :param section_name: Name of the fixed section. + :param section: Configuration section schema to describe. + :param theme: Active argparse color theme. + :return: Fixed section property paths. + """ + selector = theme.selector(f'{group_name}.{section_name}') + lines = [f' {selector}:'] + for argument in _get_argument_definitions(section): + path = theme.path(f'{group_name}.{section_name}.{argument.name}') + lines.append(f' {path}') + lines.append(f' {_format_argument_help(argument, theme)}') + + return '\n'.join(lines) + + +def _format_group_help(group: ComponentGroupSchema, theme: _HelpTheme) -> str: + """ + Format configuration paths for a configuration group. + :param group: Configuration group to describe. + :param theme: Active argparse color theme. + :return: Group configuration help, or an empty string when the group has no registered schemas. + """ + if not group.components and not group.sections: + return '' + + lines = [theme.heading(f'{group.group_name} configuration:')] + + if group.help_text: + lines.append(f' {group.help_text}') + + if group.components: + selector = theme.selector(f'{group.group_name}.NAME.type') + supported_types = ', '.join(map(theme.supported_type, group.components)) + lines.append(f' {selector}') + lines.append(f' Supported types: {supported_types}') + lines.append('') + lines.append('\n\n'.join( + _format_component_help(group.group_name, component, theme) + for component in group.components.values() + )) + + lines.extend( + _format_section_help(group.group_name, section_name, section, theme) + for section_name, section in group.sections.items() + ) + + return '\n'.join(lines) + + +def _format_configuration_help(schema: ConfigurationSchema) -> str: + """ + Format assignment paths registered in a configuration schema. + :param schema: Configuration schema to describe. + :return: Text appended to the standard argparse help. + """ + theme = _get_argparse_theme() + sections = [ + _format_assignment_help(theme), + _format_root_arguments_help(schema, theme), + *(_format_group_help(group, theme) for group in schema.groups.values()), + ] + return '\n\n'.join(section for section in sections if section) + + +class CLIArgumentParser: + """ + Parse ordinary CLI options and repeatable configuration overrides. + """ + + def __init__(self, schema: ConfigurationSchema) -> None: + """ + Initialize a schema-driven command-line parser. + :param schema: Configuration schema used to register root options and generate help. + """ + self._schema = schema + + @staticmethod + def _add_schema_arguments(argument: ArgumentDefinition, parser: argparse.ArgumentParser) -> None: + """ + Register one schema argument as a command-line option. + :param argument: Schema argument to register. + :param parser: Argument parser receiving the option. + :raises CLIParseException: If the option conflicts with an existing command-line option. + """ + kwargs = { + 'default': argparse.SUPPRESS, + 'dest': argument.name, + 'help': argument.help_text, + } + if argument.is_flag: + kwargs['action'] = 'store_true' + + try: + parser.add_argument(f'--{argument.name}', **kwargs) + except argparse.ArgumentError as error: + raise CLIParseException(f'Failed to add argument: {error}') from error + + def _build_parser(self) -> argparse.ArgumentParser: + """ + Build an argument parser from the current configuration schema. + :return: Configured argument parser. + :raises CLIParseException: If schema arguments define conflicting command-line options. + """ + parser = argparse.ArgumentParser( + allow_abbrev=False, + exit_on_error=False, + epilog=_format_configuration_help(self._schema), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + '--config-file', + dest=_CONFIG_FILE_DEST, + metavar='FILE', + help='Load configuration from a JSON file', + ) + + parser.add_argument( + '-C', '--set-config', + action='append', + default=None, + dest=_CONFIG_ASSIGNMENTS_DEST, + metavar='PATH=VALUE', + help='Set a configuration value using a dotted path; may be repeated', + ) + + for argument in _get_argument_definitions(self._schema): + self._add_schema_arguments(argument, parser) + + return parser + + def parse(self, args: list[str]) -> CLIParseResult: + """ + Parse command-line arguments without validating component schemas. + :param args: Command-line arguments without the executable name. + :return: Parsed root configuration, dotted assignments, and optional configuration file path. + :raises CLIParseException: If argparse rejects the arguments or a dotted assignment is invalid. + """ + parser = self._build_parser() + try: + namespace = parser.parse_args(args) + except argparse.ArgumentError as error: + raise CLIParseException(f'Failed to parse CLI: {error}') from error + + parsed_arguments = dict(vars(namespace)) + expressions = parsed_arguments.pop(_CONFIG_ASSIGNMENTS_DEST) or [] + config_file = parsed_arguments.pop(_CONFIG_FILE_DEST) + + configuration = _build_configuration(expressions) + configuration.update(parsed_arguments) + + return CLIParseResult(configuration=configuration, config_file=config_file) diff --git a/src/powerapi/cli/common_cli_parsing_manager.py b/src/powerapi/cli/common_cli_parsing_manager.py index 86df9594..a7cad558 100644 --- a/src/powerapi/cli/common_cli_parsing_manager.py +++ b/src/powerapi/cli/common_cli_parsing_manager.py @@ -27,8 +27,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.cli.config_parser import store_true -from powerapi.cli.parsing_manager import RootConfigParsingManager, SubgroupConfigParsingManager +from powerapi.cli.config_parser import ComponentSchema +from powerapi.cli.parsing_manager import ConfigurationParsingManager def generate_env_prefix(*components: str, root_prefix: str = 'POWERAPI') -> str: @@ -43,119 +43,107 @@ def generate_env_prefix(*components: str, root_prefix: str = 'POWERAPI') -> str: ) + '_' -class PullerConfigParsingManager(SubgroupConfigParsingManager): +class PullerSchema(ComponentSchema): """ - Subgroup parser with arguments shared by every puller input. + Component schema with arguments shared by every puller input. """ def __init__(self, name: str) -> None: """ - Initialize a puller parser with common actor name and report model arguments. + Initialize a puller schema with the common report model argument. """ super().__init__(name) self.add_argument( - 'n', 'name', - help_text='Name assigned to this puller actor' - ) - self.add_argument( - 'm', 'model', + 'model', help_text='Report type produced by this input source', default_value='HWPCReport' ) -class PusherConfigParsingManager(SubgroupConfigParsingManager): +class PusherSchema(ComponentSchema): """ - Subgroup parser with arguments shared by every pusher output. + Component schema with arguments shared by every pusher output. """ def __init__(self, name: str) -> None: """ - Initialize a pusher parser with common actor name and report model arguments. + Initialize a pusher schema with the common report model argument. """ super().__init__(name) self.add_argument( - 'n', 'name', - help_text='Name assigned to this pusher actor' - ) - self.add_argument( - 'm', 'model', + 'model', help_text='Report type consumed by this output destination', default_value='PowerReport' ) -class PreProcessorConfigParsingManager(SubgroupConfigParsingManager): +class PreProcessorSchema(ComponentSchema): """ - Subgroup parser with arguments shared by every pre-processor. + Component schema with arguments shared by every pre-processor. """ def __init__(self, name: str) -> None: """ - Initialize a pre-processor parser with common actor name and puller binding arguments. + Initialize a pre-processor schema with the puller binding argument. """ super().__init__(name) self.add_argument( - 'n', 'name', - help_text='Name assigned to this pre-processor actor' - ) - self.add_argument( - 'p', 'puller', + 'puller', help_text='Name of the puller actor this pre-processor receives reports from', is_mandatory=True, ) -class CommonCLIParsingManager(RootConfigParsingManager): +class CommonCLIParsingManager(ConfigurationParsingManager): """ - Root parser that registers PowerAPI's built-in CLI component options. + Configuration manager that registers PowerAPI's built-in CLI component options. """ def __init__(self) -> None: """ - Initialize the root parser and register all built-in component parsers. + Initialize the configuration manager and register all built-in component schemas. """ super().__init__() self._register_environment_prefixes() - self._register_subgroups() + self._register_groups() self._register_root_arguments() - self._register_input_parsers() - self._register_output_parsers() - self._register_pre_processor_parsers() + self._register_input_schemas() + self._register_output_schemas() + self._register_pre_processor_schemas() def _register_environment_prefixes(self) -> None: """ - Register environment variable prefixes accepted by the root parser. + Register environment variable prefixes accepted by the configuration manager. """ self.add_argument_prefix(generate_env_prefix()) - def _register_subgroups(self) -> None: + def _register_groups(self) -> None: """ Register top-level component groups accepted by the CLI. """ - self.add_subgroup( + self.add_group( name='input', prefix=generate_env_prefix('INPUT'), - help_text='Configure an input source: --input TYPE OPTIONS' + help_text='Configure an input source with -C input.NAME.PROPERTY=VALUE' ) - self.add_subgroup( + self.add_group( name='output', prefix=generate_env_prefix('OUTPUT'), - help_text='Configure an output destination: --output TYPE OPTIONS' + help_text='Configure an output destination with -C output.NAME.PROPERTY=VALUE' ) - self.add_subgroup( + self.add_group( name='pre-processor', prefix=generate_env_prefix('PRE_PROCESSOR'), - help_text='Configure a pre-processor: --pre-processor TYPE OPTIONS' + help_text='Configure a pre-processor with -C pre-processor.NAME.PROPERTY=VALUE' ) - self.add_subgroup( + self.add_group( name='post-processor', prefix=generate_env_prefix('POST_PROCESSOR'), - help_text='Configure a post-processor: --post-processor TYPE OPTIONS' + help_text='Configure a post-processor with -C post-processor.NAME.PROPERTY=VALUE' ) def _register_root_arguments(self) -> None: @@ -163,328 +151,316 @@ def _register_root_arguments(self) -> None: Register root-level options that apply to the whole PowerAPI process. """ self.add_argument( - 'v', 'verbose', + 'verbose', is_flag=True, - action=store_true, default_value=False, help_text='Enable verbose logging', ) self.add_argument( - 's', 'stream', + 'stream', is_flag=True, - action=store_true, default_value=False, help_text='Enable stream processing mode', ) - def _register_input_parsers(self): + def _register_input_schemas(self): """ - Register all built-in input source parsers. + Register all built-in input source schemas. """ - self._register_mongodb_input_parser() - self._register_socket_input_parser() - self._register_csv_input_parser() - self._register_json_input_parser() + self._register_mongodb_input_schema() + self._register_socket_input_schema() + self._register_csv_input_schema() + self._register_json_input_schema() - def _register_mongodb_input_parser(self): + def _register_mongodb_input_schema(self): """ - Register the MongoDB input parser. + Register the MongoDB input schema. """ - subparser_mongo_input = PullerConfigParsingManager('mongodb') + schema_mongo_input = PullerSchema('mongodb') - subparser_mongo_input.add_argument( - 'u', 'uri', + schema_mongo_input.add_argument( + 'uri', help_text='MongoDB connection URI', is_mandatory=True ) - subparser_mongo_input.add_argument( - 'd', 'db', + schema_mongo_input.add_argument( + 'db', help_text='MongoDB database name', is_mandatory=True ) - subparser_mongo_input.add_argument( - 'c', 'collection', + schema_mongo_input.add_argument( + 'collection', help_text='MongoDB collection name', is_mandatory=True ) - self.add_subgroup_parser('input', subparser_mongo_input) + self.add_component('input', schema_mongo_input) - def _register_socket_input_parser(self): + def _register_socket_input_schema(self): """ - Register the Socket input parser. + Register the Socket input schema. """ - subparser_socket_input = PullerConfigParsingManager('socket') + schema_socket_input = PullerSchema('socket') - subparser_socket_input.add_argument( - 'h', 'host', + schema_socket_input.add_argument( + 'host', help_text='Host address the socket listens on', default_value='localhost' ) - subparser_socket_input.add_argument( - 'p', 'port', + schema_socket_input.add_argument( + 'port', help_text="Port number the socket listens on", argument_type=int, default_value=9080, ) - self.add_subgroup_parser('input', subparser_socket_input) + self.add_component('input', schema_socket_input) - def _register_csv_input_parser(self): + def _register_csv_input_schema(self): """ - Register the CSV input parser. + Register the CSV input schema. """ - subparser_csv_input = PullerConfigParsingManager('csv') + schema_csv_input = PullerSchema('csv') - subparser_csv_input.add_argument( - 'f', 'files', + schema_csv_input.add_argument( + 'files', help_text='Comma-separated list of CSV input files', argument_type=list, is_mandatory=True ) - self.add_subgroup_parser('input', subparser_csv_input) + self.add_component('input', schema_csv_input) - def _register_json_input_parser(self): + def _register_json_input_schema(self): """ - Register the JSON input parser. + Register the JSON input schema. """ - subparser_json_input = PullerConfigParsingManager('json') + schema_json_input = PullerSchema('json') - subparser_json_input.add_argument( - 'f', 'filepath', + schema_json_input.add_argument( + 'filepath', help_text='Path to the JSON input file', is_mandatory=True ) - subparser_json_input.add_argument( - 'c', 'compression', + schema_json_input.add_argument( + 'compression', help_text='Input compression format: auto, gzip, lzma, or none', default_value='auto' ) - self.add_subgroup_parser('input', subparser_json_input) + self.add_component('input', schema_json_input) - def _register_output_parsers(self): + def _register_output_schemas(self): """ - Register all built-in output destination parsers. + Register all built-in output destination schemas. """ - self._register_mongodb_output_parser() - self._register_prometheus_output_parser() - self._register_csv_output_parser() - self._register_json_output_parser() - self._register_influxdb2_output_parser() - self._register_clickhouse_output_parser() + self._register_mongodb_output_schema() + self._register_prometheus_output_schema() + self._register_csv_output_schema() + self._register_json_output_schema() + self._register_influxdb2_output_schema() + self._register_clickhouse_output_schema() - def _register_mongodb_output_parser(self): + def _register_mongodb_output_schema(self): """ - Register the MongoDB output parser. + Register the MongoDB output schema. """ - subparser_mongo_output = PusherConfigParsingManager('mongodb') + schema_mongo_output = PusherSchema('mongodb') - subparser_mongo_output.add_argument( - 'u', 'uri', + schema_mongo_output.add_argument( + 'uri', help_text='MongoDB connection URI', is_mandatory=True ) - subparser_mongo_output.add_argument( - 'd', 'db', + schema_mongo_output.add_argument( + 'db', help_text='MongoDB database name', is_mandatory=True ) - subparser_mongo_output.add_argument( - 'c', 'collection', + schema_mongo_output.add_argument( + 'collection', help_text='MongoDB collection name', is_mandatory=True ) - self.add_subgroup_parser('output', subparser_mongo_output) + self.add_component('output', schema_mongo_output) - def _register_prometheus_output_parser(self): + def _register_prometheus_output_schema(self): """ - Register the Prometheus output parser. + Register the Prometheus output schema. """ - subparser_prometheus_output = PusherConfigParsingManager('prometheus') + schema_prometheus_output = PusherSchema('prometheus') - subparser_prometheus_output.add_argument( - 'u', 'addr', + schema_prometheus_output.add_argument( + 'addr', help_text='Host address the Prometheus HTTP server listens on', default_value='localhost' ) - subparser_prometheus_output.add_argument( - 'p', 'port', + schema_prometheus_output.add_argument( + 'port', help_text='Port number the Prometheus HTTP server listens on', argument_type=int, default_value=8000 ) - subparser_prometheus_output.add_argument( - 'M', 'metric-name', - help_text='Prometheus metric name to expose', - default_value='power_estimation_watts' - ) - subparser_prometheus_output.add_argument( - 'd', 'metric-description', - help_text='Prometheus metric description', - default_value='Estimated power consumption of the target' - ) - subparser_prometheus_output.add_argument( - 't', 'tags', + schema_prometheus_output.add_argument( + 'tags', help_text='Comma-separated list of report metadata fields exposed as metric labels', argument_type=list ) - self.add_subgroup_parser('output', subparser_prometheus_output) + self.add_component('output', schema_prometheus_output) - def _register_csv_output_parser(self): + def _register_csv_output_schema(self): """ - Register the CSV output parser. + Register the CSV output schema. """ - subparser_csv_output = PusherConfigParsingManager('csv') + schema_csv_output = PusherSchema('csv') - subparser_csv_output.add_argument( - 'd', 'directory', + schema_csv_output.add_argument( + 'directory', help_text='Directory where CSV output files are written', is_mandatory=True ) - self.add_subgroup_parser('output', subparser_csv_output) + self.add_component('output', schema_csv_output) - def _register_json_output_parser(self): + def _register_json_output_schema(self): """ - Register the JSON output parser. + Register the JSON output schema. """ - subparser_json_output = PusherConfigParsingManager('json') + schema_json_output = PusherSchema('json') - subparser_json_output.add_argument( - 'f', 'filepath', + schema_json_output.add_argument( + 'filepath', help_text='Path to the JSON output file', is_mandatory=True ) - subparser_json_output.add_argument( - 'c', 'compression', + schema_json_output.add_argument( + 'compression', help_text='Output compression format: auto, gzip, lzma, or none', default_value='auto' ) - self.add_subgroup_parser('output', subparser_json_output) + self.add_component('output', schema_json_output) - def _register_influxdb2_output_parser(self): + def _register_influxdb2_output_schema(self): """ - Register the InfluxDB 2 output parser. + Register the InfluxDB 2 output schema. """ - subparser_influx2_output = PusherConfigParsingManager('influxdb2') + schema_influx2_output = PusherSchema('influxdb2') - subparser_influx2_output.add_argument( - 'u', 'uri', + schema_influx2_output.add_argument( + 'uri', help_text='InfluxDB server URI', is_mandatory=True ) - subparser_influx2_output.add_argument( - 'k', 'token', + schema_influx2_output.add_argument( + 'token', help_text='InfluxDB API token', is_mandatory=True ) - subparser_influx2_output.add_argument( - 'g', 'org', + schema_influx2_output.add_argument( + 'org', help_text='InfluxDB organization name', is_mandatory=True ) - subparser_influx2_output.add_argument( - 'b', 'bucket', + schema_influx2_output.add_argument( + 'bucket', help_text='InfluxDB bucket name', is_mandatory=True ) - self.add_subgroup_parser('output', subparser_influx2_output) + self.add_component('output', schema_influx2_output) - def _register_clickhouse_output_parser(self): + def _register_clickhouse_output_schema(self): """ - Register the ClickHouse output parser. + Register the ClickHouse output schema. """ - subparser_clickhouse_output = PusherConfigParsingManager('clickhouse') + schema_clickhouse_output = PusherSchema('clickhouse') - subparser_clickhouse_output.add_argument( - 'h', 'host', + schema_clickhouse_output.add_argument( + 'host', help_text='ClickHouse server host', is_mandatory=True, ) - subparser_clickhouse_output.add_argument( - 'p', 'port', + schema_clickhouse_output.add_argument( + 'port', help_text='ClickHouse server port', argument_type=int, default_value=8123, ) - subparser_clickhouse_output.add_argument( - 'u', 'username', + schema_clickhouse_output.add_argument( + 'username', help_text='ClickHouse username', default_value='default', ) - subparser_clickhouse_output.add_argument( - 'P', 'password', + schema_clickhouse_output.add_argument( + 'password', help_text='ClickHouse password', default_value='', ) - subparser_clickhouse_output.add_argument( - 'd', 'database', + schema_clickhouse_output.add_argument( + 'database', help_text='ClickHouse database name', default_value='default', ) - self.add_subgroup_parser('output', subparser_clickhouse_output) + self.add_component('output', schema_clickhouse_output) - def _register_pre_processor_parsers(self): + def _register_pre_processor_schemas(self): """ - Register all built-in pre-processor parsers. + Register all built-in pre-processor schemas. """ - self._register_k8s_pre_processor_parser() - self._register_openstack_pre_processor_parser() + self._register_k8s_pre_processor_schema() + self._register_openstack_pre_processor_schema() - def _register_k8s_pre_processor_parser(self): + def _register_k8s_pre_processor_schema(self): """ - Register the Kubernetes pre-processor parser. + Register the Kubernetes pre-processor schema. """ - subparser_k8s_pre_processor = PreProcessorConfigParsingManager('k8s') + schema_k8s_pre_processor = PreProcessorSchema('kubernetes') - subparser_k8s_pre_processor.add_argument( - 'a', 'api-mode', + schema_k8s_pre_processor.add_argument( + 'api-mode', help_text='Kubernetes API access mode: local, manual, or cluster', default_value='cluster' ) - subparser_k8s_pre_processor.add_argument( - 'k', 'api-key', + schema_k8s_pre_processor.add_argument( + 'api-key', help_text='Kubernetes bearer token for manual API mode', ) - subparser_k8s_pre_processor.add_argument( - 'h', 'api-host', + schema_k8s_pre_processor.add_argument( + 'api-host', help_text='Kubernetes API host for manual API mode', ) - subparser_k8s_pre_processor.add_argument( - 'l', 'labels', + schema_k8s_pre_processor.add_argument( + 'labels', help_text='Comma-separated list of Kubernetes pod labels added to reports as metadata', argument_type=list ) - self.add_subgroup_parser('pre-processor', subparser_k8s_pre_processor) + self.add_component('pre-processor', schema_k8s_pre_processor) - def _register_openstack_pre_processor_parser(self): + def _register_openstack_pre_processor_schema(self): """ - Register the OpenStack pre-processor parser. + Register the OpenStack pre-processor schema. """ - subparser_openstack_pre_processor = PreProcessorConfigParsingManager('openstack') + schema_openstack_pre_processor = PreProcessorSchema('openstack') - subparser_openstack_pre_processor.add_argument( - 'i', "polling-interval", + schema_openstack_pre_processor.add_argument( + 'polling-interval', help_text='OpenStack API polling interval in seconds', argument_type=float, default_value=10.0 ) - subparser_openstack_pre_processor.add_argument( - 'm', 'metadata', + schema_openstack_pre_processor.add_argument( + 'metadata', help_text='Comma-separated list of OpenStack server metadata fields added to reports', argument_type=list ) - self.add_subgroup_parser('pre-processor', subparser_openstack_pre_processor) + self.add_component('pre-processor', schema_openstack_pre_processor) diff --git a/src/powerapi/cli/config_loader.py b/src/powerapi/cli/config_loader.py new file mode 100644 index 00000000..a5f0b8a5 --- /dev/null +++ b/src/powerapi/cli/config_loader.py @@ -0,0 +1,170 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +import os +from collections.abc import Iterable + +from powerapi.cli.config_parser import ( + ComponentGroupSchema, + ConfigurationSchema, +) +from powerapi.exception import ConfigurationError + +_EnvironmentItems = Iterable[tuple[str, str]] + + +class JSONConfigLoader: + """ + Load partial configuration values from a JSON file. + """ + + def load(self, file_name: str | None) -> dict: + """ + Load a JSON configuration. + :param file_name: Path of the JSON file, or None when no configuration file was selected. + :return: The loaded configuration, or an empty dictionary when no file was selected. + :raises FileNotFoundError: If the selected file does not exist. + :raises ConfigurationError: If the input file is invalid. + """ + if file_name is None: + return {} + + try: + with open(file_name, encoding='utf-8') as config_file: + configuration = json.load(config_file) + except json.JSONDecodeError as error: + raise ConfigurationError(f'Invalid JSON in configuration file "{file_name}": {error}') from error + + if not isinstance(configuration, dict): + raise ConfigurationError('Expected a JSON object') + + return configuration + + +class EnvironmentConfigLoader: + """ + Load configuration values using the established PowerAPI environment format. + """ + + def __init__(self, schema: ConfigurationSchema) -> None: + """ + Initialize an environment configuration loader. + :param schema: Schema describing the accepted root, section, and component properties. + """ + self._schema = schema + + def load(self) -> dict: + """ + Load raw configuration values from the environment. + :return: The configuration extracted from the current process environment. + """ + environment = tuple(os.environ.items()) + configuration = self._load_root_configuration(environment) + + for group_name, group in self._schema.groups.items(): + if not group.prefix: + continue + + group_configuration = self._extract_group_values(group, environment) + if group_configuration: + configuration[group_name] = group_configuration + + return configuration + + def _load_root_configuration(self, environment: _EnvironmentItems) -> dict: + """ + Load root configuration values from an environment snapshot. + :param environment: Environment variable names and values. + :return: Raw root configuration values. + """ + configuration = {} + group_prefixes = [group.prefix for group in self._schema.groups.values() if group.prefix] + + for prefix in self._schema.arguments_prefix: + root_values = self._extract_root_values(prefix, group_prefixes, environment) + configuration.update(root_values) + + return configuration + + def _extract_root_values(self, prefix: str, group_prefixes: list[str], environment: _EnvironmentItems) -> dict: + """ + Extract root properties belonging to one environment prefix. + :param prefix: Prefix identifying root configuration variables. + :param group_prefixes: Prefixes reserved for component groups. + :param environment: Environment variable names and values. + :return: Raw root configuration values. + """ + values = {} + for variable_name, value in environment: + if not variable_name.startswith(prefix): + continue + + if any(variable_name.startswith(group_prefix) for group_prefix in group_prefixes): + continue + + property_name = self._normalize_name(variable_name[len(prefix):]) + values[property_name] = value + + return values + + def _extract_group_values(self, group: ComponentGroupSchema, environment: _EnvironmentItems) -> dict: + """ + Extract entry properties belonging to one environment group. + :param group: Schema of the configuration group. + :param environment: Environment variable names and values. + :return: Raw configurations indexed by component or section name. + """ + values = {} + normalized_names = sorted((self._normalize_name(name) for name in [*group.get_argument_names(), 'type']), key=len, reverse=True) + + for variable_name, value in environment: + if not variable_name.startswith(group.prefix): + continue + + suffix = self._normalize_name(variable_name[len(group.prefix):]) + for property_name in normalized_names: + marker = f'{self._schema.default_separator_args_names}{property_name}' + if not suffix.endswith(marker): + continue + + component_name = suffix[:-len(marker)] + if component_name: + values.setdefault(component_name, {})[property_name] = value + + break + + return values + + def _normalize_name(self, name: str) -> str: + """ + Convert an environment name fragment to its configuration spelling. + :param name: Environment name fragment. + :return: Lowercase configuration name using the configured argument separator. + """ + return name.lower().replace(self._schema.default_separator_env_vars_names, self._schema.default_separator_args_names) diff --git a/src/powerapi/cli/config_parser.py b/src/powerapi/cli/config_parser.py index 8cd976a1..62286d76 100644 --- a/src/powerapi/cli/config_parser.py +++ b/src/powerapi/cli/config_parser.py @@ -27,752 +27,359 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import getopt -import json -import os -import sys -from collections.abc import Callable +from copy import deepcopy +from dataclasses import dataclass from typing import Any -from powerapi.exception import AlreadyAddedArgumentException, UnknownArgException, \ - MissingValueException, BadContextException, TooManyArgumentNamesException, NoNameSpecifiedForSubgroupException, \ - SubgroupAlreadyExistException, SubgroupParserWithoutNameArgumentException, BadTypeException, \ - MissingArgumentException, SameLengthArgumentNamesException, InvalidPrefixException, RepeatedArgumentException, \ - SubgroupDoesNotExistException, AlreadyAddedSubgroupException -from ._utils import find_longest_string_in_list, string_to_bool, to_lower_case_and_replace_separators, \ - get_longest_related_suffix, string_to_list +from powerapi.exception import ConfigurationError +from ._utils import string_to_bool, string_to_list -def store_val(argument_name: str, val: Any, configuration: dict, args: list | None = None) -> (list, dict): - """ - Action that stores the value of the argument on the parser result - """ - if val == '': - val = None - - configuration[argument_name] = val - return args, configuration - -def store_true(argument_name: str, configuration: dict, val: Any = None, args: list | None = None) -> (list, dict): +@dataclass(frozen=True) +class ArgumentDefinition: """ - Action that stores a True boolean value on the parser result + Definition of a configuration property. """ - configuration[argument_name] = True - return args, configuration + name: str + is_flag: bool = False + default_value: Any = None + help_text: str = '' + argument_type: type[Any] = str + is_mandatory: bool = False -class ConfigurationArgument: +class ConfigurationSectionSchema: """ - Argument provided by a formula configuration. + Schema for a flat set of configuration properties. """ - def __init__(self, names: list, is_flag: bool, default_value: Any, help_text: str, argument_type: type, - is_mandatory: bool, action: Callable | None = None): - self.names = names - self.is_flag = is_flag - self.default_value = default_value - self.help_text = help_text - self.type = argument_type - self.is_mandatory = is_mandatory - self.action = action - - def __eq__(self, arg): - names_ok = True - - for current_name in self.names: - names_ok = current_name in arg.names - - return names_ok and len(self.names) == len(arg.names) and self.is_flag == arg.is_flag and \ - self.type == arg.type and self.default_value == arg.default_value and \ - self.help_text == arg.help_text and self.is_mandatory == arg.is_mandatory - - -class BaseConfigParser: + def __init__(self) -> None: + self.arguments: dict[str, ArgumentDefinition] = {} + + def add_argument( + self, + name: str, + *, + is_flag: bool = False, + default_value: Any = None, + help_text: str = '', + argument_type: type = str, + is_mandatory: bool = False, + ) -> None: + """ + Register a configuration property. + :param name: Configuration property name. + :param is_flag: Whether the property represents a boolean flag. + :param default_value: Value used when the property is omitted. + :param help_text: User-facing description of the property. + :param argument_type: Type used to cast non-flag values. + :param is_mandatory: Whether the property must be defined. + :raises ValueError: If the property name is already registered. + """ + if name in self.arguments: + raise ValueError(f'Configuration property "{name}" is already registered') + + self.arguments[name] = ArgumentDefinition( + name=name, + is_flag=is_flag, + default_value=default_value, + help_text=help_text, + argument_type=bool if is_flag else argument_type, + is_mandatory=is_mandatory, + ) + + def validate(self, conf: dict, path: str = '') -> dict: + """ + Cast properties, require mandatory values, and apply defaults. + :param conf: Partial configuration to validate. + :param path: Dotted path prepended to configuration errors. + :return: Validated configuration with defaults applied. + :raises ConfigurationError: If the configuration is not a dictionary or contains an unknown, missing, or incorrectly typed property. + """ + if not isinstance(conf, dict): + raise ConfigurationError('Expected dict', path or None) + + validated = {} + for name, value in conf.items(): + if name not in self.arguments: + raise ConfigurationError('Unknown property', _join_path(path, name)) + + validated[name] = cast_argument_value(_join_path(path, name), value, self.arguments[name]) + + definitions = self.arguments.values() + for argument in definitions: + name = argument.name + if name not in validated: + if argument.is_mandatory: + raise ConfigurationError('Missing required value', _join_path(path, name)) + if argument.default_value is not None: + validated[name] = deepcopy(argument.default_value) + + return validated + + +class ComponentSchema(ConfigurationSectionSchema): """ - Base class for configuration parsers. + Schema for one component type in a configuration group. """ - def __init__(self): - self.arguments = {} - - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False): - """add an optional argument to the parser that will activate an action - - :param str names: names of the optional argument that will be bind to - the action (could be long or short name) - - :param bool is_flag: True if the argument doesn't require to be followed - by a value - - :param Callable action: action that will be executed when the argument is - caught by the parser. the lambda take 4 parameters - (the name of the argument caught by the parser, - the value attached to this argument, the current - list of arguments that is parsed by the parser and - the parser result) and return a list of token and - a parser result(dict) - - :param default_value: the default value attached to this argument - - :param str help_text: text that describe the argument - - :param type argument_type: type of the value that the argument must catch - - :param bool is_mandatory: True if the argument is required - - :raise AlreadyAddedArgumentException: when attempting to add an - argument that already have been - added to this parser - - """ - for name in names: - if name in self.arguments: - raise AlreadyAddedArgumentException(name) - - argument = ConfigurationArgument(names=list(names), is_flag=is_flag, action=action, - default_value=default_value, help_text=help_text, - argument_type=argument_type, is_mandatory=is_mandatory) - - for name in names: - self.arguments[name] = argument - - def _get_default_arguments_values(self): - default_values = {} - for _, argument in self.arguments.items(): - if argument.default_value is not None: - argument_name = find_longest_string_in_list(argument.names) - if argument_name not in default_values: - default_values[argument_name] = argument.default_value - - return default_values - - def _get_default_argument_value(self, argument_name: str) -> dict: + def __init__(self, name: str) -> None: """ - Return a dict containing the default value for the given argument or an empty dict - if it does not have one or if it does not exist. - :param str argument_name : The name of the argument + Initialize a component schema. + :param name: Component type handled by this schema. """ - default_values_dict = {} - if argument_name in self.arguments: - argument = self.arguments[argument_name] - if argument.default_value is not None: - longest_argument_name = find_longest_string_in_list(argument.names) - default_values_dict[longest_argument_name] = argument.default_value - - return default_values_dict - - def _get_arguments_str(self, indent: str) -> str: - already_added_argument = [] - arguments_str_representation = '' - for _, argument in self.arguments.items(): - if argument not in already_added_argument: - arguments_str_representation += indent + ', '.join( - map(lambda x: '-' + x if len(x) == 1 else '--' + x, argument.names)) - arguments_str_representation += ' : ' + argument.help_text + '\n' - already_added_argument.append(argument) - return arguments_str_representation - - def _unknown_argument_behaviour(self, arg_name: str, val: Any, args: list, configuration: dict): - raise NotImplementedError() - - def _parse(self, args: list, configuration: dict) -> (list, dict): - - while args: - arg, val = args.pop(0) - if arg not in self.arguments: - args.insert(0, (arg, val)) - return self._unknown_argument_behaviour(arg, val, args, configuration) - - argument = self.arguments[arg] - - arg_long_name = find_longest_string_in_list(argument.names) - val = cast_argument_value(arg_long_name, val, argument) - - args, configuration = argument.action(argument_name=arg_long_name, val=val, args=args, - configuration=configuration) - - return args, configuration - - def _get_mandatory_arguments(self) -> list: - """ - Return the list of mandatory arguments - """ - mand_args = [] - for _, argument in self.arguments.items(): - if argument.is_mandatory and argument not in mand_args: - mand_args.append(argument) - return mand_args - - def get_arguments(self): - """ Get the parser arguments """ - return self.arguments - - def get_longest_arguments_names(self) -> list: - """ - Return a list with the longest names of the different arguments - """ - long_arguments_names = [] - for _, argument in self.arguments.items(): - longest_name = find_longest_string_in_list(argument.names) - if longest_name not in long_arguments_names: - long_arguments_names.append(longest_name) - - return long_arguments_names - - def validate(self, conf: dict) -> dict: - """ - Check that mandatory arguments are present in the provided configuration. - Check that arguments are not repeated in the provided configuration. - Check that arguments are recognized by the parser. - It also defines default values if any for arguments that are not defined in the configuration - - """ - # Check that all the mandatory arguments are present - mandatory_args = self._get_mandatory_arguments() - for arg in mandatory_args: - is_present = False - for arg_name in arg.names: - if arg_name in conf: - is_present = True - break - if not is_present: - raise MissingArgumentException(str(arg.names)) - - # Define default values for no defined arguments if they are specified - for argument_name, argument_definition in self.arguments.items(): - is_defined = False - for name in argument_definition.names: - if name in conf: - is_defined = True - break - if not is_defined and argument_definition.default_value is not None: - conf[argument_name] = argument_definition.default_value - - # Check that arguments are not repeated and that they exist - present_arguments = [] - for current_argument_name in conf: - if current_argument_name in self.arguments: - argument = self.arguments[current_argument_name] - if argument in present_arguments: - raise RepeatedArgumentException(argument_name=current_argument_name) - present_arguments.append(argument) - elif current_argument_name != 'type': - raise UnknownArgException(argument_name=current_argument_name) - - return self.normalize_configuration(conf=conf) - - def normalize_configuration(self, conf: dict) -> dict: - """ - Return a configuration dict that has all the arguments' names in the long form. - If an argument does not exist, a UnknownArgException is raised - If an argument is repeated, a RepeatedArgumentException is raised - """ - conf_with_long_names = {} - - for current_argument_name in conf: - if current_argument_name not in self.arguments and current_argument_name != 'type': - raise UnknownArgException(current_argument_name) - - longest_argument_name = 'type' - if current_argument_name != 'type': - current_argument = self.arguments[current_argument_name] - longest_argument_name = find_longest_string_in_list(current_argument.names) - if longest_argument_name in conf_with_long_names: - raise RepeatedArgumentException(argument_name=longest_argument_name) - conf_with_long_names[longest_argument_name] = conf[current_argument_name] - - return conf_with_long_names - - def cast_arguments_values(self, arguments: dict) -> dict: - """ - Cast to the argument type the different values in the provided dictionary. - The dictionary only contains values with basic types (string, int...) - :param dict arguments: The dictionary with the values to cast - """ - for argument_name, argument_value in arguments.items(): - - if argument_name != 'type': - casted_value = cast_argument_value(val=argument_value, arg_name=argument_name, - argument=self.arguments[argument_name]) - arguments[argument_name] = casted_value - - return arguments + super().__init__() + self.name = name -class SubgroupParserGroup: +class ComponentGroupSchema: """ - Group of subgroup parsers stored in a dictionary. Each subgroup has a name + Schemas for the dynamic components and fixed sections in a configuration group. """ - def __init__(self, group_name: str, help_text: str = '', prefix: str = ''): + def __init__(self, group_name: str, help_text: str = '', prefix: str = '') -> None: """ - Create a group of subgroup parsers - :param str group_name: name of the subgroup - :param str help_text: Help text related to the subgroup - :param str prefix: Prefix related to the group for parsing environment variables + Initialize a configuration group schema. + :param group_name: Configuration name of the group. + :param help_text: User-facing description of the group. + :param prefix: Environment-variable prefix assigned to the group. """ self.group_name = group_name self.help_text = help_text - self.subparsers = {} self.prefix = prefix + self.components: dict[str, ComponentSchema] = {} + self.sections: dict[str, ConfigurationSectionSchema] = {} - def get_prefix(self) -> str: + def get_argument_names(self) -> list[str]: """ - Return the group's prefix + Return all canonical property names accepted by the group. + :return: Canonical component and section property names without duplicates. """ - return self.prefix + names = [] + for component in self.components.values(): + names.extend(component.arguments) + for section in self.sections.values(): + names.extend(section.arguments) - def contains(self, name: str): - """ - Check if the given name belongs to one of the subparser of the grouo - :param str name: Name to look for - """ - return name in self.subparsers + return list(dict.fromkeys(names)) - def add_subgroup_parser(self, name: str, subparser: BaseConfigParser): + def validate(self, conf: dict, path: str) -> dict: """ - Add a subgroup parser to the group - :param str name: Subgroup parser name - :param BaseConfigParser subparser: subparser to be added + Validate the fixed sections and dynamic components in a group. + :param conf: Group configuration to validate. + :param path: Dotted path of the group. + :return: Validated group configuration with section defaults applied. + :raises ConfigurationError: If the group or one of its entries is invalid. """ - self.subparsers[name] = subparser + if not isinstance(conf, dict): + raise ConfigurationError('Expected dict', path) - def get_subgroup_parser(self, name: str) -> BaseConfigParser: - """ - Return the subgroup parser with the given name - :param str name: Subparser name - """ - return self.subparsers[name] + validated = {} + for entry_name, entry_values in conf.items(): + entry_path = _join_path(path, entry_name) + validated[entry_name] = self._validate_entry(entry_name, entry_values, entry_path) - def __iter__(self): - return iter(self.subparsers.items()) + for section_name, section in self.sections.items(): + if section_name in validated: + continue - def get_help(self) -> str: - """ - return help string - """ - help_str = self.group_name + ' details :\n' - for subparser_name, subparser in self.subparsers.items(): - help_str += ' --' + self.group_name + ' ' + subparser_name + ':\n' - help_str += subparser.get_help() - help_str += '\n' + section_values = section.validate({}, _join_path(path, section_name)) + if section_values: + validated[section_name] = section_values - return help_str + return validated - def get_longest_arguments_names(self) -> list: + def _validate_entry(self, name: str, values: dict, path: str) -> dict: """ - Return a list of arguments names from the different parsers that are part of the group + Validate one fixed section or dynamic component entry. + :param name: Name of the section or component instance. + :param values: Entry configuration to validate. + :param path: Dotted path of the entry. + :return: Validated section or component configuration. + :raises ConfigurationError: If the entry structure, type, or properties are invalid. """ - arguments_names = [] - for _, subparser in self.subparsers.items(): - arguments_names.extend(subparser.get_longest_arguments_names()) - return list(set(arguments_names)) + if not isinstance(values, dict): + raise ConfigurationError('Expected dict', path) - def get_group_name(self) -> str: - """ - Return the name of the group - """ - return self.group_name - - -class SubgroupConfigParser(BaseConfigParser): - """ - A parser for a subgroup - """ - - def __init__(self, name: str): - """ - Create a subgroup parser with the given name - :param str name: Name of the subgroup parser - """ - BaseConfigParser.__init__(self) - self.name = name - - def _unknown_argument_behaviour(self, arg_name: str, val: Any, args: list, - configuration: dict): - return args, configuration - - def parse(self, token_list: list) -> (list, dict): - """ - Parse the given token list until an unknown argument is caught + if name in self.sections: + return self.sections[name].validate(values, path) - :param list token_list: the token list currently parsed + if 'type' not in values: + raise ConfigurationError('Missing required value', f'{path}.type') - :return dict: the result of the parsing - - """ - local_result = BaseConfigParser._get_default_argument_value(self, argument_name='name') - if not token_list: - return token_list, local_result - - return self._parse(token_list, local_result) + component_type = values['type'] + try: + schema = self.components[component_type] + except (KeyError, TypeError) as error: + raise ConfigurationError(f'Unknown component type "{component_type}"', f'{path}.type') from error - def get_help(self) -> str: - """ - return help string - """ - return self._get_arguments_str(' ') + component_values = {name: value for name, value in values.items() if name != 'type'} + return { + 'type': component_type, + **schema.validate(component_values, path), + } -class RootConfigParser(BaseConfigParser): +class ConfigurationSchema(ConfigurationSectionSchema): """ - Root configuration parser. + Schema and validation rules for a complete PowerAPI configuration. """ - def __init__(self, help_arg: bool = True, separator_env_vars_names: str = '_', separator_args_names: str = '-'): + def __init__(self, separator_env_vars_names: str = '_', separator_args_names: str = '-') -> None: """ - :param bool help_arg: if True, add a -h/--help argument that display help - :param str separator_env_vars_names: separator for the environment variables names - :param str separator_args_names: separator for arguments with composed names + Initialize a complete configuration schema. + :param separator_env_vars_names: Separator used in environment-variable names. + :param separator_args_names: Separator used in canonical configuration property names. """ - BaseConfigParser.__init__(self) - self.short_arg = '' - self.long_arg = [] - self.subgroup_parsers = {} - - self.arguments_prefix = [] + super().__init__() + self.groups: dict[str, ComponentGroupSchema] = {} + self.arguments_prefix: list[str] = [] self.default_separator_env_vars_names = separator_env_vars_names self.default_separator_args_names = separator_args_names - self.help_arg = help_arg - if help_arg: - self.add_argument('h', 'help', is_flag=True, argument_type=bool) - - def get_help(self): + def add_argument( + self, + name: str, + *, + is_flag: bool = False, + default_value: Any = None, + help_text: str = '', + argument_type: type = str, + is_mandatory: bool = False, + ) -> None: """ - return help string + Register a root configuration property. + :param name: Configuration property name. + :param is_flag: Whether the property represents a boolean flag. + :param default_value: Value used when the property is omitted. + :param help_text: User-facing description of the property. + :param argument_type: Type used to cast non-flag values. + :param is_mandatory: Whether the property must be defined. + :raises ValueError: If the name is already registered as a property or group. """ - s = 'main arguments:\n' - s += self._get_arguments_str(' ') - s += '\n' + if name in self.groups: + raise ValueError(f'Configuration name "{name}" is already registered as a group') - for _, subparser_group in self.subgroup_parsers.items(): - s += subparser_group.get_help() + super().add_argument( + name, + is_flag=is_flag, + default_value=default_value, + help_text=help_text, + argument_type=argument_type, + is_mandatory=is_mandatory, + ) - return s - - def parse(self, args: list) -> dict: + def add_group(self, name: str, help_text: str = '', prefix: str = '') -> None: """ - :param list args: list that contains the arguments and their values - - :return dict: Dictionary that contains the arguments with its associated values extracted from args - - :raise UnknownArgException: when the parser catch an argument that - this parser can't handle - - :raise BadContextException: when an argument that the parser can't - handle in the current context is caught - - :raise MissingValueException: when an argument that require a value is - caught without its value - - :raise BadTypeException: when an argument is parsed with a value of an - incorrect type + Register a top-level configuration group. + :param name: Configuration name of the group. + :param help_text: User-facing description of the group. + :param prefix: Environment-variable prefix assigned to the group. + :raises ValueError: If the name is already registered as a property or group. """ - try: - args, _ = getopt.getopt(args, self.short_arg, self.long_arg) - except getopt.GetoptError as exn: - if 'recognized' in exn.msg: - raise UnknownArgException(exn.opt) from exn - if 'requires' in exn.msg: - raise MissingValueException(exn.opt) from exn - - # Remove `-` and `--` prefix in argument name (`--test-arg` to `test-arg`) - args = [(arg[0].lstrip('-'), arg[1]) for arg in args] - - # verify if help argument exists in args - if self.help_arg: - for arg_name, _ in args: - if arg_name in ('h', 'help'): - print(self.get_help()) - sys.exit(0) - - configuration = {} - - args, configuration = self._parse(args, configuration) - - return configuration + if name in self.groups: + raise ValueError(f'Configuration group "{name}" is already registered') + if name in self.arguments: + raise ValueError(f'Configuration name "{name}" is already registered as a property') - def parse_config_dict(self, file_name: str) -> dict: - """ - Return a configuration dict that has all the arguments' names in the long form. - If an argument does not exist, a UnknownArgException is raised - """ - with open(file_name, encoding='utf-8') as config_file: - conf = json.load(config_file) - return self.normalize_configuration(conf=conf) + self.groups[name] = ComponentGroupSchema(name, help_text, prefix) - def normalize_configuration(self, conf: dict) -> dict: - """ - Return a configuration dict that has all the arguments' names in the long form. - If an argument does not exist, a UnknownArgException is raised - """ - # We normalize the simple arguments names - conf = BaseConfigParser.normalize_configuration(self, conf=conf) - - # We normalize the groups arguments names - - for argument_name, argument_value in conf.items(): - if isinstance(argument_value, dict): - for group_name, group in argument_value.items(): - group = self.subgroup_parsers[argument_name].get_subgroup_parser(name=group['type']). \ - normalize_configuration(conf=group) - argument_value[group_name] = group - - return conf - - def _unknown_argument_behaviour(self, arg_name: str, val: Any, args: list, - configuration: dict): - good_contexts = [] - for main_arg_name, subparser_group in self.subgroup_parsers.items(): - for subparser_name, subparser in subparser_group: - if arg_name in subparser.arguments: - good_contexts.append((main_arg_name, subparser_name)) - raise BadContextException(arg_name, good_contexts) - - def _add_argument_names(self, names: list, is_flag: bool): - - if len(names) > 2: - raise TooManyArgumentNamesException(names[2]) - - if len(names) > 1 and len(names[0]) == len(names[1]): - raise SameLengthArgumentNamesException(names[1]) - - def add_suffix_to_argument_name_if_required(current_name): - if len(current_name) == 1: - return current_name + ('' if is_flag else ':') - return current_name + ('' if is_flag else '=') - - for name in names: - if len(name) == 1: - self.short_arg += add_suffix_to_argument_name_if_required(name) - else: - self.long_arg.append(add_suffix_to_argument_name_if_required(name)) - - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False): - self._add_argument_names(list(names), is_flag) - BaseConfigParser.add_argument(self, *names, is_flag=is_flag, action=action, default_value=default_value, - help_text=help_text, argument_type=argument_type, is_mandatory=is_mandatory) - - def add_subgroup_parser(self, subgroup_type: str, subgroup_parser: SubgroupConfigParser): + def add_component(self, group_name: str, component: ComponentSchema) -> None: """ - Add a subparser that will be used by the argument *group_name* - The group must contain a name action - :param str subgroup_type: the group type - :param SubgroupConfigParser subgroup_parser: The subgroup parser - :param str help_text: help text related to the parser - - :raise AlreadyAddedArgumentException: when attempting to add an - argument that already have been - added to this parser - :raise SubgroupDoesNotExistException If the group related to the paser does not exist + Register a component schema in an existing group. + :param group_name: Group receiving the component schema. + :param component: Component schema to register. + :raises ValueError: If the group is unknown or the component type is already registered in it. """ + if group_name not in self.groups: + raise ValueError(f'Configuration group "{group_name}" is not registered') - if 'name' not in subgroup_parser.arguments: - raise SubgroupParserWithoutNameArgumentException() - if subgroup_type not in self.subgroup_parsers: - raise SubgroupDoesNotExistException(argument_name=subgroup_type) - if self.subgroup_parsers[subgroup_type].contains(subgroup_parser.name): - raise AlreadyAddedArgumentException(subgroup_parser.name) + group = self.groups[group_name] + if component.name in group.components: + raise ValueError(f'Component type "{component.name}" is already registered in group "{group_name}"') - self.subgroup_parsers[subgroup_type].add_subgroup_parser(subgroup_parser.name, subgroup_parser) + group.components[component.name] = component - for action_name, action in subgroup_parser.arguments.items(): - self._add_argument_names([action_name], action.is_flag) - - def add_subgroup(self, subgroup_type: str, help_text: str = '', prefix: str = ''): + def add_section(self, group_name: str, section_name: str, section: ConfigurationSectionSchema) -> None: """ - Add a subgroup that will be used by the argument *group_name* - The group must contain a name action - :param str subgroup_type: the subgroup type - :param str help_text: help text related to the subgroup - :param str prefix: the prefix related to the subgroup - - :raise AlreadyAddedSubgroupException is the subgroup already exists + Register a fixed configuration section in an existing group. + :param group_name: Group receiving the configuration section. + :param section_name: Name identifying and reserving the section in the group. + :param section: Configuration section schema to register. + :raises ValueError: If the group is unknown or the section name is already registered in it. """ + if group_name not in self.groups: + raise ValueError(f'Configuration group "{group_name}" is not registered') - def _action(argument_name: str, val: Any, args: list, configuration: dict): - if argument_name not in configuration: - configuration[argument_name] = {} - - parser = self.subgroup_parsers[argument_name].get_subgroup_parser(val) - args, parse_result = parser.parse(args) - - if 'name' not in parse_result: - raise NoNameSpecifiedForSubgroupException(subgroup_type) + group = self.groups[group_name] + if section_name in group.sections: + raise ValueError(f'Configuration section "{section_name}" is already registered in group "{group_name}"') - subgroup_name = parse_result['name'] - del parse_result['name'] + group.sections[section_name] = section - if subgroup_name in configuration[argument_name]: - raise SubgroupAlreadyExistException(subgroup_name) - - configuration[argument_name][subgroup_name] = parse_result - configuration[argument_name][subgroup_name]['type'] = parser.name - - return args, configuration - - if subgroup_type not in self.subgroup_parsers: - self.subgroup_parsers[subgroup_type] = SubgroupParserGroup(subgroup_type, help_text=help_text, - prefix=prefix) - self.add_argument(subgroup_type, action=_action, help_text=help_text) - - else: - raise AlreadyAddedSubgroupException(subgroup_type) - - def add_argument_prefix(self, argument_prefix: str): + def add_argument_prefix(self, argument_prefix: str) -> None: """ - Add a simple argument prefix to the list if argument_prefix is no prefix of an existing argument prefix or - vice-versa. Otherwise, it raises an InvalidPrefixException - :param argument_prefix: a new argument prefix to be added + Register a non-overlapping root environment-variable prefix. + :param argument_prefix: Environment-variable prefix to register. + :raises ValueError: If the prefix overlaps an existing prefix. """ - for existing_argument_prefix in self.arguments_prefix: - if argument_prefix.startswith(existing_argument_prefix) or \ - existing_argument_prefix.startswith(argument_prefix): - raise InvalidPrefixException(existing_prefix=existing_argument_prefix, new_prefix=argument_prefix) + for existing_prefix in self.arguments_prefix: + if argument_prefix.startswith(existing_prefix) or existing_prefix.startswith(argument_prefix): + raise ValueError(f'Environment prefix "{argument_prefix}" conflicts with "{existing_prefix}"') self.arguments_prefix.append(argument_prefix) - def parse_config_environment_variables(self) -> dict: + def validate(self, conf: dict, path: str = '') -> dict: """ - Parse environment variables to extract a configuration. Then merges the extracted configution with the one - provided as parameter - :param dict current_conf: Configuration to execute the merge + Validate the complete nested configuration against the registered schema. + :param conf: Merged configuration to validate. + :param path: Dotted path prepended to configuration errors. + :return: Canonical validated configuration with defaults applied. + :raises ConfigurationError: If any root, section, or component configuration value is invalid. """ + if not isinstance(conf, dict): + raise ConfigurationError('Expected dict', path or None) - conf = {} - - for current_environment_var_prefix in self.arguments_prefix: - conf = self._extract_simple_environment_variables_with_prefix( - simple_variables_prefix=current_environment_var_prefix, - groups_variables_prefix=self.get_groups_prefixes()) - # We normalize the arguments names - conf = self.normalize_configuration(conf=conf) - - # We cast every value in conf to the correct type - conf = self.cast_arguments_values(arguments=conf) - - groups_names = [] - for _, group in self.subgroup_parsers.items(): + root_values = {name: value for name, value in conf.items() if name not in self.groups} + validated = super().validate(root_values, path) - group_name = group.get_group_name() - group_conf = self._extract_group_environment_variables(group=group) - if len(group_conf) > 0: - conf[group_name] = group_conf - groups_names.append(group_name) + for group_name, group in self.groups.items(): + group_path = _join_path(path, group_name) + validated_group = group.validate(conf.get(group_name, {}), group_path) - for group_name in groups_names: - for subgroup_name, subgroup_arguments in conf[group_name].items(): - # We normalize the names in each group. The argument type for each subgroup - # is required - if 'type' not in subgroup_arguments: - raise MissingArgumentException(argument_name=group_name + '>' + subgroup_name + '>' + 'type') - subgroup_parser = self.subgroup_parsers[group_name].get_subgroup_parser(name=subgroup_arguments['type']) - subgroup_arguments = subgroup_parser.normalize_configuration(conf=subgroup_arguments) + if validated_group or group_name in conf: + validated[group_name] = validated_group - # We cast every value in subgroup_arguments to the correct type - subgroup_arguments = subgroup_parser.cast_arguments_values(arguments=subgroup_arguments) - conf[group_name][subgroup_name] = subgroup_arguments + return validated - return conf - def get_groups_prefixes(self) -> list: - """ - Return a list with the prefixes of different groups - """ - prefixes = [] - for _, subgroup in self.subgroup_parsers.items(): - prefixes.append(subgroup.get_prefix()) - - return prefixes - - def _extract_group_name_from_prefix(self, prefix: str) -> str: - """ - Extract the group name from the given prefix. It assumes the words in the prefix - are separated by self.default_separator_env_vars_names and the group name is composed by a single word - - """ - return prefix.split(self.default_separator_env_vars_names)[1].lower() # [, , ''] - - def _extract_simple_environment_variables_with_prefix(self, simple_variables_prefix: str, - groups_variables_prefix: list) -> dict: - """ - Extract from environment variables the ones starting with prefix and that do not belong to groups. - The returned dictionary contains the variables names as keys without prefix and in lower case - :param str simple_variables_prefix: Prefix to extract the simple environment variables - :param list groups_variables_prefix: List of group prefix for identifying simple variables - """ - simple_variables_with_prefix = {} - for var_name in os.environ: - is_group_variable = False - for group_variable_prefix in groups_variables_prefix: - if var_name.startswith(group_variable_prefix): - is_group_variable = True - break - if not is_group_variable and var_name.startswith(simple_variables_prefix): - var_name_without_prefix = var_name[len(simple_variables_prefix) - len(var_name):]. \ - lower().replace(self.default_separator_env_vars_names, self.default_separator_args_names) - simple_variables_with_prefix[var_name_without_prefix] = os.environ[var_name] - return simple_variables_with_prefix - - def _extract_group_environment_variables(self, group: SubgroupParserGroup) -> dict: - """ - Extract from environment variables the ones starting with group_prefix. - The returned dictionary contains the variables names as keys without prefix and in lower case - :param group: The subgroup related to the environment variables to be extracted - """ - group_variables_with_prefix = {} - group_prefix = group.get_prefix() - subgroups_arguments_names = group.get_longest_arguments_names() - subgroups_arguments_names.append('type') - subgroups_arguments_names = to_lower_case_and_replace_separators(subgroups_arguments_names, - self.default_separator_env_vars_names, - self.default_separator_args_names) - for environ_var_name in os.environ: - if environ_var_name.startswith(group_prefix): - # We remove the prefix and put the name in lower case - suffix_environ_var_name = environ_var_name[len(group_prefix) - len(environ_var_name):]. \ - lower().replace(self.default_separator_env_vars_names, self.default_separator_args_names) - - # We look for a related argument name in the subgroup - group_variable_name_lower_case = get_longest_related_suffix(suffix_environ_var_name, - subgroups_arguments_names) - - if group_variable_name_lower_case and \ - suffix_environ_var_name[ - suffix_environ_var_name.rfind(group_variable_name_lower_case) - 1] == \ - self.default_separator_args_names: - - # The subgroup's name is at the beginning of the suffix - subgroup_name = suffix_environ_var_name[0: - suffix_environ_var_name.rfind( - group_variable_name_lower_case) - 1] - - if subgroup_name not in group_variables_with_prefix: - group_variables_with_prefix[subgroup_name] = {} - group_variables_with_prefix[subgroup_name][group_variable_name_lower_case] = os.environ[ - environ_var_name] - - return group_variables_with_prefix - - -def cast_argument_value(arg_name: str, val: Any, argument: ConfigurationArgument): +def cast_argument_value(path: str, value: Any, argument: ArgumentDefinition) -> Any: """ - Cast the given value to argument.type - :param str arg_name: The argument name - :param Any val: Current value given to the argument - :param ConfigurationAgument argument: The argument definition + Cast one value according to its property definition. + :param path: Dotted path of the property being cast. + :param value: Raw configuration value. + :param argument: Definition describing the expected type. + :return: Value cast to the declared type. + :raises ConfigurationError: If the value cannot be cast to the declared type. """ try: - if argument.type is bool and val is not None and isinstance(val, str): - return string_to_bool(val) - - if argument.type is list and val is not None and isinstance(val, str): - return string_to_list(val) - - return argument.type(val) - except ValueError as exn: - raise BadTypeException(arg_name, argument.type) from exn + if isinstance(value, argument.argument_type): + return value + if argument.argument_type is bool and isinstance(value, str): + return string_to_bool(value) + if argument.argument_type is list and isinstance(value, str): + return string_to_list(value) + return argument.argument_type(value) + except (TypeError, ValueError) as error: + raise ConfigurationError(f'Expected {argument.argument_type.__name__}', path) from error + + +def _join_path(path: str, property_name: str) -> str: + """ + Append a property name to a configuration path. + :param path: Existing dotted configuration path, or an empty string for the root. + :param property_name: Property segment to append. + :return: Combined dotted path. + """ + return f'{path}.{property_name}' if path else property_name diff --git a/src/powerapi/cli/config_validator.py b/src/powerapi/cli/config_validator.py deleted file mode 100644 index dfdd0869..00000000 --- a/src/powerapi/cli/config_validator.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import logging -import os - -from powerapi.exception import MissingArgumentException, NotAllowedArgumentValueException, FileDoesNotExistException, \ - UnexistingActorException - - -class ConfigValidator: - """ - Validate powerapi config and initialize missing default values - """ - - @staticmethod - def validate(config: dict): - """ - Validate powerapi config and initialize missing default values - """ - if 'verbose' not in config: - config['verbose'] = logging.NOTSET - if 'stream' not in config: - config['stream'] = False - if 'output' not in config: - logging.error("no output configuration found") - raise MissingArgumentException(argument_name='output') - - if 'input' not in config: - logging.error("no input configuration found") - raise MissingArgumentException(argument_name='input') - - for input_id in config['input']: - input_config = config['input'][input_id] - if input_config['type'] == 'csv' \ - and ( - 'files' not in input_config or input_config['files'] is None or len(input_config['files']) == 0): - logging.error("no files parameter found for csv input") - raise MissingArgumentException(argument_name='files') - - if input_config['type'] == 'csv' and config['stream']: - logging.error("stream mode cannot be used for csv input") - raise NotAllowedArgumentValueException("Stream mode cannot be used for csv input") - - if 'pre-processor' in config: - for pre_processor_id in config['pre-processor']: - pre_processor_config = config['pre-processor'][pre_processor_id] - - if 'puller' not in pre_processor_config: - logging.error("No puller name found for pre-processor: %s ", pre_processor_id) - raise MissingArgumentException(argument_name='puller') - - puller_id = pre_processor_config['puller'] - - if puller_id not in config['input']: - logging.error("Puller actor '%s' does not exist", puller_id) - raise UnexistingActorException(actor=puller_id) - - elif 'post-processor' in config: - for post_processor_id in config['post-processor']: - post_processor_config = config['post-processor'][post_processor_id] - - if 'pusher' not in post_processor_config: - logging.error("No pusher name found for post-processor: %s", post_processor_id) - raise MissingArgumentException(argument_name='pusher') - - pusher_id = post_processor_config['pusher'] - - if pusher_id not in config['output']: - logging.error("Pusher actor '%s' does not exist", pusher_id) - raise UnexistingActorException(actor=pusher_id) - - ConfigValidator._validate_input(config) - - @staticmethod - def _validate_input(config: dict): - """ - Check that csv input type has files that exist - """ - for input_config in config['input'].values(): - if input_config['type'] == 'csv': - for file_name in input_config['files']: - if not os.access(file_name, os.R_OK): - raise FileDoesNotExistException(file_name=file_name) - - @staticmethod - def _validate_binding(config: dict): - """ - Check that defined bindings use existing actors defined by the configuration - """ - for _, binding_infos in config['binding'].items(): - - if 'from' not in binding_infos: - logging.error("no from parameter found for binding") - raise MissingArgumentException(argument_name='from') - - if 'to' not in binding_infos: - logging.error("no to parameter found for binding") - raise MissingArgumentException(argument_name='to') - - # from_info[0] is the subgroup and from_info[1] the actor name - from_infos = binding_infos['from'].split('.') - - if from_infos[0] not in config or from_infos[1] not in config[from_infos[0]]: - logging.error("from actor does not exist") - raise UnexistingActorException(actor=binding_infos['from']) - - # to_info[0] is the subgroup and to_info[1] the actor name - to_infos = binding_infos['to'].split('.') - - if to_infos[0] not in config or to_infos[1] not in config[to_infos[0]]: - logging.error("to actor does not exist") - raise UnexistingActorException(actor=binding_infos['to']) diff --git a/src/powerapi/cli/generator.py b/src/powerapi/cli/generator.py index b9d91b8e..2c938be8 100644 --- a/src/powerapi/cli/generator.py +++ b/src/powerapi/cli/generator.py @@ -426,7 +426,7 @@ class PreProcessorGenerator(ProcessorGenerator): def __init__(self): super().__init__('pre-processor') - self.add_processor_factory('k8s', self._k8s_pre_processor_factory) + self.add_processor_factory('kubernetes', self._k8s_pre_processor_factory) self.add_processor_factory('openstack', self._openstack_pre_processor_factory) @staticmethod diff --git a/src/powerapi/cli/parsing_manager.py b/src/powerapi/cli/parsing_manager.py index c2a70acd..dc229362 100644 --- a/src/powerapi/cli/parsing_manager.py +++ b/src/powerapi/cli/parsing_manager.py @@ -27,241 +27,135 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import json -import logging import sys -from collections.abc import Callable from typing import Any -from powerapi.cli.config_parser import RootConfigParser, SubgroupConfigParser, store_val -from powerapi.exception import MissingArgumentException, BadTypeException, AlreadyAddedSubparserException, \ - UnknownArgException, MissingValueException, BadContextException, RepeatedArgumentException, \ - AlreadyAddedSubgroupException -from ._utils import merge_dictionaries - - -class BaseConfigParsingManagerInterface: - """ - Abstract class for dealing with parsing of configurations. - """ - - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False) -> None: - """ - Add an argument to the parser. - """ - raise NotImplementedError +from powerapi.cli.cli_parser import CLIArgumentParser +from powerapi.cli.config_loader import EnvironmentConfigLoader, JSONConfigLoader +from powerapi.cli.config_parser import ( + ComponentSchema, + ConfigurationSchema, + ConfigurationSectionSchema, +) - def validate(self, conf: dict) -> dict: - """ - Validate the parsed configuration. - """ - raise NotImplementedError +from ._utils import merge_dictionaries -class SubgroupConfigParsingManager(BaseConfigParsingManagerInterface): +class ConfigurationParsingManager: """ - Sub Parser for MainConfigParser + Register the schema and orchestrate all configuration sources. """ - def __init__(self, name: str): - self.subparser = {} - self.name = name - self.cli_parser = SubgroupConfigParser(name) - - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False) -> None: - """ - Add an argument to the parser. - """ - self.cli_parser.add_argument(*names, is_flag=is_flag, action=action, default_value=default_value, - help_text=help_text, argument_type=bool if is_flag else argument_type, - is_mandatory=is_mandatory) + def __init__(self) -> None: + self.schema = ConfigurationSchema() + self.argument_parser = CLIArgumentParser(self.schema) + self.json_loader = JSONConfigLoader() + self.environment_loader = EnvironmentConfigLoader(self.schema) + + def add_argument_prefix(self, argument_prefix: str) -> None: + """ + Register an environment-variable prefix for root properties. + :param argument_prefix: Environment-variable prefix to register. + :raises ValueError: If the prefix overlaps an existing prefix. + """ + self.schema.add_argument_prefix(argument_prefix=argument_prefix) + + def add_group(self, name: str, help_text: str = '', prefix: str = '') -> None: + """ + Register a configuration group and its environment prefix. + :param name: Configuration name of the group. + :param help_text: User-facing description of the group. + :param prefix: Environment-variable prefix assigned to the group. + :raises ValueError: If the group name is already registered. + """ + self.schema.add_group(name, help_text=help_text, prefix=prefix) + + def add_component(self, group_name: str, component: ComponentSchema) -> None: + """ + Register a component schema in a group. + :param group_name: Group receiving the component schema. + :param component: Component schema to register. + :raises ValueError: If the group is unknown or the component type is already registered. + """ + self.schema.add_component(group_name, component) + + def add_section(self, group_name: str, section_name: str, section: ConfigurationSectionSchema) -> None: + """ + Register a fixed configuration section in a group. + :param group_name: Group receiving the configuration section. + :param section_name: Name identifying and reserving the section in the group. + :param section: Configuration section schema to register. + :raises ValueError: If the group is unknown or the section name is already registered. + """ + self.schema.add_section(group_name, section_name, section) + + def add_argument( + self, + name: str, + *, + is_flag: bool = False, + default_value: Any = None, + help_text: str = '', + argument_type: type = str, + is_mandatory: bool = False, + ) -> None: + """ + Register a root property in the configuration schema. + :param name: Configuration property name. + :param is_flag: Whether the option is a boolean flag. + :param default_value: Value used when the property is omitted. + :param help_text: Description displayed in command-line help. + :param argument_type: Type used to cast non-flag values. + :param is_mandatory: Whether the property must be defined. + :raises ValueError: If the property name is already registered. + """ + self.schema.add_argument( + name, + is_flag=is_flag, + default_value=default_value, + help_text=help_text, + argument_type=argument_type, + is_mandatory=is_mandatory, + ) def validate(self, conf: dict) -> dict: """ - Check the parsed configuration. - """ - - # check types - for args, value in conf.items(): - for _, waited_value in self.cli_parser.get_arguments().items(): - if args in waited_value.names: - # check type - if not isinstance(value, waited_value.type) and not waited_value.is_flag: - raise BadTypeException(args, waited_value.type) - - # Check that all the mandatory arguments are present - conf = self.cli_parser.validate(conf=conf) - - return conf - - -class RootConfigParsingManager(BaseConfigParsingManagerInterface): - """ - Parser abstraction for the configuration - """ - - def __init__(self): - self.subparser = {} - self.cli_parser = RootConfigParser() - - def add_argument_prefix(self, argument_prefix: str): - """ - Add a simple argument prefix to the cli_parser - :param argument_prefix: a new argument prefix to be added - """ - self.cli_parser.add_argument_prefix(argument_prefix=argument_prefix) - - def add_subgroup(self, name: str, help_text: str = '', prefix: str = ''): - """ - Add a group to the cli_parser - :param name: the group's name - :param help_text: a help text for the subgroup - :param prefix: a prefix related to the subgroup - """ - try: - self.cli_parser.add_subgroup(subgroup_type=name, help_text=help_text, prefix=prefix) - except AlreadyAddedSubgroupException as exn: - logging.error("Configuration error: %s", exn.msg) - sys.exit(-1) - - def add_subgroup_parser(self, subgroup_name: str, subgroup_parser: SubgroupConfigParsingManager): - + Validate and canonicalize a merged configuration. + :param conf: Merged configuration to validate. + :return: Canonical validated configuration with defaults applied. + :raises ConfigurationError: If the configuration is invalid. """ - Add a Subgroup Parser to call when is encountered - When name is encountered, the subgroup parser such as subgroup_parser.name match conf[name].type - """ - if subgroup_name in self.subparser: - if subgroup_parser.name in list(self.subparser[subgroup_name]): - raise AlreadyAddedSubparserException(subgroup_name) - else: - self.subparser[subgroup_name] = {} - - self.subparser[subgroup_name][subgroup_parser.name] = subgroup_parser - - self.cli_parser.add_subgroup_parser(subgroup_type=subgroup_name, subgroup_parser=subgroup_parser.cli_parser) - - def _parse_cli(self, cli_line: list) -> dict: - return self.cli_parser.parse(cli_line) - - def _parse_config_from_json_file(self, file_name: str, current_conf: dict) -> dict: - return merge_dictionaries(current_conf, self.cli_parser.parse_config_dict(file_name)) - - def _parse_config_from_environment_variables(self, current_conf: dict) -> dict: - return merge_dictionaries(current_conf, self.cli_parser.parse_config_environment_variables()) + return self.schema.validate(conf) - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False) -> None: + def _parse_configuration_sources(self, cli_line: list[str]) -> dict: """ - Add an argument to the parser. + Load and merge every configuration source. + :param cli_line: Command-line arguments without the executable name. + :return: Merged configuration with CLI, environment, then file precedence. + :raises CLIParseException: If command-line arguments are invalid. + :raises FileNotFoundError: If the selected JSON configuration file does not exist. + :raises ConfigurationError: If the selected file does not contain a valid JSON configuration object. """ - self.cli_parser.add_argument(*names, is_flag=is_flag, action=action, default_value=default_value, - help_text=help_text, argument_type=bool if is_flag else argument_type, - is_mandatory=is_mandatory) - - def validate(self, conf: dict) -> dict: - """ - Check the parsed configuration - """ - - # check types - for current_argument_name, current_argument_value in conf.items(): - is_an_arg = False - if current_argument_name in self.subparser: - for _, dic_value in current_argument_value.items(): - self.subparser[current_argument_name][dic_value["type"]].validate(dic_value) - is_an_arg = True - - if not is_an_arg: - for _, argument_definition in self.cli_parser.get_arguments().items(): - if current_argument_name in argument_definition.names: - is_an_arg = True - # check type - if not isinstance(current_argument_value, - argument_definition.type) and not argument_definition.is_flag: - raise BadTypeException(current_argument_name, argument_definition.type) - - if not is_an_arg: - raise UnknownArgException(current_argument_name) + parsed_cli = self.argument_parser.parse(cli_line) + parsed_config_file = self.json_loader.load(parsed_cli.config_file) + parsed_environment = self.environment_loader.load() - # Check that all the mandatory arguments are present - conf = self.cli_parser.validate(conf) + return merge_dictionaries(parsed_config_file, parsed_environment, parsed_cli.configuration) - return conf - - def parse(self, args: list | None = None) -> dict: + def parse(self, args: list[str] | None = None) -> dict: """ - Parse the configurations defined via le CLI, Environment Variables and configuration file. - The priority of defined values is the following: - 1. CLI - 2. Environment Variables - 3. Configuration File + Load, merge, and validate configuration values. - Call the method to produce a configuration dictionary - check the configuration + Precedence is CLI, then environment variables, then the JSON file. + Parsing and validation errors propagate to the application boundary. + :param args: Command-line arguments including an optional executable name, or None to use ``sys.argv``. + :return: Merged, canonical, and validated PowerAPI configuration. + :raises CLIParseException: If command-line arguments are invalid. + :raises FileNotFoundError: If the selected JSON configuration file does not exist. + :raises ConfigurationError: If the selected file contains invalid JSON or the merged configuration is invalid. """ - - if not args: + if args is None: args = sys.argv - current_position = 0 - filename = None - for current_arg in args: - if current_arg == '--config-file': - if current_position + 1 == len(args): - logging.error("CLI Error: Config filepath needed with argument --config-file") - sys.exit(-1) - - filename = args[current_position + 1] - args.pop(current_position + 1) - args.pop(current_position) - break - current_position += 1 - - try: - - if len(args) > 1: - conf = self._parse_cli(args[1:]) - else: - conf = {} - conf = self._parse_config_from_environment_variables(current_conf=conf) - if filename: - conf = self._parse_config_from_json_file(file_name=filename, current_conf=conf) - - # We validate the conf - conf = self.validate(conf) - - except MissingValueException as exn: - logging.error('CLI error: Argument "%s" expect a value', exn.argument_name) - sys.exit(-1) - - except BadTypeException as exn: - logging.error('Configuration error: %s', exn.msg) - sys.exit(-1) - - except UnknownArgException as exn: - logging.error('Configuration error: Argument "%s" is unknown', exn.argument_name) - sys.exit(-1) - - except BadContextException as exn: - logging.error('CLI error: %s', exn.msg) - sys.exit(-1) - - except FileNotFoundError: - logging.error("Configuration Error: Configuration file not found") - sys.exit(-1) - - except json.JSONDecodeError as exn: - logging.error('JSON error: "%s" at line %d column %d', exn.msg, exn.lineno, exn.colno) - sys.exit(-1) - - except MissingArgumentException as exn: - logging.error("Configuration Error: %s", exn.msg) - sys.exit(-1) - - except RepeatedArgumentException as exn: - logging.error("Configuration Error: %s", exn.msg) - sys.exit(-1) - - return conf + cli_line = args[1:] if args and not args[0].startswith('-') else args + return self.validate(self._parse_configuration_sources(cli_line)) diff --git a/src/powerapi/exception.py b/src/powerapi/exception.py index dbd301a0..51a69895 100644 --- a/src/powerapi/exception.py +++ b/src/powerapi/exception.py @@ -34,7 +34,7 @@ class PowerAPIException(Exception): """ def __init__(self, *args: object): - Exception.__init__(self, args) + Exception.__init__(self, *args) class PowerAPIExceptionWithMessage(PowerAPIException): @@ -43,10 +43,27 @@ class PowerAPIExceptionWithMessage(PowerAPIException): """ def __init__(self, msg): - PowerAPIException.__init__(self) + PowerAPIException.__init__(self, msg) self.msg = msg +class ConfigurationError(PowerAPIExceptionWithMessage): + """ + Exception raised when configuration loading or validation fails. + """ + + def __init__(self, reason: str, path: str | None = None): + """ + Initialize a configuration error. + :param reason: User-facing explanation of the invalid configuration. + :param path: Dotted path of the invalid value, or None for an error affecting the full configuration. + """ + self.reason = reason + self.path = path + prefix = f'Invalid configuration at "{path}": ' if path else 'Invalid configuration: ' + super().__init__(prefix + reason) + + class BadInputData(PowerAPIException): """ Exception raised when the data read in input are not diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 92bd7be7..4107c81b 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -27,24 +27,9 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os -import sys -from copy import deepcopy - import pytest -import tests.utils.cli as test_files_module -from powerapi.cli.config_parser import SubgroupConfigParser, BaseConfigParser, store_true, RootConfigParser -from powerapi.cli.parsing_manager import RootConfigParsingManager, SubgroupConfigParsingManager -from tests.utils.cli.base_config_parser import load_configuration_from_json_file, generate_cli_configuration_from_json_file - - -@pytest.fixture(name="invalid_csv_io_stream_config") -def csv_input_output_stream_config(): - """ - Invalid configuration with csv as input and output and stream mode enabled - """ - return load_configuration_from_json_file(file_name='csv_input_output_stream_mode_enabled_configuration.json') +from tests.utils.cli.base_config_parser import load_configuration_from_json_file @pytest.fixture @@ -55,21 +40,12 @@ def several_inputs_outputs_stream_config(): return load_configuration_from_json_file('several_inputs_outputs_stream_mode_enabled_configuration.json') -@pytest.fixture -def single_input_multiple_outputs_with_different_report_type(): - """ - Configuration with several inputs and outputs and stream mode enabled - """ - return load_configuration_from_json_file('single_input_multiple_outputs_with_different_report_type_configuration.json') - - @pytest.fixture def several_inputs_outputs_stream_socket_without_some_arguments_config(several_inputs_outputs_stream_config): """ - Configuration with several inputs and outputs and stream mode enabled. Some arguments - of socket input are removed + Configuration with a socket input missing a required argument. """ - for _, current_input in several_inputs_outputs_stream_config["input"].items(): + for current_input in several_inputs_outputs_stream_config['input'].values(): if current_input['type'] == 'socket': current_input.pop('port') @@ -77,285 +53,20 @@ def several_inputs_outputs_stream_socket_without_some_arguments_config(several_i @pytest.fixture -def csv_io_postmortem_config(invalid_csv_io_stream_config): - """ - Valid configuration with csv as input and output and stream mode disabled - """ - invalid_csv_io_stream_config["stream"] = False - return invalid_csv_io_stream_config - - -@pytest.fixture -def csv_io_postmortem_config_without_optional_arguments(csv_io_postmortem_config): - """ - Valid configuration with csv as input and output without optional arguments, i.e., - stream, verbose, and name model for inputs and outputs - """ - csv_io_postmortem_config.pop('stream') - csv_io_postmortem_config.pop('verbose') - - for current_input_name in csv_io_postmortem_config['input']: - csv_io_postmortem_config['input'][current_input_name].pop('model') - csv_io_postmortem_config['input'][current_input_name].pop('name') - - for current_ouput_name in csv_io_postmortem_config['output']: - csv_io_postmortem_config['output'][current_ouput_name].pop('model') - csv_io_postmortem_config['output'][current_ouput_name].pop('name') - - return csv_io_postmortem_config - - -@pytest.fixture -def config_without_input(csv_io_postmortem_config): - """ - Invalid configuration without inputs - """ - csv_io_postmortem_config.pop('input') - - return csv_io_postmortem_config - - -@pytest.fixture -def config_without_output(csv_io_postmortem_config): - """ - Invalid configuration without inputs - """ - csv_io_postmortem_config.pop('output') - - return csv_io_postmortem_config - - -@pytest.fixture -def subgroup_parser(): - """ - A subgroup parser with one argument "-a" - """ - parser = SubgroupConfigParser('test') - parser.add_argument('a', is_flag=True) - return parser - - -@pytest.fixture -def create_empty_files_from_config(invalid_csv_io_stream_config: dict): +def several_inputs_outputs_postmortem_config(several_inputs_outputs_stream_config): """ - Create on the module path the files that are indicated on csv input. - When they are no longer required, those files are erased - """ - for _, input_config in invalid_csv_io_stream_config['input'].items(): - if input_config['type'] == 'csv': - for file_str in input_config['files']: - if not os.path.isfile(file_str): - with open(file_str, 'w') as file: - file.close() - - yield - - for _, input_config in invalid_csv_io_stream_config['input'].items(): - if input_config['type'] == 'csv': - for file_str in input_config['files']: - if os.path.isfile(file_str): - os.remove(file_str) - - -@pytest.fixture -def base_config_parser(): - """ - Return a BaseConfigParser with mandatory and optional arguments - """ - - parser = BaseConfigParser() - - parser.add_argument('arg1', 'argument1', 'argumento1', default_value=3, argument_type=int, is_mandatory=False) - - parser.add_argument('argumento2', 'arg2', argument_type=str, is_mandatory=True) - - parser.add_argument('arg3', 'argument3', argument_type=bool, is_mandatory=False) - - parser.add_argument('dded', 'arg4', argument_type=float, is_mandatory=True) - - parser.add_argument('arg5', '5', default_value='default value', argument_type=str, help_text='help 5') - - return parser - - -@pytest.fixture -def root_config_parser_with_mandatory_and_optional_arguments(): - """ - Return a RootConfigParser with mandatory and optional arguments - """ - - parser = RootConfigParser() - - parser.add_argument('a', argument_type=bool, is_flag=True, action=store_true) - - parser.add_argument('argument1', 'arg1', default_value=3, argument_type=int, is_mandatory=False) - - parser.add_argument('argumento2', '2', argument_type=str, is_mandatory=True) - - parser.add_argument('arg3', 'argument3', argument_type=bool, is_mandatory=False) - - parser.add_argument('d', 'arg4', argument_type=float, is_mandatory=True) - - parser.add_argument('arg5', '5', default_value='default value', argument_type=str, - help_text='help 5') - - return parser - - -@pytest.fixture -def root_config_parser_with_subgroups(root_config_parser_with_mandatory_and_optional_arguments): - """ - Return a RootConfigParser with subgroups - """ - - root_config_parser_with_mandatory_and_optional_arguments.add_argument_prefix(argument_prefix='TEST_') - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='g1', prefix='TEST_G1_') - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='g2', prefix='TEST_G2_') - - subgroup_parser_g1 = SubgroupConfigParser(name='type1') - subgroup_parser_g1.add_argument('1', 'a1', argument_type=str, is_mandatory=True) - subgroup_parser_g1.add_argument('2', 'a2', argument_type=bool, default_value=True) - subgroup_parser_g1.add_argument('3', 'a3', argument_type=str, default_value=69) - subgroup_parser_g1.add_argument('n', 'name', argument_type=str) - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup_parser(subgroup_type='g1', - subgroup_parser=subgroup_parser_g1) - - subgroup_parser_g2 = SubgroupConfigParser(name='type2') - subgroup_parser_g2.add_argument('1', 'a1', argument_type=float, is_mandatory=False) - subgroup_parser_g2.add_argument('2', 'a2', argument_type=str) - subgroup_parser_g2.add_argument('3', 'a3', argument_type=str) - subgroup_parser_g2.add_argument('4', 'a4', argument_type=str) - subgroup_parser_g2.add_argument('n', 'name', argument_type=str) - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup_parser(subgroup_type='g2', - subgroup_parser=subgroup_parser_g2) - - return root_config_parser_with_mandatory_and_optional_arguments - - -@pytest.fixture -def base_config_parser_no_mandatory_arguments(): + Configuration with several inputs and outputs and stream mode disabled. """ - Return a BaseConfigParser without mandatory arguments - """ - parser = BaseConfigParser() - - parser.add_argument('arg1', default_value=4, argument_type=int) - - parser.add_argument('arg2', argument_type=str) - - parser.add_argument('arg3', argument_type=bool) - - parser.add_argument('arg4', argument_type=int) - - parser.add_argument('arg5', argument_type=int) - - return parser - - -@pytest.fixture -def base_config_parser_str_representation(): - """ - Return expected representation for a BaseConfigParser used in unit tests - """ - return ' --arg1, --argument1, --argumento1 : \n' + \ - ' --argumento2, --arg2 : \n' + \ - ' --arg3, --argument3 : \n' + \ - ' --dded, --arg4 : \n' + \ - ' --arg5, -5 : help 5\n' - - -@pytest.fixture -def root_config_parsing_manager(): - """ - Return a RootConfigParsingManager with a flag argument 'a' - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a', argument_type=bool, is_flag=True, action=store_true) - parser_manager.add_subgroup(name='sub') - - return parser_manager - - -@pytest.fixture -def root_config_parsing_manager_with_mandatory_and_optional_arguments(): - """ - Return a RootConfigParsingManager with several arguments, some of them are mandatory - """ - parser_manager = RootConfigParsingManager() - - parser_manager.add_argument_prefix(argument_prefix='TEST_') - - parser_manager.add_subgroup(name='input', prefix='TEST_INPUT_') - - parser_manager.add_subgroup(name='output', prefix='TEST_OUTPUT_') - - parser_manager.add_argument('a', argument_type=bool, is_flag=True, action=store_true) - - parser_manager.add_argument('1', 'argument1', default_value=3, argument_type=int, is_mandatory=False) - - parser_manager.add_argument('argumento2', '2', argument_type=str, is_mandatory=True) - - parser_manager.add_argument('arg3', 'argument3', argument_type=bool, is_mandatory=False) - - parser_manager.add_argument('d', 'arg4', argument_type=float, is_mandatory=True) - - parser_manager.add_argument('arg5', '5', default_value='default value', argument_type=str, - help_text='help 5') - - i1_type_subgroup_parser_manager = SubgroupConfigParsingManager(name="i1_type") - i1_type_subgroup_parser_manager.add_argument('model', 'm', argument_type=str, is_mandatory=True) - i1_type_subgroup_parser_manager.add_argument('db', 'd', argument_type=str, is_mandatory=False) - i1_type_subgroup_parser_manager.add_argument('port', 'p', argument_type=int, is_mandatory=False) - i1_type_subgroup_parser_manager.add_argument('name', 'n', argument_type=str, is_mandatory=False, - default_value='my_i1_instance') - - parser_manager.add_subgroup_parser(subgroup_name="input", subgroup_parser=i1_type_subgroup_parser_manager) - - o1_type_subgroup_parser_manager = SubgroupConfigParsingManager(name="o1_type") - o1_type_subgroup_parser_manager.add_argument('model', 'm', argument_type=str, is_mandatory=True) - o1_type_subgroup_parser_manager.add_argument('db', 'd', argument_type=str, is_mandatory=False) - o1_type_subgroup_parser_manager.add_argument('name', 'n', argument_type=str, is_mandatory=False, - default_value='my_o1_instance') - o1_type_subgroup_parser_manager.add_argument('collection', 'c', argument_type=str) - - parser_manager.add_subgroup_parser(subgroup_name="output", subgroup_parser=o1_type_subgroup_parser_manager) - - o2_type_subgroup_parser_manager = SubgroupConfigParsingManager(name="o2_type") - o2_type_subgroup_parser_manager.add_argument('model', 'm', argument_type=str, is_mandatory=True) - o2_type_subgroup_parser_manager.add_argument('db', 'd', argument_type=str, is_mandatory=False) - o2_type_subgroup_parser_manager.add_argument('name', 'n', argument_type=str, is_mandatory=False, - default_value='my_o2_instance') - o2_type_subgroup_parser_manager.add_argument('collection', 'c', argument_type=str) - - parser_manager.add_subgroup_parser(subgroup_name="output", subgroup_parser=o2_type_subgroup_parser_manager) - - return parser_manager - - -@pytest.fixture -def test_files_path(): - """ - Return the path of directory containing tests files - """ - return test_files_module.__path__[0] - - -@pytest.fixture -def cli_configuration(config_file: str, monkeypatch): - """ - Load in sys.argv a configuration with arguments extracted from a json file - """ - monkeypatch.setattr(sys, 'argv', generate_cli_configuration_from_json_file(file_name=config_file)) + several_inputs_outputs_stream_config['stream'] = False + return several_inputs_outputs_stream_config @pytest.fixture -def empty_cli_configuration(monkeypatch): +def single_input_multiple_outputs_with_different_report_type(): """ - Clean the CLI arguments + Configuration with several inputs and outputs and stream mode enabled """ - monkeypatch.setattr(sys, 'argv', []) + return load_configuration_from_json_file('single_input_multiple_outputs_with_different_report_type_configuration.json') @pytest.fixture @@ -374,17 +85,6 @@ def pre_processor_complete_configuration(request): return load_configuration_from_json_file(file_name=request.param) -@pytest.fixture -def pre_processor_config_without_puller(pre_processor_complete_configuration): - """ - Return a configuration with processors but without bindings - """ - - pre_processor_complete_configuration['pre-processor']['my_processor'].pop('puller') - - return pre_processor_complete_configuration - - @pytest.fixture def empty_pre_processor_config(pre_processor_complete_configuration): """ @@ -402,61 +102,3 @@ def pre_processor_with_unexisting_puller_configuration(request): Return a dictionary containing a pre-processor with a puller that doesn't exist """ return load_configuration_from_json_file(request.param) - - -def get_config_with_longest_argument_names(config: dict, arguments: dict): - """ - Return a copy of the provided configuration with the longest name for each argument - :param dict config: Configuration to be modified - :param dict arguments: Arguments definition - """ - config_longest_names = {} - for argument_name in config.keys(): - current_argument = arguments[argument_name] - longest_argument_name = get_longest_name(current_argument.names) - config_longest_names[longest_argument_name] = config[argument_name] - - return config_longest_names - - -def get_longest_name(names: list): - """ - Return the longest name in the provide list - :param list names: List of names - """ - longest_name = "" - - for name in names: - if len(name) > len(longest_name): - longest_name = name - - return longest_name - - -def get_config_with_default_values(config: dict, arguments: dict): - """ - Get a configuration that contains all optional arguments with their default values - :param dict config: Configuration to be modified - :param dict arguments: Arguments definition - """ - - processed_arguments = [] - - config_all_values = deepcopy(config) - - for current_argument_name, current_argument in arguments.items(): - if current_argument not in processed_arguments: - argument_value_already_defined = False - - for name in current_argument.names: - if name in config: - argument_value_already_defined = True - - break - - if not argument_value_already_defined and current_argument.default_value is not None: - config_all_values[current_argument_name] = current_argument.default_value - - processed_arguments.append(current_argument) - - return config_all_values diff --git a/tests/unit/cli/test_cli_parser.py b/tests/unit/cli/test_cli_parser.py new file mode 100644 index 00000000..8da90f52 --- /dev/null +++ b/tests/unit/cli/test_cli_parser.py @@ -0,0 +1,318 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse + +import pytest + +from powerapi.cli import cli_parser +from powerapi.cli.cli_parser import CLIArgumentParser, CLIParseException +from powerapi.cli.config_parser import ( + ComponentSchema, + ConfigurationSchema, + ConfigurationSectionSchema, +) + + +def create_parser() -> CLIArgumentParser: + """ + Create a CLI parser backed by an empty configuration schema. + :return: CLI parser accepting only built-in options. + """ + return CLIArgumentParser(ConfigurationSchema()) + + +def test_configuration_help_uses_plain_theme_without_argparse_theme(monkeypatch): + """ + Test that configuration help falls back to plain text when argparse has no theme. + """ + monkeypatch.setattr(cli_parser.argparse, 'RawDescriptionHelpFormatter', lambda _: object()) + + assert cli_parser._get_argparse_theme() == cli_parser._HelpTheme() + + +def test_cli_argument_parser_preserves_equals_in_assignment_value(): + """ + Test that assignment values preserve equals signs after the first separator. + """ + result = create_parser().parse([ + '-C', 'input.mongo.uri=mongodb://localhost/?option=value', + ]) + + assert result.configuration == { + 'input': {'mongo': {'uri': 'mongodb://localhost/?option=value'}}, + } + + +@pytest.mark.parametrize('expression', [ + 'input.mongo.uri', + '=value', + 'input..uri=value', + '.input=value', + 'input.sensor=value', + 'input.sensor.connection.uri=value', +]) +def test_cli_argument_parser_rejects_invalid_assignment(expression): + """ + Test that malformed or unsupported assignment paths are rejected. + """ + with pytest.raises(CLIParseException): + create_parser().parse(['-C', expression]) + + +def test_cli_argument_parser_builds_nested_configuration(): + """ + Test that root and component assignments build a nested configuration. + """ + result = create_parser().parse([ + '-C', 'input.sensor.type=socket', + '-C', 'input.sensor.port=9080', + '-C', 'stream=false', + ]) + + assert result.configuration == { + 'input': { + 'sensor': { + 'type': 'socket', + 'port': '9080', + }, + }, + 'stream': 'false', + } + + +def test_cli_argument_parser_uses_last_repeated_value(): + """ + Test that the last assignment to a path replaces previous values. + """ + result = create_parser().parse([ + '-C', 'input.sensor.port=9080', + '-C', 'input.sensor.port=9090', + ]) + + assert result.configuration['input']['sensor']['port'] == '9090' + + +@pytest.mark.parametrize('expressions', [ + ['input=value', 'input.sensor.port=9090'], + ['input.sensor.port=9090', 'input=value'], +]) +def test_cli_argument_parser_rejects_structural_conflict(expressions): + """ + Test that scalar and nested assignments cannot target the same path. + """ + arguments = [argument for expression in expressions for argument in ('-C', expression)] + + with pytest.raises(CLIParseException): + create_parser().parse(arguments) + + +def test_cli_argument_parser_parses_repeatable_config_assignments(): + """ + Test repeatable assignment options and configuration-file selection. + """ + parser = create_parser() + + result = parser.parse([ + '-C', 'input.sensor.type=socket', + '--set-config', 'input.sensor.port=9080', + '--config-file', '/tmp/powerapi.json', + ]) + + assert result.config_file == '/tmp/powerapi.json' + assert result.configuration == { + 'input': { + 'sensor': { + 'type': 'socket', + 'port': '9080', + }, + }, + } + + +def test_cli_argument_parser_keeps_internal_values_separate_from_root_arguments(): + """ + Test that internal parser destinations do not collide with schema properties. + """ + schema = ConfigurationSchema() + schema.add_argument('config_assignments') + schema.add_argument('config_file') + parser = CLIArgumentParser(schema) + + result = parser.parse([ + '--config_assignments', 'root-value', + '--config_file', 'another-root-value', + '-C', 'stream=false', + '--config-file', '/tmp/powerapi.json', + ]) + + assert result.config_file == '/tmp/powerapi.json' + assert result.configuration == { + 'config_assignments': 'root-value', + 'config_file': 'another-root-value', + 'stream': 'false', + } + + +def test_cli_argument_parser_reports_argument_conflict_as_cli_error(): + """ + Test that schema option conflicts are reported as CLIParseException. + """ + schema = ConfigurationSchema() + schema.add_argument('config-file') + parser = CLIArgumentParser(schema) + + with pytest.raises(CLIParseException) as result: + parser.parse([]) + + assert result.value.msg == 'Failed to add argument: argument --config-file: conflicting option string: --config-file' + assert isinstance(result.value.__cause__, argparse.ArgumentError) + + +def test_cli_argument_parser_does_not_add_absent_root_argument(): + """ + Test that omitted root options are absent from the raw CLI configuration. + """ + schema = ConfigurationSchema() + schema.add_argument('stream', is_flag=True) + parser = CLIArgumentParser(schema) + + result = parser.parse([]) + + assert 'stream' not in result.configuration + + +def test_cli_argument_parser_parses_registered_root_flag(): + """ + Test that a registered root flag produces its configuration property. + """ + schema = ConfigurationSchema() + schema.add_argument('stream', is_flag=True) + parser = CLIArgumentParser(schema) + + result = parser.parse(['--stream']) + + assert result.configuration == {'stream': True} + + +def test_cli_argument_parser_keeps_non_flag_root_value_raw(): + """ + Test that non-flag root values remain raw until schema validation. + """ + schema = ConfigurationSchema() + schema.add_argument('port') + parser = CLIArgumentParser(schema) + + result = parser.parse(['--port', '9080']) + + assert result.configuration == {'port': '9080'} + + +def test_cli_argument_parser_preserves_root_argument_underscores(): + """ + Test that underscores in root option names are preserved. + """ + schema = ConfigurationSchema() + schema.add_argument('some_value') + parser = CLIArgumentParser(schema) + + result = parser.parse(['--some_value', 'test']) + + assert result.configuration == {'some_value': 'test'} + + +def test_cli_argument_parser_rejects_contextual_argument(): + """ + Test that legacy contextual component options are rejected. + """ + parser = create_parser() + + with pytest.raises(CLIParseException) as result: + parser.parse(['--input', 'socket']) + + assert result.value.msg == 'Failed to parse CLI: unrecognized arguments: --input socket' + assert isinstance(result.value.__cause__, argparse.ArgumentError) + + +def test_cli_argument_parser_prints_help(capsys): + """ + Test that --help prints usage information and exits successfully. + """ + parser = create_parser() + + with pytest.raises(SystemExit) as result: + parser.parse(['--help']) + + help_message = capsys.readouterr().out + + assert result.value.code == 0 + assert help_message.startswith('usage:') + assert '-C PATH=VALUE' in help_message + assert '--config-file FILE' in help_message + + +def test_cli_argument_parser_help_contains_schema_paths(capsys): + """ + Test that help exposes canonical schema paths and supported component types. + """ + schema = ConfigurationSchema() + schema.add_argument('verbose', is_flag=True) + schema.add_group('input') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int) + schema.add_component('input', component) + schema.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + schema.add_section('formula', 'smartwatts', smartwatts) + + with pytest.raises(SystemExit): + CLIArgumentParser(schema).parse(['--help']) + + help_lines = { + line.strip() + for line in capsys.readouterr().out.splitlines() + if line.strip() + } + expected_lines = { + 'verbose', + 'input.NAME.type', + 'Supported types: socket', + 'input.NAME.type=socket', + 'input.NAME.port', + 'formula.smartwatts.learn-error-window-size', + } + excluded_lines = { + 'v', + 'input.NAME.p', + 'formula.smartwatts.w', + 'formula.smartwatts.type', + } + + assert expected_lines.issubset(help_lines) + assert excluded_lines.isdisjoint(help_lines) diff --git a/tests/unit/cli/test_common_cli_parsing_manager.py b/tests/unit/cli/test_common_cli_parsing_manager.py index ca15d68d..7134f5a1 100644 --- a/tests/unit/cli/test_common_cli_parsing_manager.py +++ b/tests/unit/cli/test_common_cli_parsing_manager.py @@ -28,10 +28,15 @@ import pytest -from powerapi.cli.common_cli_parsing_manager import CommonCLIParsingManager, PullerConfigParsingManager, \ - PusherConfigParsingManager, PreProcessorConfigParsingManager, generate_env_prefix -from powerapi.cli.config_parser import store_true -from powerapi.exception import NoNameSpecifiedForSubgroupException +from powerapi.cli.cli_parser import CLIParseException +from powerapi.cli.common_cli_parsing_manager import ( + CommonCLIParsingManager, + PreProcessorSchema, + PullerSchema, + PusherSchema, + generate_env_prefix, +) +from powerapi.exception import ConfigurationError def test_generate_env_prefix_with_no_component(): @@ -76,69 +81,36 @@ def test_generate_env_prefix_with_custom_root_prefix(): assert generate_env_prefix('INPUT', root_prefix='MYAPP') == 'MYAPP_INPUT_' -def test_puller_config_parser_registers_shared_name_argument(): +def test_puller_config_schema_registers_default_model_argument(): """ - Test that PullerConfigParsingManager registers the shared subgroup name argument. + Test that PullerSchema registers the default report model argument. """ - parser = PullerConfigParsingManager('pytest') - name_argument = parser.cli_parser.get_arguments()['name'] + schema = PullerSchema('pytest') + model_argument = schema.arguments['model'] - assert name_argument.names == ['n', 'name'] - assert name_argument.is_mandatory is False - - -def test_puller_config_parser_registers_default_model_argument(): - """ - Test that PullerConfigParsingManager registers the default report model argument. - """ - parser = PullerConfigParsingManager('pytest') - model_argument = parser.cli_parser.get_arguments()['model'] - - assert model_argument.names == ['m', 'model'] + assert model_argument.name == 'model' assert model_argument.default_value == 'HWPCReport' -def test_pusher_config_parser_registers_shared_name_argument(): - """ - Test that PusherConfigParsingManager registers the shared subgroup name argument. - """ - parser = PusherConfigParsingManager('pytest') - name_argument = parser.cli_parser.get_arguments()['name'] - - assert name_argument.names == ['n', 'name'] - assert name_argument.is_mandatory is False - - -def test_pusher_config_parser_registers_default_model_argument(): +def test_pusher_config_schema_registers_default_model_argument(): """ - Test that PusherConfigParsingManager registers the default report model argument. + Test that PusherSchema registers the default report model argument. """ - parser = PusherConfigParsingManager('pytest') - model_argument = parser.cli_parser.get_arguments()['model'] + schema = PusherSchema('pytest') + model_argument = schema.arguments['model'] - assert model_argument.names == ['m', 'model'] + assert model_argument.name == 'model' assert model_argument.default_value == 'PowerReport' -def test_pre_processor_config_parser_registers_shared_name_argument(): +def test_pre_processor_config_schema_registers_mandatory_puller_argument(): """ - Test that PreProcessorConfigParsingManager registers the shared subgroup name argument. + Test that PreProcessorSchema registers a mandatory puller argument. """ - parser = PreProcessorConfigParsingManager('pytest') - name_argument = parser.cli_parser.get_arguments()['name'] - - assert name_argument.names == ['n', 'name'] - assert name_argument.is_mandatory is False - + schema = PreProcessorSchema('pytest') + puller_argument = schema.arguments['puller'] -def test_pre_processor_config_parser_registers_mandatory_puller_argument(): - """ - Test that PreProcessorConfigParsingManager registers a mandatory puller argument. - """ - parser = PreProcessorConfigParsingManager('pytest') - puller_argument = parser.cli_parser.get_arguments()['puller'] - - assert puller_argument.names == ['p', 'puller'] + assert puller_argument.name == 'puller' assert puller_argument.is_mandatory is True @@ -146,32 +118,32 @@ def test_common_cli_manager_registers_root_environment_prefix(): """ Test that CommonCLIParsingManager registers the root PowerAPI environment prefix. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert parser_manager.cli_parser.arguments_prefix == ['POWERAPI_'] + assert manager.schema.arguments_prefix == ['POWERAPI_'] -def test_common_cli_manager_registers_subgroup_environment_prefixes(): +def test_common_cli_manager_registers_group_environment_prefixes(): """ - Test that CommonCLIParsingManager registers every subgroup environment prefix. + Test that CommonCLIParsingManager registers every group environment prefix. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert parser_manager.cli_parser.get_groups_prefixes() == [ - 'POWERAPI_INPUT_', - 'POWERAPI_OUTPUT_', - 'POWERAPI_PRE_PROCESSOR_', - 'POWERAPI_POST_PROCESSOR_', - ] + assert {name: group.prefix for name, group in manager.schema.groups.items() if group.prefix} == { + 'input': 'POWERAPI_INPUT_', + 'output': 'POWERAPI_OUTPUT_', + 'pre-processor': 'POWERAPI_PRE_PROCESSOR_', + 'post-processor': 'POWERAPI_POST_PROCESSOR_', + } -def test_common_cli_manager_registers_top_level_subgroups(): +def test_common_cli_manager_registers_top_level_groups(): """ - Test that CommonCLIParsingManager registers every top-level subgroup. + Test that CommonCLIParsingManager registers every top-level group. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert set(parser_manager.cli_parser.subgroup_parsers) == { + assert set(manager.schema.groups) == { 'input', 'output', 'pre-processor', @@ -179,13 +151,13 @@ def test_common_cli_manager_registers_top_level_subgroups(): } -def test_common_cli_manager_registers_input_parsers(): +def test_common_cli_manager_registers_input_schemas(): """ - Test that CommonCLIParsingManager registers every built-in input parser. + Test that CommonCLIParsingManager registers every built-in input schema. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert set(parser_manager.subparser['input']) == { + assert set(manager.schema.groups['input'].components) == { 'mongodb', 'socket', 'csv', @@ -193,13 +165,13 @@ def test_common_cli_manager_registers_input_parsers(): } -def test_common_cli_manager_registers_output_parsers(): +def test_common_cli_manager_registers_output_schemas(): """ - Test that CommonCLIParsingManager registers every built-in output parser. + Test that CommonCLIParsingManager registers every built-in output schema. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert set(parser_manager.subparser['output']) == { + assert set(manager.schema.groups['output'].components) == { 'mongodb', 'prometheus', 'csv', @@ -209,14 +181,14 @@ def test_common_cli_manager_registers_output_parsers(): } -def test_common_cli_manager_registers_pre_processor_parsers(): +def test_common_cli_manager_registers_pre_processor_schemas(): """ - Test that CommonCLIParsingManager registers every built-in pre-processor parser. + Test that CommonCLIParsingManager registers every built-in pre-processor schema. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert set(parser_manager.subparser['pre-processor']) == { - 'k8s', + assert set(manager.schema.groups['pre-processor'].components) == { + 'kubernetes', 'openstack', } @@ -225,12 +197,11 @@ def test_common_cli_manager_registers_verbose_argument(): """ Test that CommonCLIParsingManager registers the verbose root argument. """ - parser_manager = CommonCLIParsingManager() - verbose_argument = parser_manager.cli_parser.get_arguments()['verbose'] + manager = CommonCLIParsingManager() + verbose_argument = manager.schema.arguments['verbose'] - assert verbose_argument.names == ['v', 'verbose'] + assert verbose_argument.name == 'verbose' assert verbose_argument.is_flag is True - assert verbose_argument.action is store_true assert verbose_argument.default_value is False @@ -238,12 +209,11 @@ def test_common_cli_manager_registers_stream_argument(): """ Test that CommonCLIParsingManager registers the stream root argument. """ - parser_manager = CommonCLIParsingManager() - stream_argument = parser_manager.cli_parser.get_arguments()['stream'] + manager = CommonCLIParsingManager() + stream_argument = manager.schema.arguments['stream'] - assert stream_argument.names == ['s', 'stream'] + assert stream_argument.name == 'stream' assert stream_argument.is_flag is True - assert stream_argument.action is store_true assert stream_argument.default_value is False @@ -251,7 +221,7 @@ def test_common_cli_manager_validates_output_config_without_name(): """ Test that output config validation can use the output key as the pusher name. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() config = { 'output': { 'powerrep': { @@ -262,7 +232,7 @@ def test_common_cli_manager_validates_output_config_without_name(): }, } - result = parser_manager.validate(config) + result = manager.validate(config) assert result['output']['powerrep']['type'] == 'json' assert result['output']['powerrep']['model'] == 'PowerReport' @@ -270,33 +240,33 @@ def test_common_cli_manager_validates_output_config_without_name(): assert result['output']['powerrep']['compression'] == 'auto' -def test_common_cli_manager_requires_name_for_cli_subgroup(): +def test_common_cli_manager_rejects_legacy_contextual_configuration(): """ - Test that CLI subgroup parsing still requires -n/--name. + Test that the breaking CLI no longer accepts contextual component arguments. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - with pytest.raises(NoNameSpecifiedForSubgroupException): - parser_manager._parse_cli([ + with pytest.raises(CLIParseException) as result: + manager.parse([ '--output', 'json', '--filepath', '/tmp/powerapi-output.jsonl', ]) + assert 'unrecognized arguments' in result.value.msg + def test_common_cli_manager_parse_cli_configuration(): """ Test that CommonCLIParsingManager parses a representative CLI configuration. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - result = parser_manager._parse_cli([ + result = manager.parse([ '--verbose', - '--input', 'csv', - '--name', 'pytest-puller', - '--files', 'a.csv,b.csv', - '--output', 'json', - '--name', 'pytest-pusher', - '--filepath', '/tmp/pytest-powerapi.jsonl' + '-C', 'input.pytest-puller.type=csv', + '-C', 'input.pytest-puller.files=a.csv,b.csv', + '-C', 'output.pytest-pusher.type=json', + '-C', 'output.pytest-pusher.filepath=/tmp/pytest-powerapi.jsonl', ]) assert result['verbose'] is True @@ -304,3 +274,71 @@ def test_common_cli_manager_parse_cli_configuration(): assert result['input']['pytest-puller']['files'] == ['a.csv', 'b.csv'] assert result['output']['pytest-pusher']['type'] == 'json' assert result['output']['pytest-pusher']['filepath'] == '/tmp/pytest-powerapi.jsonl' + + +def test_common_cli_manager_resolves_component_type_after_merging_environment(monkeypatch): + """ + Test that a partial CLI override inherits its component type from the environment. + """ + monkeypatch.setenv('POWERAPI_INPUT_sensor_TYPE', 'socket') + manager = CommonCLIParsingManager() + + result = manager.parse([ + 'powerapi', + '-C', 'input.sensor.port=9090', + ]) + + assert result['input']['sensor'] == { + 'type': 'socket', + 'model': 'HWPCReport', + 'host': 'localhost', + 'port': 9090, + } + + +def test_common_cli_manager_rejects_unknown_dotted_component_property(): + """ + Test that an unknown dotted component property is rejected with its full path. + """ + manager = CommonCLIParsingManager() + + with pytest.raises(ConfigurationError) as result: + manager.parse([ + 'powerapi', + '-C', 'input.sensor.type=socket', + '-C', 'input.sensor.unknown=value', + ]) + + assert result.value.path == 'input.sensor.unknown' + + +@pytest.mark.parametrize(('property_name', 'value'), [ + ('p', 9090), + ('name', 'legacy-name'), +]) +def test_common_cli_manager_rejects_legacy_component_properties(property_name, value): + """ + Test that removed shortened component property names are rejected. + """ + manager = CommonCLIParsingManager() + config = {'input': {'sensor': {'type': 'socket', property_name: value}}} + + with pytest.raises(ConfigurationError) as result: + manager.validate(config) + + assert result.value.path == f'input.sensor.{property_name}' + + +def test_common_cli_manager_requires_component_type_after_merging(): + """ + Test that a component type is required after all configuration sources are merged. + """ + manager = CommonCLIParsingManager() + + with pytest.raises(ConfigurationError) as result: + manager.parse([ + 'powerapi', + '-C', 'input.sensor.port=9090', + ]) + + assert result.value.path == 'input.sensor.type' diff --git a/tests/unit/cli/test_config_parser.py b/tests/unit/cli/test_config_parser.py index 52e16a84..4f66d26a 100644 --- a/tests/unit/cli/test_config_parser.py +++ b/tests/unit/cli/test_config_parser.py @@ -1,21 +1,20 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille +# Copyright (c) 2026, Inria # All rights reserved. - +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: - +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. - +# # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. - +# # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. - +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -26,953 +25,559 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest -from powerapi.cli.config_parser import ConfigurationArgument, BaseConfigParser, RootConfigParser, SubgroupConfigParser -from powerapi.cli.config_parser import store_true, store_val -from powerapi.exception import AlreadyAddedArgumentException, BadTypeException, UnknownArgException, \ - BadContextException, MissingValueException, SubgroupAlreadyExistException, \ - SubgroupParserWithoutNameArgumentException, \ - NoNameSpecifiedForSubgroupException, TooManyArgumentNamesException, MissingArgumentException, \ - SameLengthArgumentNamesException, AlreadyAddedSubgroupException -from tests.unit.cli.conftest import get_config_with_longest_argument_names, get_config_with_default_values - -from tests.utils.cli.base_config_parser import load_configuration_from_json_file, \ - generate_configuration_tuples_from_json_file, define_environment_variables_configuration_from_json_file, \ - remove_environment_variables_configuration - - -############### -# PARSER TEST # -############### -def test_add_argument_that_already_exists(): - """ - Test if an AlreadyAddedArgumentException is raised when an argument that already exists is added to a - BaseConfigParser - """ - - parser = BaseConfigParser() - parser.add_argument('a') - parser.add_argument('bb', 'bbb', 'b', 'bbbbbb') - - with pytest.raises(AlreadyAddedArgumentException): - parser.add_argument('a') - - with pytest.raises(AlreadyAddedArgumentException): - parser.add_argument('bbb') - - with pytest.raises(AlreadyAddedArgumentException): - parser.add_argument('bbbbbb') - - with pytest.raises(AlreadyAddedArgumentException): - parser.add_argument('ccc', 'a') - - assert len(parser.arguments) == 5 - - -def test_get_arguments_returns_all_stored_arguments(): - """ - Test if all the arguments are correctly stored by BasePaserConfig - """ - longest_name_arg_a = 'aaa' - name_arg_a_1 = 'a' - name_arg_a_2 = 'ab' - expected_argument_a = ConfigurationArgument(names=[name_arg_a_1, longest_name_arg_a, name_arg_a_2], - argument_type=bool, default_value=False, - help_text='This a parameter', is_mandatory=False, is_flag=True) +import json - longest_name_arg_xx = 'XXXX' - name_arg_xx_1 = 'xx' - name_arg_xx_2 = 'xax' - expected_argument_xx = ConfigurationArgument(names=[name_arg_xx_1, longest_name_arg_xx, name_arg_xx_2], - argument_type=str, default_value='Hi', - help_text='This is another parameter', is_mandatory=True, - is_flag=False, action=store_val) - - parser = BaseConfigParser() - parser.add_argument(name_arg_a_1, longest_name_arg_a, name_arg_a_2, is_mandatory=expected_argument_a.is_mandatory, - is_flag=expected_argument_a.is_flag, - argument_type=expected_argument_a.type, help_text=expected_argument_a.help_text, - default_value=expected_argument_a.default_value) - - parser.add_argument(name_arg_xx_1, longest_name_arg_xx, name_arg_xx_2, - is_mandatory=expected_argument_xx.is_mandatory, is_flag=expected_argument_xx.is_flag, - argument_type=expected_argument_xx.type, help_text=expected_argument_xx.help_text, - default_value=expected_argument_xx.default_value) - - arguments = parser.get_arguments() - - assert len(arguments) == 6 - - assert longest_name_arg_a in arguments - - assert longest_name_arg_xx in arguments - - assert expected_argument_a == arguments.get(longest_name_arg_a) +import pytest - assert expected_argument_xx == arguments.get(longest_name_arg_xx) +from powerapi.cli.config_loader import EnvironmentConfigLoader, JSONConfigLoader +from powerapi.cli.config_parser import ( + ComponentSchema, + ConfigurationSchema, + ConfigurationSectionSchema, +) +from powerapi.exception import ConfigurationError -def test_get_mandatory_arguments_return_all_mandatory_argument(base_config_parser): +def test_schema_registers_argument_definition(): """ - Test that all the mandatory arguments are identified by the paser + Test that a configuration property is registered with its definition. """ - expected_mandatory_args_names = ['arg2', 'arg4'] + schema = ConfigurationSectionSchema() - mandatory_args = base_config_parser._get_mandatory_arguments() + schema.add_argument('port', argument_type=int, default_value=9080) - assert len(mandatory_args) == len(expected_mandatory_args_names) + definition = schema.arguments['port'] + assert definition.name == 'port' + assert definition.argument_type is int + assert definition.default_value == 9080 - for expected_mandatory_args_name in expected_mandatory_args_names: - is_present = False - for mandatory_arg in mandatory_args: - if expected_mandatory_args_name in mandatory_arg.names: - is_present = True - break - assert is_present - -def test_get_mandatory_arguments_return_empty_list_with_no_mandatory_args(base_config_parser_no_mandatory_arguments): +def test_schema_rejects_duplicate_property(): """ - Test that mandatory arguments list is empty if parser does not have mandatory arguments + Test that a configuration property cannot be registered more than once. """ - mandatory_args = base_config_parser_no_mandatory_arguments._get_mandatory_arguments() - - assert not mandatory_args + schema = ConfigurationSectionSchema() + schema.add_argument('port') + with pytest.raises(ValueError, match='already registered'): + schema.add_argument('port') -def test_validate_check_mandatory_arguments_on_configuration(base_config_parser): +def test_schema_casts_and_applies_defaults(): """ - Test if mandatory arguments are verified by the parser + Test that schema validation casts canonical property values and applies defaults. """ - conf = load_configuration_from_json_file('basic_configuration.json') - config_longest_names = get_config_with_default_values(config=conf, arguments=base_config_parser.arguments) - config_longest_names = get_config_with_longest_argument_names(config=config_longest_names, - arguments=base_config_parser.arguments) - conf_without_mandatory_arguments = \ - load_configuration_from_json_file('basic_configuration_without_mandatory_arguments.json') + schema = ConfigurationSectionSchema() + schema.add_argument('port', argument_type=int) + schema.add_argument('enabled', argument_type=bool, default_value=False) + schema.add_argument('tags', argument_type=list) - try: - validated_config = base_config_parser.validate(conf) - assert validated_config == config_longest_names - except MissingArgumentException as e: - pytest.fail(f'Missing arguments: {e}') + result = schema.validate({'port': '9080', 'tags': 'host,socket'}) - with pytest.raises(MissingArgumentException): - _ = base_config_parser.validate(conf_without_mandatory_arguments) + assert result == { + 'port': 9080, + 'enabled': False, + 'tags': ['host', 'socket'], + } -def test_validate_accept_configuration_when_no_mandatory_arguments_exist(base_config_parser_no_mandatory_arguments): +def test_schema_copies_mutable_defaults(): """ - Test if a configuration passes the validation if there is no mandatory argument + Test that mutating a validated default does not modify later configurations. """ - conf = load_configuration_from_json_file('basic_configuration_without_mandatory_arguments.json') + schema = ConfigurationSectionSchema() + schema.add_argument('tags', argument_type=list, default_value=[]) - try: - validated_config = base_config_parser_no_mandatory_arguments.validate(conf) - assert validated_config == conf - except MissingArgumentException as e: - pytest.fail(f'Missing argument: {e}') + first_result = schema.validate({}) + first_result['tags'].append('sensor') - -def test_validate_adds_default_values_for_no_arguments_defined_in_configuration_that_have_one(base_config_parser): - """ - Test if parser add default values for arguments that are not in configuration and that have one - """ - conf = load_configuration_from_json_file('basic_configuration_without_arguments_with_default_values.json') - - expected_conf_default_values = get_config_with_default_values(config=conf, arguments=base_config_parser.arguments) - expected_conf_default_values = get_config_with_longest_argument_names(config=expected_conf_default_values, - arguments=base_config_parser.arguments) - - validated_config = base_config_parser.validate(conf) - assert validated_config == expected_conf_default_values + assert schema.validate({}) == {'tags': []} -def test_get_arguments_str_return_str_with_all_information(base_config_parser, base_config_parser_str_representation): +def test_schema_rejects_invalid_boolean_value(): """ - Test that the parser is able to return a string with all the information related to it in a correct format + Test that an unrecognized textual boolean value is rejected. """ + schema = ConfigurationSectionSchema() + schema.add_argument('enabled', argument_type=bool) - arguments_str = base_config_parser._get_arguments_str(' ') + with pytest.raises(ConfigurationError) as result: + schema.validate({'enabled': 'invalid'}) - assert arguments_str == base_config_parser_str_representation + assert result.value.path == 'enabled' + assert result.value.reason == 'Expected bool' -def test_parser_return_correct_values_for_each_argument(base_config_parser): +def test_schema_rejects_missing_mandatory_property(): """ - Test that the _parser method return correct values for different arguments in configuration + Test that a missing mandatory property is reported with its path. """ + schema = ConfigurationSectionSchema() + schema.add_argument('uri', is_mandatory=True) - args = generate_configuration_tuples_from_json_file('basic_configuration.json') - acc = {} + with pytest.raises(ConfigurationError) as result: + schema.validate({}) - expected_acc = {'argumento1': 5, - "argumento2": "this a mandatory argument", - "argument3": False, - "dded": 10.5} + assert result.value.path == 'uri' + assert result.value.reason == 'Missing required value' - args, acc = base_config_parser._parse(args, acc) - assert not args - assert acc == expected_acc - - -def test_parser_raise_an_exception_with_an_unknown_argument(base_config_parser): +@pytest.mark.parametrize(('argument_type', 'value', 'expected'), [ + (str, '', ''), + (list, '', []), +]) +def test_schema_accepts_empty_mandatory_property(argument_type, value, expected): """ - Test that the _parser method return correct values for different arguments in configuration + Test that mandatory properties require presence but may contain empty values. """ + schema = ConfigurationSectionSchema() + schema.add_argument('value', argument_type=argument_type, is_mandatory=True) - args = generate_configuration_tuples_from_json_file('basic_configuration.json') - args.append(('unknown_arg', 'This is a new argument')) - acc = {} - - with pytest.raises(NotImplementedError): - _, _ = base_config_parser._parse(args, acc) - - -##################### -# ROOT PARSER TESTS # -##################### -# Test add_argument optargs # + assert schema.validate({'value': value}) == {'value': expected} -def test_add_short_argument(): +def test_schema_rejects_unknown_property(): """ - Test if an argument when a short name is added to the short_arg string + Test that an unknown property is reported with its path. """ - parser = RootConfigParser(help_arg=False) - assert parser.short_arg == '' - parser.add_argument('a') - assert parser.short_arg == 'a:' + with pytest.raises(ConfigurationError) as result: + ConfigurationSectionSchema().validate({'unknown': 'value'}) - assert len(parser.arguments) == 1 + assert result.value.path == 'unknown' + assert result.value.reason == 'Unknown property' -def test_add_flag_argument_with_short_name(): +def test_schema_rejects_cli_alias_as_configuration_property(): """ - Test if a flag argument with a short name was added to the short_arg string + Test that schema validation only accepts canonical configuration property names. """ - parser = RootConfigParser(help_arg=False) - assert parser.short_arg == '' - parser.add_argument('a', is_flag=True) - assert parser.short_arg == 'a' + schema = ConfigurationSectionSchema() + schema.add_argument('port', argument_type=int) - assert len(parser.arguments) == 1 + with pytest.raises(ConfigurationError) as result: + schema.validate({'p': '9080'}) + assert result.value.path == 'p' + assert result.value.reason == 'Unknown property' -def test_add_several_arguments_with_short_names(): - """ - Test if the arguments were added to the short_arg string - """ - parser = RootConfigParser(help_arg=False) - assert parser.short_arg == '' - parser.add_argument('a', is_flag=True) - assert parser.short_arg == 'a' - parser.add_argument('b', is_flag=True) - assert parser.short_arg == 'ab' - parser.add_argument('c') - assert parser.short_arg == 'abc:' - parser.add_argument('d') - assert parser.short_arg == 'abc:d:' - parser.add_argument('e') - parser.add_argument('f') - parser.add_argument('g') - assert parser.short_arg == 'abc:d:e:f:g:' - - assert len(parser.arguments) == 7 - -def test_add_argument_with_two_short_names_raise_an_exception(): +def test_schema_reports_bad_type(): """ - Test if adding an argument with two short names raises a SameLengthArgumentNamesException + Test that an invalid property value reports the expected type and path. """ - parser = RootConfigParser(help_arg=False) + schema = ConfigurationSectionSchema() + schema.add_argument('port', argument_type=int) - with pytest.raises(SameLengthArgumentNamesException): - parser.add_argument('a', 'b') + with pytest.raises(ConfigurationError) as result: + schema.validate({'port': 'not-an-integer'}) - assert parser.short_arg == '' - assert not parser.arguments + assert result.value.path == 'port' + assert result.value.reason == 'Expected int' -def test_add_argument_with_long_name(): +def test_root_schema_validates_component_without_synthetic_name_property(): """ - Test if an argument with a long name is added to the long_arg list + Test that component names come from group keys rather than synthetic properties. """ - parser = RootConfigParser(help_arg=False) - assert not parser.long_arg - parser.add_argument('aaa') - assert parser.long_arg == ['aaa='] + schema = ConfigurationSchema() + schema.add_group('input', prefix='POWERAPI_INPUT_') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int, default_value=9080) + schema.add_component('input', component) - assert len(parser.arguments) == 1 + result = schema.validate({'input': {'sensor': {'type': 'socket'}}}) + assert result == {'input': {'sensor': {'type': 'socket', 'port': 9080}}} -def test_add_flag_argument_with_long(): - """ - Test if a flag argument with a long name is added to the long_arg list - """ - parser = RootConfigParser(help_arg=False) - assert not parser.long_arg - parser.add_argument('aaa', is_flag=True) - assert parser.long_arg == ['aaa'] - assert len(parser.arguments) == 1 - - -def test_add_argument_with_more_than_two_names_raise_an_exception(): +def test_root_schema_prefixes_component_validation_error(): """ - Test if adding an argument with more than two names raises a TooManyArgumentNamesException + Test that component validation errors contain the complete dotted path. """ - parser = RootConfigParser(help_arg=False) - assert not parser.long_arg - with pytest.raises(TooManyArgumentNamesException): - parser.add_argument('aaa', 'bbb', 'ccc', is_flag=True) - - with pytest.raises(TooManyArgumentNamesException): - parser.add_argument('aaa', 'b', 'c') + schema = ConfigurationSchema() + schema.add_group('input') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int) + schema.add_component('input', component) - assert not parser.long_arg - assert not parser.arguments + with pytest.raises(ConfigurationError) as result: + schema.validate({'input': {'sensor': {'type': 'socket', 'port': 'invalid'}}}) + assert result.value.path == 'input.sensor.port' + assert result.value.reason == 'Expected int' + assert result.value.msg == 'Invalid configuration at "input.sensor.port": Expected int' -# full parsing test # -def check_parsing_result(parser, input_str, outputs): +@pytest.mark.parametrize('configuration', [ + {'input': {'sensor': {}}}, + {'input': {'sensor': {'type': 'unknown'}}}, + {'input': {'sensor': {'type': {}}}}, +]) +def test_root_schema_requires_known_component_type(configuration): """ - Check that input_str is correctly parsed by parser + Test that components require a registered component type. """ - result = parser.parse(input_str.split()) + schema = ConfigurationSchema() + schema.add_group('input') - assert len(result) == len(outputs) - assert result == outputs + with pytest.raises(ConfigurationError) as result: + schema.validate(configuration) + assert result.value.path == 'input.sensor.type' -def test_parsing_of_config_of_empty_parser(): - """ - Test the parsing of strings with an empty base parser and retrieve the following results: - - - "": {} - - "-z": UnknownArgException(z) - - "-a": UnknownArgException(a) - - "-a --sub toto -b": UnknownArgException(a) - - "-b": UnknownArgException(b) - Parser description: - - - root parser arguments: None +def test_root_schema_validates_fixed_section_without_component_type(): """ - parser = RootConfigParser(help_arg=False) - - check_parsing_result(parser, '', {}) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-z', None) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-a', None) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-a --sub toto -b', None) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-b', None) - - -def test_parsing_of_config_of_a_parser(): + Test that a fixed group section is validated without a component type. """ - Test the parsing of strings with a not empty root parser and retrieve the following results: + schema = ConfigurationSchema() + schema.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + schema.add_section('formula', 'smartwatts', smartwatts) - - "": {} - - "-z": UnknownArgException(z) - - "-a": {a: True} - - "-a --sub toto -b": UnknownArgException(sub) - - "-b": UnknownArgException(b) + result = schema.validate({ + 'formula': { + 'smartwatts': { + 'learn-error-window-size': '10', + }, + }, + }) - Parser description: - - - root parser arguments: -a - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', is_flag=True, action=store_true) + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, + } - check_parsing_result(parser, '', {}) - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-z', None) - - check_parsing_result(parser, '-a', {'a': True}) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-a --sub toto -b', None) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-b', None) - - -def test_parsing_of_config_of_a_parser_with_a_subgroup_parser(): +def test_root_schema_applies_defaults_from_fixed_section(): """ - Test the parsing of strings with a not empty rooy parser and retrieve the following results: - - - "" : {} - - "-z": UnknownArgException(z) - - "-a": {a: True} - - "-a --sub toto -b": NoNameSpecifiedForSubgroupException - - "-a --sub toto -b --name titi" : {a:True, sub: { titi: { 'type': 'toto', b: True}}} - - "-b": BadContextException(b, [toto]) - - Parser description: - - - root parser arguments: -a - - subparser toto bound to the argument sub with sub arguments: -b and --name + Test that fixed section defaults are applied when the section is omitted. """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', is_flag=True, action=store_true) - parser.add_subgroup(subgroup_type='sub') - - subparser = SubgroupConfigParser('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - parser.add_subgroup_parser(subgroup_type='sub', subgroup_parser=subparser) - - check_parsing_result(parser, '', {}) + schema = ConfigurationSchema() + schema.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int, default_value=10) + schema.add_section('formula', 'smartwatts', smartwatts) - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-z', None) + result = schema.validate({}) - check_parsing_result(parser, '-a', {'a': True}) + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, + } - with pytest.raises(NoNameSpecifiedForSubgroupException): - check_parsing_result(parser, '-a --sub toto -b', {}) - check_parsing_result(parser, '-a --sub toto -b --name titi', - {'a': True, 'sub': {'titi': {'type': 'toto', 'b': True}}}) - - with pytest.raises(BadContextException): - check_parsing_result(parser, '-b', None) - - -def test_parsing_of_config_with_several_subgroups_with_the_same_name_in_a_parser_with_a_subgroup_parser(): +def test_root_schema_rejects_component_type_in_fixed_section(): """ - Test the parsing of several subgroups with the same name. The result is: - {sub:{'titi' : {'type': 'toto'}, 'tutu': {'type': 'toto', 'b':True}, 'tata': {'type': 'toto'}}} - - The subgroups are created with the following cli: - --sub toto --name titi --sub toto -b --name tutu --sub toto --name tata - - Parser description: - - - root parser arguments: None - - subparser toto bound to the argument sub with sub arguments: -b and -n --name - + Test that a fixed group section does not accept a component type selector. """ - parser = RootConfigParser(help_arg=False) - parser.add_subgroup(subgroup_type='sub') + schema = ConfigurationSchema() + schema.add_group('formula') + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) - subparser = SubgroupConfigParser('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - parser.add_subgroup_parser(subgroup_type='sub', subgroup_parser=subparser) + with pytest.raises(ConfigurationError) as result: + schema.validate({'formula': {'smartwatts': {'type': 'smartwatts'}}}) - check_parsing_result(parser, '--sub toto --name titi --sub toto -b --name tutu --sub toto --name tata', - {'sub': {'titi': {'type': 'toto'}, 'tutu': {'type': 'toto', 'b': True}, - 'tata': {'type': 'toto'}}}) + assert result.value.path == 'formula.smartwatts.type' + assert result.value.reason == 'Unknown property' -def test_parsing_of_several_subgroups_with_different_name_in_a_parser_with_several_subgroup_parsers(): +def test_root_schema_rejects_unregistered_type_property(): """ - Test the parsing of several subgroups with different name. The result is: - {sub:{'titi' : {'type': 'toto'}, 'tete': {'type': 'tutu'}}} - - The subgroups are created with the following cli: - --sub toto --name titi --sub tutu --name tete --sub tata --name tate - - Parser description: - - - root parser arguments: None - - subparser toto bound to the argument sub with sub argument: -n --name - - subparser tutu bound to the argument sub with sub argument: -n --name - - subparser tata bound to the argument sub with sub argument: -n --name + Test that the component type selector is not accepted as a root property. """ - parser = RootConfigParser(help_arg=False) + with pytest.raises(ConfigurationError) as result: + ConfigurationSchema().validate({'type': 'socket'}) - parser.add_subgroup(subgroup_type='sub') + assert result.value.path == 'type' + assert result.value.reason == 'Unknown property' - subparser = SubgroupConfigParser('toto') - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) - subparser = SubgroupConfigParser('tutu') - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) - - subparser = SubgroupConfigParser('tata') - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) - - check_parsing_result(parser, '--sub toto --name titi --sub tutu --name tete --sub tata --name tate', - {'sub': {'titi': {'type': 'toto'}, 'tete': {'type': 'tutu'}, 'tate': {'type': 'tata'}}}) - - -def test_parsing_of_config_with_repeated_subgroups_names_raise_an_exception(): +@pytest.mark.parametrize(('configuration', 'path'), [ + ({'input': []}, 'input'), + ({'input': {'sensor': []}}, 'input.sensor'), +]) +def test_root_schema_rejects_non_dictionary_group_values(configuration, path): """ - Test if an SubgroupAlreadyExistException is raised with two subgroups of the same name. - The subgroups are created with the following cli: - --sub toto --name titi --sub toto --name titi - - Parser description: - - - root parser arguments: None - - subparser toto bound to the argument sub with sub arguments: -b (flag) and -n --name + Test that configuration groups and their entries must be dictionaries. """ - parser = RootConfigParser(help_arg=False) - parser.add_subgroup(subgroup_type='sub') + schema = ConfigurationSchema() + schema.add_group('input') - subparser = SubgroupConfigParser('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) + with pytest.raises(ConfigurationError) as result: + schema.validate(configuration) - with pytest.raises(SubgroupAlreadyExistException): - check_parsing_result(parser, '--sub toto --name titi --sub toto --name titi', None) + assert result.value.path == path + assert result.value.reason == 'Expected dict' -def test_parsing_of_argument_with_val(): +def test_json_loader_rejects_shortened_property_names(tmp_path): """ - Test the parsing of strings with a root parser and retrieve the following results : - - - "-c" : MissingValue(c) - - "-c 1" : {c : 1} - - Parser description : - - - root parser arguments : -c (not flag) + Test that JSON loading only accepts registered configuration property names. """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('c') + schema = ConfigurationSchema() + schema.add_argument('port', argument_type=int, default_value=9080) + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(json.dumps({'p': '9090'}), encoding='utf-8') - with pytest.raises(MissingValueException): - check_parsing_result(parser, '-c', None) + configuration = JSONConfigLoader().load(str(config_file)) + with pytest.raises(ConfigurationError) as result: + schema.validate(configuration) - check_parsing_result(parser, '-c 1', {'c': '1'}) + assert result.value.path == 'p' + assert result.value.reason == 'Unknown property' -# multi name tests # -def test_parsing_of_argument_with_long_short_names_and_val(): +def test_json_loader_loads_fixed_section_without_component_type(tmp_path): """ - Test if the parsing of an argument with long and short names and a value works correctly. - The value is only bound to the long name in the parsing result. - - Parser description: - - - root parser arguments: -c --coco - + Test that JSON loading preserves a fixed group section without a type. """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('c', 'coco') - - check_parsing_result(parser, '-c 1', {'coco': '1'}) + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(json.dumps({ + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, + }), encoding='utf-8') + result = JSONConfigLoader().load(str(config_file)) -def test_add_argument_with_two_long_names_with_same_length_raise_an_exception(): - """ - Test if the parser raise an exception SameLengthArgumentNamesException when a - long argument with the same length is added. No subgroup parse must be added + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, + } - Parser description: - - - root parser arguments: None +def test_json_loader_returns_empty_configuration_without_file(): """ - parser = RootConfigParser(help_arg=False) - with pytest.raises(SameLengthArgumentNamesException): - parser.add_argument('coco', 'dodo') - - assert len(parser.arguments) == 0 - - -# Type tests # -def test_parsing_arguments_with_val_has_correct_default_type(): + Test that JSON loading returns an empty configuration when no file is selected. """ - Test if parsing arguments created with default type have default type in the parsing result. + assert JSONConfigLoader().load(None) == {} - Parser description: - - root parser arguments: -a, -b --bb, -c --cc +def test_json_loader_reports_invalid_json_as_configuration_error(tmp_path): """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a') - parser.add_argument('b', 'bb') - parser.add_argument('c', 'cc') - result = parser.parse('-a 1 --bb string -c string_again'.split()) - assert len(result) == 3 - assert 'a' in result - assert isinstance(result['a'], str) - assert result['a'] == '1' - - assert 'bb' in result - assert isinstance(result['bb'], str) - assert result['bb'] == 'string' - - assert 'cc' in result - assert isinstance(result['cc'], str) - assert result['cc'] == 'string_again' - - -def test_parsing_arguments_with_val_has_correct_type(): + Test that invalid JSON is exposed as a ConfigurationError with decoder details. """ - Test if parsing arguments created with no default type have correct type in the parsing result. + config_file = tmp_path / 'powerapi-pytest.json' + invalid_json = '{"stream": true,}' + config_file.write_text(invalid_json, encoding='utf-8') - Parser description: + with pytest.raises(json.JSONDecodeError) as decode_error: + json.loads(invalid_json) - - root parser arguments: -a, -b --bb, -c --cc + with pytest.raises(ConfigurationError) as result: + JSONConfigLoader().load(str(config_file)) - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', argument_type=int) - result = parser.parse('-a 1'.split()) - assert len(result) == 1 - assert 'a' in result - assert isinstance(result['a'], int) + assert result.value.path is None + assert result.value.reason == f'Invalid JSON in configuration file "{config_file}": {decode_error.value}' -def test_parsing_argument_with_wrong_type_raise_an_exception(): +@pytest.mark.parametrize('content', ['[]', 'null', '"value"']) +def test_json_loader_rejects_non_object_root(content, tmp_path): """ - Test if parsing arguments with a given value that can be parsed to the defined type raises an exception - - Parser description: - - - root parser arguments: -a --xx + Test that a JSON configuration must contain an object at its root. """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', 'xx', argument_type=int) + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(content, encoding='utf-8') - with pytest.raises(BadTypeException): - parser.parse('-a a'.split()) + with pytest.raises(ConfigurationError) as result: + JSONConfigLoader().load(str(config_file)) - with pytest.raises(BadTypeException): - parser.parse('--xx toto'.split()) + assert result.value.reason == 'Expected a JSON object' -# parse with ComponentSubparser tests # -def test_add_subgroup_parser_that_already_exist_raise_an_exception(): +def test_environment_loader_preserves_root_and_component_format(monkeypatch): """ - Test if adding a subgroup parser that already exists raises an AlreadyAddedArgumentException. - The subgroup parsers of root parser must not be affected - - Parser description: - - - root parser arguments: None - - subparser titi bound to the argument toto with sub arguments: -n + Test that environment loading preserves raw root and nested component values. """ - parser = RootConfigParser(help_arg=False) - parser.add_subgroup(subgroup_type='toto') - subparser = SubgroupConfigParser('titi') - subparser.add_argument('n', 'name') - - repeated_subparser = SubgroupConfigParser('titi') - repeated_subparser.add_argument('n', 'name') - repeated_subparser.add_argument('arg2', 'a') - - parser.add_subgroup_parser('toto', subparser) + schema = ConfigurationSchema() + schema.add_argument_prefix('POWERAPI_') + schema.add_argument('stream', argument_type=bool) + schema.add_group('input', prefix='POWERAPI_INPUT_') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int) + schema.add_component('input', component) + monkeypatch.setenv('POWERAPI_STREAM', 'true') + monkeypatch.setenv('POWERAPI_INPUT_SENSOR_TYPE', 'socket') + monkeypatch.setenv('POWERAPI_INPUT_SENSOR_PORT', '9080') - with pytest.raises(AlreadyAddedArgumentException): - parser.add_subgroup_parser('toto', repeated_subparser) + result = EnvironmentConfigLoader(schema).load() - assert len(parser.subgroup_parsers) == 1 - assert len(parser.subgroup_parsers['toto'].subparsers['titi'].arguments) == 2 + assert result == { + 'stream': 'true', + 'input': {'sensor': {'type': 'socket', 'port': '9080'}}, + } -def test_add_subgroup_parser_with_argument_name_work(): +def test_environment_loader_ignores_group_without_prefix(monkeypatch): """ - Test if adding a subgroup parser with a name argument works - Parser description: - - - root parser arguments: None - - subparser titi bound to the argument sub with sub arguments: -a --aaa, -n --name + Test that a prefixless group does not hide root values or inspect unrelated environment variables. """ - parser = RootConfigParser(help_arg=False) - parser.add_subgroup(subgroup_type='sub') - subparser = SubgroupConfigParser('titi') - subparser.add_argument('a', 'aaa', is_flag=True, action=store_true, default_value=False) - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) + schema = ConfigurationSchema() + schema.add_argument_prefix('POWERAPI_') + schema.add_argument('stream', argument_type=bool) + schema.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + schema.add_section('formula', 'smartwatts', smartwatts) + monkeypatch.setenv('POWERAPI_STREAM', 'true') + monkeypatch.setenv('SMARTWATTS_LEARN_ERROR_WINDOW_SIZE', '10') - assert len(parser.subgroup_parsers) == 1 - assert len(parser.subgroup_parsers['sub'].subparsers['titi'].arguments) == 4 - assert 'a' in parser.subgroup_parsers['sub'].subparsers['titi'].arguments - assert 'aaa' in parser.subgroup_parsers['sub'].subparsers['titi'].arguments - assert 'n' in parser.subgroup_parsers['sub'].subparsers['titi'].arguments - assert 'name' in parser.subgroup_parsers['sub'].subparsers['titi'].arguments + result = EnvironmentConfigLoader(schema).load() + assert result == {'stream': 'true'} -def test_add_subgroup_parser_without_argument_name_raise_an_exception(): - """ - Test if adding a subgroup parser with no argument 'name' raises a - SubgroupParserWithoutNameArgumentException - Parser description: - - root parser arguments: None +def test_environment_loader_preserves_fixed_section_without_component_type(monkeypatch): """ - parser = RootConfigParser(help_arg=False) - subparser = SubgroupConfigParser('titi') - - with pytest.raises(SubgroupParserWithoutNameArgumentException): - parser.add_subgroup_parser('toto', subparser) - - assert len(parser.subgroup_parsers) == 0 - - -def test_parsing_empty_string_return_an_empty_dict(): + Test that environment loading preserves a raw fixed group section. """ - Test that the result of parsing an empty string is an empty dict - - Parser description: + schema = ConfigurationSchema() + schema.add_group('formula', prefix='POWERAPI_FORMULA_') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + schema.add_section('formula', 'smartwatts', smartwatts) + monkeypatch.setenv('POWERAPI_FORMULA_SMARTWATTS_LEARN_ERROR_WINDOW_SIZE', '10') - - root parser arguments: -a, -b + result = EnvironmentConfigLoader(schema).load() - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', default_value=1) - parser.add_argument('b', default_value=False, argument_type=bool) - result = parser.parse(''.split()) - assert len(result) == 0 + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': '10', + }, + }, + } -def test_parsing_dict_return_configuration_with_arguments_long_name( - root_config_parser_with_mandatory_and_optional_arguments, - test_files_path): +def test_environment_loader_rejects_component_type_in_fixed_section(monkeypatch): """ - Test that the result of parsing a dictionary configuration with long and short names for arguments - returns a dictionary configuration only with long names for arguments - - Parser description: - - - root parser arguments: -a, -1 --argument1, -2 --argumento2, --arg3 --argument3, -d --arg4, --arg5 -5 - + Test that an environment fixed section does not accept a component type selector. """ - config_file = 'root_manager_basic_configuration_with_long_and_short_names.json' - expected_result = load_configuration_from_json_file(config_file) - expected_result['argumento2'] = expected_result.pop('2') - expected_result['arg5'] = expected_result.pop('5') + schema = ConfigurationSchema() + schema.add_group('formula', prefix='POWERAPI_FORMULA_') + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) + monkeypatch.setenv('POWERAPI_FORMULA_SMARTWATTS_TYPE', 'smartwatts') - result = root_config_parser_with_mandatory_and_optional_arguments.parse_config_dict( - file_name=test_files_path + '/' + config_file) - assert result == expected_result + configuration = EnvironmentConfigLoader(schema).load() + with pytest.raises(ConfigurationError) as result: + schema.validate(configuration) + assert result.value.path == 'formula.smartwatts.type' + assert result.value.reason == 'Unknown property' -############################ -# SUBGROUP_PARSER TEST # -############################ -def test_subgroup_parser_empty(subgroup_parser): - """ - Test subgroup parser on an empty token list. - Must return an empty dictionary as parse result and an empty token - list - """ - assert subgroup_parser.parse([]) == ([], {}) - - -def test_subgroup_parser_parsing_an_argument(subgroup_parser): - """ - Test component_subparser, parse a token list which contain only subparser - argument [('a', '')]. - - must return return a dictionary {'a':''} as parse result - and an empty token list - - """ - assert subgroup_parser.parse([('a', '')]) == ([], {'a': None}) - - -def test_subgroup_parsing_parser_several_arguments(subgroup_parser): - """ - Test subgroup parser parses a token list which contains subparser - argument and arguments from other parser[('a', ''), ('b', '')]. - must return a dictionary {'a':''} as parse result - and a token list that contains the unparsed arguments : [('b', '')]. - - """ - assert subgroup_parser.parse([('a', ''), ('b', '')]) == ([('b', '')], - {'a': None}) - - -def test_subgroup_parser_parsing_empty_argument_list_return_an_empty_dict(): - """ - Test that the result of parsing an empty string is an empty dict +def test_environment_loader_preserves_component_without_type(monkeypatch): """ - subparser = SubgroupConfigParser('toto') - subparser.add_argument('a', default_value=1) - subparser.add_argument('x', default_value=False) - _acc, result = subparser.parse([]) - assert len(result) == 0 - - -def test_normalize_configuration_dict_select_long_names_for_every_argument_in_config(base_config_parser): - """ - Test that every argument in a configuration has at the end its long name after the normalization + Test that environment loading preserves a partial component for later merging. """ - conf = load_configuration_from_json_file(file_name='basic_configuration_with_long_and_short_names.json') - expected_conf = load_configuration_from_json_file(file_name='basic_configuration.json') - expected_conf['argumento1'] = expected_conf.pop('arg1') - expected_conf['arg5'] = conf['5'] - expected_conf['dded'] = expected_conf.pop('arg4') + schema = ConfigurationSchema() + schema.add_group('input', prefix='POWERAPI_INPUT_') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int) + schema.add_component('input', component) + monkeypatch.setenv('POWERAPI_INPUT_SENSOR_PORT', '9080') - result = base_config_parser.normalize_configuration(conf=conf) + assert EnvironmentConfigLoader(schema).load() == { + 'input': {'sensor': {'port': '9080'}}, + } - assert result == expected_conf - -def test_normalize_configuration_dict_select_long_names_for_every_argument_in_config_for_root_config_parser( - root_config_parser_with_mandatory_and_optional_arguments): +def test_root_schema_rejects_overlapping_environment_prefixes(): """ - Test that every argument in a configuration has at the end its long name after the normalization + Test that overlapping root environment prefixes are rejected. """ - conf = load_configuration_from_json_file(file_name='basic_configuration_with_long_and_short_names.json') - expected_conf = load_configuration_from_json_file(file_name='basic_configuration.json') - expected_conf['argument1'] = expected_conf.pop('arg1') - expected_conf['arg5'] = conf['5'] - - result = root_config_parser_with_mandatory_and_optional_arguments.normalize_configuration(conf=conf) + schema = ConfigurationSchema() + schema.add_argument_prefix('POWERAPI_') - assert result == expected_conf + with pytest.raises(ValueError, match='conflicts with'): + schema.add_argument_prefix('POWERAPI_INPUT_') -def test_normalize_config_dict_select_long_names_for_every_argument_in_config_with_subgroups_for_root_config_parser( - root_config_parser_with_subgroups): +def test_root_schema_rejects_duplicate_group(): """ - Test that every argument in a configuration has at the end its long name after the normalization + Test that a configuration group cannot be registered more than once. """ - conf_file = 'basic_configuration_with_subgroups.json' - conf = load_configuration_from_json_file(file_name=conf_file) - expected_conf = load_configuration_from_json_file(file_name=conf_file) - expected_conf['argument1'] = expected_conf.pop('arg1') - expected_conf['argument3'] = expected_conf.pop('arg3') - expected_conf['arg5'] = expected_conf.pop('5') - expected_conf['g2']['g2_sg1']['a4'] = expected_conf['g2']['g2_sg1'].pop('a4') + schema = ConfigurationSchema() + schema.add_group('input') - result = root_config_parser_with_subgroups.normalize_configuration(conf=conf) + with pytest.raises(ValueError, match='already registered'): + schema.add_group('input') - assert result == expected_conf - -def test_parse_config_environment_variables_return_correct_configuration(root_config_parser_with_subgroups): +def test_root_schema_rejects_group_matching_property(): """ - Test that the parsing of environment variables works correctly + Test that a group cannot reuse a registered root property name. """ - conf_file = 'basic_configuration_with_subgroups.json' - - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=conf_file, - simple_argument_prefix=root_config_parser_with_subgroups.arguments_prefix[0], - group_arguments_prefix=root_config_parser_with_subgroups.get_groups_prefixes()) - - expected_conf = load_configuration_from_json_file(file_name=conf_file) - - expected_conf['g1']['g1-sg1'] = {} - expected_conf['g1']['g1-sg2'] = {} - expected_conf['g2']['g2-sg1'] = {} - g1_sg1 = expected_conf['g1'].pop('g1_sg1') - g1_sg2 = expected_conf['g1'].pop('g1_sg2') - g2_sg1 = expected_conf['g2'].pop('g2_sg1') - - expected_conf['g1']['g1-sg1']['a1'] = str(g1_sg1['a1']) - expected_conf['g1']['g1-sg1']['a2'] = bool(g1_sg1['a2']) - expected_conf['g1']['g1-sg1']['a3'] = str(g1_sg1['a3']) - expected_conf['g1']['g1-sg1']['type'] = str(g1_sg1['type']) - - expected_conf['g1']['g1-sg2']['a1'] = str(g1_sg2['a1']) - expected_conf['g1']['g1-sg2']['a2'] = g1_sg2['a2'] - expected_conf['g1']['g1-sg2']['a3'] = g1_sg2['a3'] - expected_conf['g1']['g1-sg2']['type'] = g1_sg2['type'] - - expected_conf['argument1'] = expected_conf.pop('arg1') - expected_conf['argument3'] = expected_conf.pop('arg3') - expected_conf['arg5'] = expected_conf.pop('5') - - expected_conf['g2']['g2-sg1']['a1'] = float(g2_sg1['a1']) - expected_conf['g2']['g2-sg1']['a3'] = g2_sg1['a3'] - expected_conf['g2']['g2-sg1']['a4'] = g2_sg1['a4'] - expected_conf['g2']['g2-sg1']['type'] = g2_sg1['type'] - - result = root_config_parser_with_subgroups.parse_config_environment_variables() - - assert result == expected_conf + schema = ConfigurationSchema() + schema.add_argument('input') - remove_environment_variables_configuration(variables_names=created_environment_variables) + with pytest.raises(ValueError, match='already registered as a property'): + schema.add_group('input') -def test_parse_config_environment_variables_with_wrong_argument_raise_an_exception( - root_config_parser_with_subgroups): +def test_root_schema_rejects_property_matching_group(): """ - Test that the parsing of environment variables raises a BadTypeException with wrong types + Test that a root property cannot reuse a registered group name. """ - conf_file = 'basic_configuration_with_subgroups_wrong_argument_type_value.json' + schema = ConfigurationSchema() + schema.add_group('input') - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=conf_file, - simple_argument_prefix=root_config_parser_with_subgroups.arguments_prefix[0], - group_arguments_prefix=root_config_parser_with_subgroups.get_groups_prefixes()) + with pytest.raises(ValueError, match='already registered as a group'): + schema.add_argument('input') - with pytest.raises(BadTypeException): - _ = root_config_parser_with_subgroups.parse_config_environment_variables() - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_add_subgroup(root_config_parser_with_mandatory_and_optional_arguments): +def test_root_schema_rejects_duplicate_component_type(): """ - Test that a subgroup is correctly added + Test that a component type cannot be registered twice in one group. """ - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 0 - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='sub') - - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 1 - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='sub1') + schema = ConfigurationSchema() + schema.add_group('input') + schema.add_component('input', ComponentSchema('socket')) - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 2 + with pytest.raises(ValueError, match='already registered'): + schema.add_component('input', ComponentSchema('socket')) - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='sub2') - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 3 - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='sub3') - - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 4 - - -def test_add_repeated_subgroup_raise_an_exception(root_config_parser_with_subgroups): +def test_root_schema_rejects_component_for_unknown_group(): """ - Test that adding a repeated subgroup raises an AlreadyAddedSubgroupException + Test that a component cannot be registered in an unknown group. """ - assert len(root_config_parser_with_subgroups.subgroup_parsers) == 2 - - with pytest.raises(AlreadyAddedSubgroupException): - root_config_parser_with_subgroups.add_subgroup(subgroup_type='g2') + schema = ConfigurationSchema() - assert len(root_config_parser_with_subgroups.subgroup_parsers) == 2 + with pytest.raises(ValueError, match='is not registered'): + schema.add_component('input', ComponentSchema('socket')) -def test_get_subgroups_prefix(root_config_parser_with_subgroups): +def test_root_schema_rejects_section_for_unknown_group(): """ - Test that all the subgroups prefixes are returned + Test that a fixed section cannot be registered in an unknown group. """ - expected_prefixes = ['TEST_G1_', 'TEST_G2_'] + schema = ConfigurationSchema() - result = root_config_parser_with_subgroups.get_groups_prefixes() - assert len(result) == len(expected_prefixes) - assert result == expected_prefixes + with pytest.raises(ValueError, match='is not registered'): + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) -def test_get_longest_arguments_names(root_config_parser_with_subgroups): +def test_root_schema_rejects_duplicate_section(): """ - Test that all the arguments of the parser are returned + Test that a fixed section name cannot be registered twice in one group. """ - expected_arguments_names = ['help', 'a', 'argument1', 'argumento2', 'argument3', 'arg4', 'arg5', 'g1', 'g2'] + schema = ConfigurationSchema() + schema.add_group('formula') + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) - result = root_config_parser_with_subgroups.get_longest_arguments_names() - assert len(result) == len(expected_arguments_names) - assert result == expected_arguments_names + with pytest.raises(ValueError, match='already registered'): + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) diff --git a/tests/unit/cli/test_config_validator.py b/tests/unit/cli/test_config_validator.py deleted file mode 100644 index b169e0ac..00000000 --- a/tests/unit/cli/test_config_validator.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (c) 2023, INRIA -# Copyright (c) 2023, University of Lille -# All rights reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest - -from powerapi.cli import ConfigValidator -from powerapi.exception import NotAllowedArgumentValueException, MissingArgumentException, FileDoesNotExistException, \ - UnexistingActorException, PowerAPIException -from tests.utils.cli.base_config_parser import load_configuration_from_json_file - - -def test_config_in_stream_mode_with_csv_input_raise_an_exception(invalid_csv_io_stream_config): - """ - Test that an invalid configuration with stream mode enabled and csv as input is detected by the ConfigValidator - """ - with pytest.raises(NotAllowedArgumentValueException): - ConfigValidator.validate(invalid_csv_io_stream_config) - - -def test_config_in_postmortem_mode_with_csv_input_is_validated(create_empty_files_from_config, - csv_io_postmortem_config): - """ - Test that a valid configuration is detected by the ConfigValidator when stream mode is disabled. - The files list for the input has to be transformed into a list - """ - try: - expected_result = load_configuration_from_json_file('csv_input_output_stream_mode_enabled_configuration.json') - expected_result['stream'] = False - - ConfigValidator.validate(csv_io_postmortem_config) - assert csv_io_postmortem_config == expected_result - - except NotAllowedArgumentValueException as e: - pytest.fail(f'Invalid argument value: {e}') - - -def test_valid_config_postmortem_csv_input_without_optional_arguments_is_validated(create_empty_files_from_config, - csv_io_postmortem_config_without_optional_arguments): - """ - Test that a valid configuration is detected by the ConfigValidator when stream mode is disabled. - Default values has to be defined and the files list for the input has to be transformed into a list - """ - expected_result = csv_io_postmortem_config_without_optional_arguments.copy() - for current_input in expected_result['input']: - expected_result['input'][current_input]['name'] = 'default_puller' - expected_result['input'][current_input]['model'] = 'HWPCReport' - - expected_result['stream'] = False - expected_result['verbose'] = False - - ConfigValidator.validate(csv_io_postmortem_config_without_optional_arguments) - - assert csv_io_postmortem_config_without_optional_arguments == expected_result - - -def test_config_with_csv_input_with_files_that_do_not_exist_raise_an_exception(csv_io_postmortem_config): - """ - Test that validation of a configuration indicating files that do not exist in csv as input raises a - FileDoesNotExistException - """ - with pytest.raises(FileDoesNotExistException) as raised_exception: - ConfigValidator.validate(csv_io_postmortem_config) - - assert raised_exception.value.file_name == '/tmp/rapl.csv' - - -def test_config_without_inputs_raise_an_exception(config_without_input): - """ - Test that validation of an invalid configuration without inputs raises a MissingArgumentException - """ - with pytest.raises(MissingArgumentException) as raised_exception: - ConfigValidator.validate(config_without_input) - - assert raised_exception.value.argument_name == 'input' - - -def test_config_without_outputs_raise_an_exception(config_without_output): - """ - Test that validation of an invalid configuration without outputs raises a MissingArgumentException - """ - with pytest.raises(MissingArgumentException) as raised_exception: - ConfigValidator.validate(config_without_output) - - assert raised_exception.value.argument_name == 'output' - - -def test_config_with_pre_processor_but_without_puller_raise_an_exception(pre_processor_config_without_puller): - """ - Test that validation of a configuration with pre-processors but without a related puller raises a - MissingArgumentException - """ - with pytest.raises(MissingArgumentException) as raised_exception: - ConfigValidator.validate(pre_processor_config_without_puller) - - assert raised_exception.value.argument_name == 'puller' - - -def test_config_with_empty_pre_processor_pass_validation(empty_pre_processor_config): - """ - Test that validation of a configuration without pre-processors passes validation - """ - try: - ConfigValidator.validate(empty_pre_processor_config) - except MissingArgumentException as e: - pytest.fail(f'Missing argument: {e}') - - -def test_config_with_pre_processor_with_unexisting_puller_actor_raise_an_exception( - pre_processor_with_unexisting_puller_configuration): - """ - Test that validation of a configuration with unexisting actors raise an exception - """ - with pytest.raises(UnexistingActorException) as raised_exception: - ConfigValidator.validate(pre_processor_with_unexisting_puller_configuration) - - assert raised_exception.value.actor == \ - pre_processor_with_unexisting_puller_configuration['pre-processor']['my_processor']['puller'] - - -def test_validation_of_correct_configuration_with_pre_processors(pre_processor_complete_configuration): - """ - Test that a correct configuration with processors and bindings passes the validation - """ - try: - ConfigValidator.validate(pre_processor_complete_configuration) - except PowerAPIException as e: - pytest.fail(f'Configuration validation failed: {e}') - - -def test_validation_of_correct_configuration_without_pre_processors_and_bindings(output_input_configuration): - """ - Test that a correct configuration without pre-processors passes the validation - """ - try: - ConfigValidator.validate(output_input_configuration) - except PowerAPIException as e: - pytest.fail(f'Configuration validation failed: {e}') diff --git a/tests/unit/cli/test_generator_k8s.py b/tests/unit/cli/test_generator_k8s.py index 8c5eb5c3..9726caeb 100644 --- a/tests/unit/cli/test_generator_k8s.py +++ b/tests/unit/cli/test_generator_k8s.py @@ -45,7 +45,7 @@ def k8s_processor_config(): 'verbose': True, 'pre-processor': { 'pytest-k8s-preprocessor': { - 'type': 'k8s', + 'type': 'kubernetes', 'api-mode': 'manual', 'api-host': 'https://127.0.0.1:36599', 'api-key': 'pytest-token-powerapi', diff --git a/tests/unit/cli/test_parsing_manager.py b/tests/unit/cli/test_parsing_manager.py index 8463b567..c19ade09 100644 --- a/tests/unit/cli/test_parsing_manager.py +++ b/tests/unit/cli/test_parsing_manager.py @@ -27,1246 +27,205 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import json import sys -import pytest - -from powerapi.cli.config_parser import store_true, store_val -from powerapi.cli.parsing_manager import RootConfigParsingManager, \ - SubgroupConfigParsingManager -from powerapi.exception import AlreadyAddedArgumentException, BadTypeException, UnknownArgException, \ - MissingValueException, SubgroupAlreadyExistException, SubgroupParserWithoutNameArgumentException, \ - NoNameSpecifiedForSubgroupException, AlreadyAddedSubparserException, \ - SameLengthArgumentNamesException, BadContextException -from tests.utils.cli.base_config_parser import load_configuration_from_json_file, \ - define_environment_variables_configuration_from_json_file, \ - remove_environment_variables_configuration - - -############### -# PARSER TEST # -############### -def test_add_argument_to_cli_parser_that_already_exist_raise_an_exception(): - """ - Tests if adding an argument that already exists to a parser raises an - AlreadyAddedArgumentException - """ - - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a') - - with pytest.raises(AlreadyAddedArgumentException): - parser_manager.add_argument('a') - - with pytest.raises(AlreadyAddedArgumentException): - parser_manager.add_argument('help') - - assert len(parser_manager.cli_parser.arguments) == 3 # help argument + a argument - - -##################### -# MAIN PARSER TESTS # -##################### -def test_add_flag_arguments_with_short_name_to_cli_parser(): - """ - Test that adding flag arguments with a short name to cli parser modified short_arg string - """ - parser_manager = RootConfigParsingManager() - assert parser_manager.cli_parser.short_arg == 'h' - parser_manager.add_argument('a', is_flag=True) - parser_manager.add_argument('x', is_flag=True) - assert parser_manager.cli_parser.short_arg == 'hax' - - -def test_add_flag_arguments_and_no_flag_arguments_with_short_name_to_cli_parser(): - """ - Test if adding arguments (flag and no flag) to the parser modified short_arg string. - long_arg list is not changed - """ - parser_manager = RootConfigParsingManager() - assert parser_manager.cli_parser.short_arg == 'h' - parser_manager.add_argument('a', is_flag=True) - assert parser_manager.cli_parser.short_arg == 'ha' - parser_manager.add_argument('b') - assert parser_manager.cli_parser.short_arg == 'hab:' - parser_manager.add_argument('c', is_flag=True) - assert parser_manager.cli_parser.short_arg == 'hab:c' - - assert len(parser_manager.cli_parser.long_arg) == 1 # Only help is arg argument - - assert parser_manager.cli_parser.long_arg == ["help"] # Only help is arg argument - - -def test_add_arguments_with_long_name_to_cli_parser(): - """ - Test if adding arguments with long name to cli parser modifies long_arg list. - short_arg string is not changed - """ - parser_manager = RootConfigParsingManager() - assert parser_manager.cli_parser.long_arg == ['help'] - parser_manager.add_argument('aaa') - assert parser_manager.cli_parser.long_arg == ['help', 'aaa='] - parser_manager.add_argument('xx') - assert parser_manager.cli_parser.long_arg == ['help', 'aaa=', 'xx='] - - assert parser_manager.cli_parser.short_arg == 'h' +import pytest -def test_add_flag_arguments_with_long_name_to_cli_parser(): - """ - Test if adding a flag arguments with long to the parser modifies the long_arg list. - short_arg string is not changed - """ - parser_manager = RootConfigParsingManager() - assert parser_manager.cli_parser.long_arg == ['help'] - parser_manager.add_argument('aaa', is_flag=True) - assert parser_manager.cli_parser.long_arg == ['help', 'aaa'] - parser_manager.add_argument('tttt', is_flag=True) - assert parser_manager.cli_parser.long_arg == ['help', 'aaa', 'tttt'] - - assert parser_manager.cli_parser.short_arg == 'h' - - -# full parsing test # -def check_parse_cli_result(parser_manager: RootConfigParsingManager, input_str: str, outputs: dict): - """ - Check that input_str is correctly parsed by parser - """ - result = parser_manager._parse_cli(input_str.split()) - - assert len(result) == len(outputs) - assert result == outputs - - -def test_arguments_string_parsing_with_empty_parsing_manager(): - """ - Test to parse arguments provided as string with a parsing parser and retrieve the following results : - - - "" : {} - - "-z" : UnknownArgException(z) - - "-a" : UnknownArgException(a) - - "-a --sub toto -b" : UnknownArgException(a) - - "-b" : UnknownArgException(b) - - ConfigParsingManager description: - - - base parser arguments : None - """ - parser_manager = RootConfigParsingManager() - - check_parse_cli_result(parser_manager, '', {}) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(parser_manager, '-z', None) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(parser_manager, '-a', None) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(parser_manager, '-a --sub toto -b', None) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(parser_manager, '-b', None) +from powerapi.cli.cli_parser import CLIParseException +from powerapi.cli.config_parser import ComponentSchema, ConfigurationSectionSchema +from powerapi.cli.parsing_manager import ConfigurationParsingManager +from powerapi.exception import ConfigurationError -def test_arguments_dict_validation_with_empty_parsing_manager(): +@pytest.fixture +def parsing_manager() -> ConfigurationParsingManager: """ - Test validation of arguments dictionary with a parsing manager and retrieve the following results: - - - "" : {} - - "-z value" : UnknownArgException(z) - - "-a 10 " : UnknownArgException(a) - - "-a 10 --sub toto -b" : UnknownArgException(a) - - "-b" : UnknownArgException(b) - - ConfigParsingManager description: - - - base parser arguments : None + Create a parsing manager with representative root and component configuration. + :return: Configured parsing manager. """ - parser_manager = RootConfigParsingManager() - dic_z = { - "z": "value" - } - - dic_a = { - "a": 10 - } - - dic_a_sub = { - "a": 10, - "sub": { - "type": "toto", - "b": True - } - } - - dic_b = { - "b": True - } + manager = ConfigurationParsingManager() + manager.add_argument_prefix('TEST_POWERAPI_') + manager.add_argument('verbose', is_flag=True, default_value=False) + manager.add_argument('interval', argument_type=int, default_value=10) + manager.add_group('input', prefix='TEST_POWERAPI_INPUT_') - with pytest.raises(UnknownArgException): - parser_manager.validate(dic_z) + socket = ComponentSchema('socket') + socket.add_argument('host', default_value='localhost') + socket.add_argument('port', argument_type=int, default_value=9080) + socket.add_argument('tags', argument_type=list) + manager.add_component('input', socket) - with pytest.raises(UnknownArgException): - parser_manager.validate(dic_a) + return manager - with pytest.raises(UnknownArgException): - parser_manager.validate(dic_a_sub) - with pytest.raises(UnknownArgException): - parser_manager.validate(dic_b) - - -def test_arguments_dict_validation_with_parsing_manager(root_config_parsing_manager): +def test_parse_merges_cli_assignments_and_validates_schema(parsing_manager): """ - Test validation of arguments dictionary with a parsing manager and retrieve the following results: - - - "" : {} - - "-z" : UnknownArgException(z) - - "-a" : {a: True} - - "-a --sub toto -b" : UnknownArgException(sub) - - "-b" : UnknownArgException(b) - - ConfigParsingManager description: - - - base parser arguments : -a + Test that CLI values are merged, cast, and completed with schema defaults. """ + result = parsing_manager.parse([ + 'powerapi', + '--verbose', + '-C', 'input.sensor.type=socket', + '-C', 'input.sensor.port=9090', + '-C', 'input.sensor.tags=host,pod', + ]) - dic_z = { - "z": True + assert result == { + 'verbose': True, + 'interval': 10, + 'input': { + 'sensor': { + 'type': 'socket', + 'host': 'localhost', + 'port': 9090, + 'tags': ['host', 'pod'], + }, + }, } - dic_a = { - "a": True - } - dic_a_sub = { - "a": True, - "sub1": { - "type": "toto", - "b": True - } - } - - dic_b = { - "b": True - } - - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_z) - - assert root_config_parsing_manager.validate(dic_a) == dic_a - - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_a_sub) - - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_b) - - -def test_arguments_string_parsing_with_subgroup_parser_in_subgroup_parsing_manager(root_config_parsing_manager): +def test_parse_validates_fixed_section_without_component_type(): """ - Test to parse arguments with a parsing manager containing a subgroup parser. It must retrieve the following - results : - - - "" : {} - - "-z" : UnknownArgException(z) - - "-a" : {a: True} - - "-a --sub toto -b" : NoNameSpecifiedForSubgroupException - - "-a --sub toto -b --name titi" : {a:True, sub: { titi: { 'type': 'toto', b: True}}} - - "-b" : BadContextException(b, [toto]) - - ConfigParsingManager description: - - - base parser arguments: -a - - subgroup parser toto bound to the argument sub with sub arguments: -b and --name + Test that CLI assignments configure a fixed group section without a type. """ + manager = ConfigurationParsingManager() + manager.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + manager.add_section('formula', 'smartwatts', smartwatts) - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) + result = manager.parse([ + 'powerapi', + '-C', 'formula.smartwatts.learn-error-window-size=10', + ]) - check_parse_cli_result(root_config_parsing_manager, "", {}) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(root_config_parsing_manager, "-z", {}) - - check_parse_cli_result(root_config_parsing_manager, '-a', {'a': True}) - - with pytest.raises(NoNameSpecifiedForSubgroupException): - check_parse_cli_result(root_config_parsing_manager, '-a --sub toto -b', {}) - - check_parse_cli_result(root_config_parsing_manager, '-a --sub toto -b --name titi', - {'a': True, 'sub': {'titi': {'type': 'toto', 'b': True}}}) - - with pytest.raises(BadContextException): - check_parse_cli_result(root_config_parsing_manager, "-b", {}) - - -def test_arguments_dict_validation_with_subgroup_parser_in_subgroup_parsing_manager(root_config_parsing_manager): - """ - Test to validate arguments with a parsing manager containing a subgroup parser. It must retrieve the following - results: - - - "" : {} - - "-z" : UnknownArgException(z) - - "-a" : {a: True} - - "-a --sub toto -b --name titi" : {a:True, sub: { titi: { 'type': 'toto', b: True}}} - - "-b" : UnknownArgException(b) - - ConfigParsingManager description: - - - base parser arguments : -a - - subparser toto bound to the argument sub with sub arguments : -b and --name - """ - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('type', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - dic_a = {'a': True} - - dic_z = { - "z": True - } - - dic_b = { - 'b': "type" - } - - dic_a_sub = { - 'a': True, - 'sub': { - 'titi': - { - 'type': 'toto', - 'b': "type" - } - } + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, } - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_z) - - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_b) - - assert root_config_parsing_manager.validate(dic_a) == dic_a - - assert root_config_parsing_manager.validate(dic_a_sub) == dic_a_sub - - assert root_config_parsing_manager.validate({}) == {} - - -def test_parsing_of_two_subgroups_of_the_same_type_with_subgroup_parsing_manager(root_config_parsing_manager): - """ - Test the parsing of two subgroups of the same type created with the following cli: - --sub toto --name titi --sub toto -b --name tutu - - The result must be: - {sub:{'titi' : {'type': 'toto'}, 'tutu': {'type': 'toto', 'b':True}}} - - """ - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - check_parse_cli_result(root_config_parsing_manager, '--sub toto --name titi --sub toto -b --name tutu', - {'sub': {'titi': {'type': 'toto'}, 'tutu': {'type': 'toto', 'b': True}}}) - - -def test_validation_of_two_subgroups_of_the_same_type_in_subgroup_parsing_manager(root_config_parsing_manager): - """ - Test the validation of two subgroups of the same type created with the following cli: - --sub toto --name titi --sub toto -b -n 'my_name' - - The result must be: - {sub:{'titi' : {'type': 'toto'}, 'tutu': {'type': 'toto', 'n':'my_name'}}} - - """ - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('type') - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - expected_dic = {'sub': {'titi': {'type': 'toto'}, 'tutu': {'type': 'toto', 'n': 'my_name'}}} - assert root_config_parsing_manager.validate(expected_dic) == expected_dic - - -def test_parsing_of_two_subgroups_of_different_type_in_subgroup_parsing_manager(root_config_parsing_manager): - """ - Test the validation of two subgroups of different type created with the following cli: - Create two component with different type with the following cli : - --sub toto --name titi --sub tutu --name tete - - The result must be: - {sub:{'titi' : {'type': 'toto'}, 'tete': {'type': 'tutu'}}} - - """ - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - subparser = SubgroupConfigParsingManager('tutu') - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - check_parse_cli_result(root_config_parsing_manager, '--sub toto --name titi --sub tutu --name tete', - {'sub': {'titi': {'type': 'toto'}, 'tete': {'type': 'tutu'}}}) - - -def test_parsing_of_repeated_subgroups_in_subgroup_parsing_manager_raise_an_exception(root_config_parsing_manager): - """ - Test the parsing of two subgroups with same type and name created with the following cli: - --sub toto --name titi --sub toto --name titi - - SubgroupAlreadyExistException must be raised - """ - - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - with pytest.raises(SubgroupAlreadyExistException): - check_parse_cli_result(root_config_parsing_manager, '--sub toto --name titi --sub toto --name titi', None) - - -def test_arguments_string_parsing_with_and_without_val_in_root_parsing_manager(root_config_parsing_manager): - """ - Test to parse arguments with and without value. The expected results are: - - - "-c" : MissingValue(c) - - "-d" : MissingValue(d) - - "-c 1" : {c : 1} - - "-d 10" : {d : 10} - - ConfigParsingManager description: - - - base parser arguments: -a (flag), -c (no flag), -d (int) - """ - - root_config_parsing_manager.add_argument('c') - - root_config_parsing_manager.add_argument('d', argument_type=int) - - with pytest.raises(MissingValueException): - check_parse_cli_result(root_config_parsing_manager, '-c', None) - - with pytest.raises(MissingValueException): - check_parse_cli_result(root_config_parsing_manager, '-d', None) - - check_parse_cli_result(root_config_parsing_manager, '-c 1', {'c': '1'}) - - check_parse_cli_result(root_config_parsing_manager, '-d 10', {'d': 10}) - - -def test_validation_of_arguments_dict_parsing_with_val_in_root_parsing_manager(root_config_parsing_manager): - """ - Test the validation of arguments with value. The expected results are: - - - "-c 1" : {c : 1} - - "-d 89" : {d : 89} - - ConfigParsingManager description: - - - base parser arguments: -a (flag), -c (not flag), -d (int) - """ - root_config_parsing_manager.add_argument('c') - - root_config_parsing_manager.add_argument('d', argument_type=int) - - dic_c = {'c': '1'} - dic_d = {'d': 89} - - assert root_config_parsing_manager.validate(dic_c) == dic_c - - assert root_config_parsing_manager.validate(dic_d) == dic_d - - -def test_arguments_string_parsing_type_checking_in_root_parsing_manager(root_config_parsing_manager): - """ - Test that the type of argument is correctly checked by the parsing manager when a string is used as input - """ - root_config_parsing_manager.add_argument('c', argument_type=int) - - with pytest.raises(BadTypeException): - check_parse_cli_result(root_config_parsing_manager, '-c string', {'c': 'string'}) - - check_parse_cli_result(root_config_parsing_manager, '-c 1', {'c': 1}) - - -def test_validation_of_arguments_dict_type_checking_in_root_parsing_manager(root_config_parsing_manager): - """ - Test that the argument type is correctly validated by the parser when a dict is used as input - """ - root_config_parsing_manager.add_argument('c', argument_type=int) - - str_dic = {'c': 'string'} - int_dic = {'c': 42} - - with pytest.raises(BadTypeException): - root_config_parsing_manager.validate(str_dic) - - assert root_config_parsing_manager.validate(int_dic) == int_dic - - -# multi name tests # -def test_arguments_string_parsing_with_long_and_short_names_in_root_parsing_manager(root_config_parsing_manager): - """ - Test that arguments parsing only relates parsing result to long name in arguments with long and short names - """ - root_config_parsing_manager.add_argument('c', 'coco') - root_config_parsing_manager.add_argument('d', 'xx', argument_type=int) - - check_parse_cli_result(root_config_parsing_manager, '-c 1', {'coco': '1'}) - - check_parse_cli_result(root_config_parsing_manager, '-d 555', {'xx': 555}) - - -def test_add_arguments_with_two_short_names_raise_an_exception_in_root_parsing_manager(root_config_parsing_manager): - """ - Test if adding arguments to a parser with two short names raise a SameLengthArgumentNamesException - The arguments are not added - """ - with pytest.raises(SameLengthArgumentNamesException): - root_config_parsing_manager.add_argument('c', 'd') - - with pytest.raises(SameLengthArgumentNamesException): - root_config_parsing_manager.add_argument('t', 's') - - assert len(root_config_parsing_manager.cli_parser.arguments) == 4 # --help, -h and sub - - assert root_config_parsing_manager.cli_parser.long_arg == ['help', 'sub='] - assert root_config_parsing_manager.cli_parser.short_arg == 'ha' - - -def test_add_arguments_with_two_long_names_raise_an_exception_in_root_parsing_manager(root_config_parsing_manager): - """ - Test if adding arguments to a parser with long names raise a SameLengthArgumentNamesException. - The arguments are not added - """ - with pytest.raises(SameLengthArgumentNamesException): - root_config_parsing_manager.add_argument('coco', 'dodo') - - with pytest.raises(SameLengthArgumentNamesException): - root_config_parsing_manager.add_argument('ddddd', 'plplp') - - assert len(root_config_parsing_manager.cli_parser.arguments) == 4 # -a, --help, -h and sub - - assert root_config_parsing_manager.cli_parser.long_arg == ['help', 'sub='] - assert root_config_parsing_manager.cli_parser.short_arg == 'ha' - - -# Type tests # -def test_add_argument_with_default_type_in_root_parsing_manager(): - """ - Test if adding arguments without type has string (default type) as type - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a') - parser_manager.add_argument('b') - result_a = parser_manager.parse('python -a 1'.split()) - result_b = parser_manager.parse('python3 -b string'.split()) - assert len(result_a) == 1 - assert 'a' in result_a - assert isinstance(result_a['a'], str) - - assert len(result_b) == 1 - assert 'b' in result_b - assert isinstance(result_b['b'], str) - - -def test_add_argument_with_type_in_root_parsing_manager(): - """ - Test if adding arguments with a type have currently this type - - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a', argument_type=int) - parser_manager.add_argument('b', argument_type=bool) - - result = parser_manager.parse('python -a 1'.split()) - assert len(result) == 1 - assert 'a' in result - assert isinstance(result['a'], int) - - result = parser_manager.parse('python3 -b false'.split()) - assert len(result) == 1 - assert 'b' in result - assert isinstance(result['b'], bool) - - -def test_parsing_of_arguments_string_with_wrong_type_raise_an_exception_in_root_parsing_manager(): - """ - Test that parsing arguments with a wrong value type raises a BadTypeException - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a', argument_type=int) - - with pytest.raises(BadTypeException): - parser_manager._parse_cli('-a a'.split()) - - -# parse with Subparser tests # -def test_add_subgroup_parser_that_already_exists_raises_an_exception_in_root_parsing_manager(): - """ - Test that adding a subgroup parser that already exists raises an - AlreadyAddedSubparserException - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_subgroup(name='toto') - subparser = SubgroupConfigParsingManager('titi') - subparser.add_argument('n', 'name') - parser_manager.add_subgroup_parser('toto', subparser) - - repeated_subparser = SubgroupConfigParsingManager('titi') - repeated_subparser.add_argument('n', 'name') - - with pytest.raises(AlreadyAddedSubparserException): - parser_manager.add_subgroup_parser('toto', repeated_subparser) - - -def test_parsing_of_arguments_string_with_subgroup_parser_with_long_and_short_arguments_names_in_root_parsing_manager(): - """ - Tests that parsing arguments of a subgroup parser with long and short names arguments - only binds parser results to the long name - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_subgroup(name='sub') - subparser = SubgroupConfigParsingManager('titi') - subparser.add_argument('a', 'aaa', is_flag=True, action=store_true, default_value=False) - subparser.add_argument('c', 'ttt', is_flag=False, action=store_val, argument_type=int) - subparser.add_argument('n', 'name') - parser_manager.add_subgroup_parser('sub', subparser) - check_parse_cli_result(parser_manager, '--sub titi -a --name tutu -c 15', - {'sub': {'tutu': {'aaa': True, 'type': 'titi', 'ttt': 15}}}) - - -def test_add_subgroup_parser_without_name_argument_raise_an_exception_in_root_parsing_manager(): - """ - Test that adding a subgroup parser with no argument 'name' raises a - SubgroupParserWithoutNameArgumentException - """ - parser = RootConfigParsingManager() - subparser = SubgroupConfigParsingManager('titi') - - with pytest.raises(SubgroupParserWithoutNameArgumentException): - parser.add_subgroup_parser('toto', subparser) - - -def test_parsing_empty_string_return_empty_configuration_in_root_parsing_manager(): - """ - Test that the result of parsing an empty string is a empty dict - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a', default_value=1) - parser_manager.add_argument('xxx', default_value='val') - result = parser_manager._parse_cli(''.split()) - assert len(result) == 0 - - -def test_validate_empty_dict_return_default_values_of_arguments_in_root_parsing_manager(): - """ - Test that the result of parsing an empty dict is a dict of arguments with their default value - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('c', argument_type=int, default_value=1) - parser_manager.add_argument('hello', argument_type=str, default_value="world") - - default_dic = {} - expected_dic = {'c': 1, 'hello': 'world'} - - assert parser_manager.validate(default_dic) == expected_dic - - -def test_parsing_configuration_file_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration is correctly parsed - """ - config_file = 'root_manager_basic_configuration.json' - expected_dict = load_configuration_from_json_file(config_file) - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=f'--config-file {test_files_path}/{config_file}'.split()) - assert result == expected_dict - - -def test_parsing_configuration_file_with_long_and_short_names_for_arguments_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration with long and short names for arguments is correctly parsed - """ - config_file = 'root_manager_basic_configuration_with_long_and_short_names.json' - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['argumento2'] = expected_dict.pop('2') - expected_dict['arg5'] = expected_dict.pop('5') - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=f'--config-file {test_files_path}/{config_file}'.split()) - assert result == expected_dict - - -def test_parsing_configuration_file_with_no_argument_with_default_value_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration with no values for arguments with default values - is correctly parsed - """ - config_file = 'root_manager_basic_configuration_with_no_argument_with_default_value.json' - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['arg5'] = 'default value' - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=f'--config-file {test_files_path}/{config_file}'.split()) - - assert result == expected_dict - - -def test_parsing_configuration_file_with_unknown_argument_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration with unknown arguments stops execution of the application - """ - config_file = 'root_manager_basic_configuration_with_unknown_argument.json' - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=f'--config-file {test_files_path}/{config_file}'.split()) - - assert result.type is SystemExit - assert result.value.code == -1 - - -def test_parsing_configuration_file_with_wrong_argument_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration with unknown arguments stops execution of the application - """ - config_file = 'root_manager_basic_configuration_with_argument_type_value.json' - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=('--config-file ' + test_files_path + '/' + config_file).split()) - - assert result.type is SystemExit - assert result.value.code == -1 - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration.json']) -def test_parsing_cli_configuration_in_root_parsing_manager( - config_file, - root_config_parsing_manager_with_mandatory_and_optional_arguments, - cli_configuration): - """ - Test that a list of strings containing a configuration is correctly parsed - """ - expected_dict = load_configuration_from_json_file(config_file) - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - assert result == expected_dict - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_long_and_short_names.json']) -def test_parsing_cli_configuration_with_long_and_short_names_for_arguments_in_root_parsing_manager( - config_file, cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that a list of strings containing a configuration with long and short names for arguments is correctly parsed - """ - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['argumento2'] = expected_dict.pop('2') - expected_dict['arg5'] = expected_dict.pop('5') - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - assert result == expected_dict - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_no_argument_with_default_value.json']) -def test_parsing_cli_configuration_with_no_argument_with_default_value_in_root_parsing_manager( - config_file, cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that a list of strings containing a configuration with no values for arguments with default values - is correctly parsed - """ - expected_dict = load_configuration_from_json_file(config_file) - - expected_dict['arg5'] = 'default value' - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_unknown_argument.json']) -def test_parsing_cli_configuration_with_unknown_argument_terminate_execution_in_root_parsing_manager( - config_file, cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path): - """ - Test that a list of strings containing a configuration with unknown arguments stops execution of the application - """ - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result.type is SystemExit - assert result.value.code == -1 - - -def test_parsing_environment_variables_configuration_in_root_parsing_manager( - empty_cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that a list of environment variables containing a configuration is correctly parsed - """ - config_file = 'root_manager_basic_configuration.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_unknown_argument_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that a list of environment variables containing a configuration is correctly parsed - """ - config_file = 'root_manager_basic_configuration_with_unknown_argument.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result.type is SystemExit - assert result.value.code == -1 - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_long_and_short_names_for_arguments_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a configuration defined via environment variables with long and short names for arguments is correctly - parsed - """ - config_file = 'root_manager_basic_configuration_with_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['argumento2'] = expected_dict.pop('2') - expected_dict['arg5'] = expected_dict.pop('5') - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_no_argument_with_default_value_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that the parsing of a configuration defined via environment variables missing arguments with - default values results in a dict with the default values for those arguments - """ - config_file = 'root_manager_basic_configuration_with_no_argument_with_default_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['arg5'] = 'default value' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_no_argument_with_default_value.json']) -def test_configuration_priority_between_cli_and_environment_variables_in_root_parsing_manager( - config_file, cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that arguments values defined via the CLI are preserved regarding values defined via environment variables - """ - config_file_environment_variables = 'root_manager_basic_configuration.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict["arg5"] = "this is a value" # This value is not defined by the CLI but it has to be present - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_no_argument_with_default_value.json']) -def test_configuration_priority_between_cli_and_configuration_file_in_root_parsing_manager(config_file, - cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, - monkeypatch): - """ - Test that arguments values defined via the CLI are preserved regarding values defined via a configuration file - """ - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_basic_configuration.json']) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict["arg5"] = "this is a value" # This value is not defined by the CLI but it has to be present - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - -def test_configuration_priority_between_environment_variables_and_configuration_file_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): - """ - Test that arguments values defined via environment variables are preserved regarding values defined via - a configuration file - """ - config_file_environment_variables = 'root_manager_basic_configuration_with_no_argument_with_default_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_basic_configuration.json']) - - expected_dict = load_configuration_from_json_file(config_file_environment_variables) - expected_dict["arg5"] = "this is a value" # This value is not defined by the CLI but it has to be present - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_no_argument_with_default_value.json']) -def test_configuration_priority_between_cli_environment_variables_and_configuration_file_in_root_parsing_manager( - config_file, cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): - """ - Test the following argument definition priority: - 1. CLI - 2. Environment Variables - 3. Configuration file - """ - - config_file_environment_variables = 'root_manager_basic_configuration_with_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_basic_configuration.json']) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict["arg5"] = "this is a value 3" - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_subgroups_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a configuration defined via environment variables with subgroups is correctly parsed - """ - config_file = 'root_manager_configuration_with_subgroups.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_subgroups_and_long_and_short_names_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): +@pytest.mark.parametrize('args', [ + ['powerapi', '--interval', '12'], + ['--interval', '12'], +]) +def test_parse_accepts_arguments_with_or_without_executable(parsing_manager, args): """ - Test that a configuration defined via environment variables with subgroups is correctly parsed + Test that parsing accepts argument lists with or without an executable name. """ - config_file = 'root_manager_configuration_with_subgroups_and_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file('root_manager_configuration_with_subgroups.json') - expected_dict['input']['in1']['name'] = 'i1_name' - expected_dict['output']['o1']['model'] = 'o1_model_x' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict + assert parsing_manager.parse(args) == {'verbose': False, 'interval': 12} - remove_environment_variables_configuration(variables_names=created_environment_variables) - -def test_parsing_environment_variables_with_subgroups_and_unknown_arguments_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): +def test_parse_uses_sys_argv_by_default(parsing_manager, monkeypatch): """ - Test that a configuration defined via environment variables with subgroups and unknown arguments terminates - the execution + Test that parsing uses the process arguments when no argument list is provided. """ - config_file = 'root_manager_configuration_with_subgroups_and_unknown_arguments.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result.type is SystemExit - assert result.value.code == -1 + monkeypatch.setattr(sys, 'argv', ['powerapi', '--interval', '12']) - remove_environment_variables_configuration(variables_names=created_environment_variables) + assert parsing_manager.parse() == {'verbose': False, 'interval': 12} -def test_parsing_environment_variables_with_subgroups_and_no_arguments_with_default_value_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): +def test_parse_applies_defaults_when_sources_are_empty(parsing_manager): """ - Test that a configuration defined via environment variables with subgroups without variables with default values - is correctly parsed + Test that parsing an empty configuration applies root defaults. """ - config_file = 'root_manager_configuration_with_subgroups_and_no_argument_default_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['input']['in1']['name'] = 'my_i1_instance' - expected_dict['output']['o1']['name'] = 'my_o1_instance' - expected_dict['output']['o2']['name'] = 'my_o2_instance' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict + assert parsing_manager.parse([]) == {'verbose': False, 'interval': 10} - remove_environment_variables_configuration(variables_names=created_environment_variables) - -def test_parsing_environment_variables_with_subgroups_and_wrong_type_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a configuration defined via environment variables with subgroups and wrong argument type terminates - the execution - """ - config_file = 'root_manager_configuration_with_subgroups_and_wrong_argument_type_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result.type is SystemExit - assert result.value.code == -1 - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', - ['root_manager_configuration_with_subgroups_and_no_argument_default_value.json']) -def test_config_priority_between_cli_environ_variables_and_configuration_file_with_subgroups_in_root_parsing_manager( - config_file, cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): +def test_parse_merges_sources_with_cli_then_environment_then_file_precedence(parsing_manager, monkeypatch, tmp_path): """ - Test the following argument definition priority for a configuration with subgroups: - 1. CLI - 2. Environment Variables - 3. Configuration file + Test CLI, environment, and file precedence for root and component values. """ + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(json.dumps({ + 'interval': 1, + 'input': { + 'sensor': { + 'type': 'socket', + 'host': 'file', + 'port': 1001, + 'tags': ['file'], + }, + }, + }), encoding='utf-8') + monkeypatch.setenv('TEST_POWERAPI_INTERVAL', '2') + monkeypatch.setenv('TEST_POWERAPI_INPUT_SENSOR_HOST', 'environment') + monkeypatch.setenv('TEST_POWERAPI_INPUT_SENSOR_PORT', '2002') - config_file_environment_variables = 'root_manager_configuration_with_subgroups_and_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_configuration_with_subgroups.json']) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['input']['in1']['name'] = 'i1_name' - expected_dict['output']['o1']['name'] = 'o1_name' - expected_dict['output']['o2']['name'] = 'o2_name' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict + result = parsing_manager.parse([ + 'powerapi', + '--config-file', str(config_file), + '-C', 'interval=3', + '-C', 'input.sensor.port=3003', + ]) - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', - ['root_manager_configuration_with_subgroups_and_no_argument_default_value.json']) -def test_config_priority_between_cli_and_environ_variables_with_subgroups_in_root_parsing_manager( - config_file, cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path): - """ - Test that arguments values defined via the CLI are preserved regarding values defined via environment variables - with subgroups in configuration - """ - - config_file_environment_variables = 'root_manager_configuration_with_subgroups_and_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(file_name=config_file) - expected_dict['input']['in1']['name'] = 'i1_name' - expected_dict['output']['o1']['name'] = 'o1_name' - expected_dict['output']['o2']['name'] = 'o2_name' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) + assert result == { + 'verbose': False, + 'interval': 3, + 'input': { + 'sensor': { + 'type': 'socket', + 'host': 'environment', + 'port': 3003, + 'tags': ['file'], + }, + }, + } -@pytest.mark.parametrize('config_file', - ['root_manager_configuration_with_subgroups_and_no_argument_default_value.json']) -def test_config_priority_between_cli_and_configuration_file_with_subgroups_in_root_parsing_manager( - config_file, cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): +def test_parse_resolves_environment_component_type_for_partial_file_configuration(parsing_manager, monkeypatch, tmp_path): """ - Test that arguments values defined via the CLI are preserved regarding values defined via a config file - with subgroups in configuration + Test that a partial file component inherits its type from the environment after merging. """ - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_configuration_with_subgroups.json']) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['input']['in1']['name'] = 'in1_name' - expected_dict['output']['o1']['name'] = 'o1_name' - expected_dict['output']['o2']['name'] = 'o2_name' + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(json.dumps({ + 'input': { + 'sensor': { + 'port': 9080, + }, + }, + }), encoding='utf-8') + monkeypatch.setenv('TEST_POWERAPI_INPUT_SENSOR_TYPE', 'socket') - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() + result = parsing_manager.parse([ + 'powerapi', + '--config-file', str(config_file), + ]) - assert result == expected_dict + assert result['input']['sensor'] == { + 'type': 'socket', + 'host': 'localhost', + 'port': 9080, + } -def test_config_priority_between_environ_variables_and_configuration_file_with_subgroups_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): +def test_parse_propagates_cli_errors(parsing_manager): """ - Test that arguments values defined via the environment variables are preserved regarding values defined via a config - file with subgroups in configuration + Test that command-line parsing errors propagate to the caller. """ + with pytest.raises(CLIParseException, match='unrecognized arguments'): + parsing_manager.parse(['powerapi', '--unknown']) - config_file_environment_variables = 'root_manager_configuration_with_subgroups_and_no_argument_default_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - monkeypatch.setattr(sys, 'argv', ['--config-file', test_files_path + '/root_manager_configuration_with_subgroups_and_long_and_short_names.json']) - - expected_dict = load_configuration_from_json_file(config_file_environment_variables) - expected_dict['input']['in1']['name'] = 'i1_name' - expected_dict['output']['o1']['name'] = 'o1_name' - expected_dict['output']['o2']['name'] = 'o2_name' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_add_subgroup_in_root_parsing_manager(): +def test_parse_propagates_configuration_errors(parsing_manager): """ - Test that a subgroup is correctly added to a parsing manager + Test that schema validation errors propagate with their configuration path. """ - parser_manager = RootConfigParsingManager() - - assert len(parser_manager.cli_parser.subgroup_parsers) == 0 - - parser_manager.add_subgroup(name='sub') - - assert len(parser_manager.cli_parser.subgroup_parsers) == 1 - - parser_manager.add_subgroup(name='sub1') - - assert len(parser_manager.cli_parser.subgroup_parsers) == 2 - - parser_manager.add_subgroup(name='sub3') + with pytest.raises(ConfigurationError) as result: + parsing_manager.parse(['powerapi', '-C', 'input.sensor.type=unknown']) - assert len(parser_manager.cli_parser.subgroup_parsers) == 3 + assert result.value.path == 'input.sensor.type' -def test_add_repeated_subgroup_terminate_execution_in_root_parsing_manager(root_config_parsing_manager): +def test_parse_propagates_missing_configuration_file(parsing_manager, tmp_path): """ - Test that adding a repeated terminates the execution + Test that selecting a missing configuration file raises FileNotFoundError. """ - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager.add_subgroup(name='sub') + missing_file = tmp_path / 'powerapi-pytest-missing.json' - assert result.type is SystemExit - assert result.value.code == -1 + with pytest.raises(FileNotFoundError): + parsing_manager.parse(['powerapi', '--config-file', str(missing_file)]) diff --git a/tests/unit/cli/test_utils.py b/tests/unit/cli/test_utils.py new file mode 100644 index 00000000..f0fc8bef --- /dev/null +++ b/tests/unit/cli/test_utils.py @@ -0,0 +1,98 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest + +from powerapi.cli._utils import merge_dictionaries, string_to_bool, string_to_list + + +@pytest.mark.parametrize('value', [' YES ', 'y', 'true', 't', '1']) +def test_string_to_bool_converts_true_values(value): + """ + Test that supported textual true values are normalized and converted. + """ + assert string_to_bool(value) is True + + +@pytest.mark.parametrize('value', [' No ', 'n', 'false', 'f', '0']) +def test_string_to_bool_converts_false_values(value): + """ + Test that supported textual false values are normalized and converted. + """ + assert string_to_bool(value) is False + + +def test_string_to_bool_rejects_unknown_value(): + """ + Test that an unsupported textual boolean value is rejected. + """ + with pytest.raises(ValueError, match='Invalid boolean value: invalid'): + string_to_bool('invalid') + + +@pytest.mark.parametrize(('value', 'expected'), [ + ('', []), + ('host', ['host']), + ('host, pod', ['host', 'pod']), +]) +def test_string_to_list_converts_comma_separated_values(value, expected): + """ + Test that comma-separated values are split and stripped. + """ + assert string_to_list(value) == expected + + +def test_merge_dictionaries_uses_last_configuration_as_highest_precedence(): + """ + Test that later configurations override earlier values while preserving nested values. + """ + config_file = {'stream': False, 'input': {'sensor': {'port': 8080, 'uri': 'file'}}} + environment = {'input': {'sensor': {'port': 9080}}} + cli = {'stream': True, 'input': {'sensor': {'uri': 'cli'}}} + + result = merge_dictionaries(config_file, environment, cli) + + assert result == { + 'stream': True, + 'input': {'sensor': {'port': 9080, 'uri': 'cli'}}, + } + + +def test_merge_dictionaries_does_not_modify_or_reuse_inputs(): + """ + Test that merged dictionaries do not share mutable values with their inputs. + """ + first = {'input': {'sensor': {'port': 8080, 'tags': ['host']}}} + second = {'input': {'sensor': {'uri': 'socket'}}} + + result = merge_dictionaries(first, second) + result['input']['sensor']['port'] = 9080 + result['input']['sensor']['tags'].append('socket') + + assert first == {'input': {'sensor': {'port': 8080, 'tags': ['host']}}} + assert second == {'input': {'sensor': {'uri': 'socket'}}} diff --git a/tests/utils/cli/base_config_parser.py b/tests/utils/cli/base_config_parser.py index f174bfed..2c4f8000 100644 --- a/tests/utils/cli/base_config_parser.py +++ b/tests/utils/cli/base_config_parser.py @@ -78,36 +78,21 @@ def generate_cli_configuration_from_json_file(file_name: str) -> list: def generate_cli_configuration_from_dictionary(configuration: dict, group_name: str = '') -> list: """ - Generate a list with arguments defined in dictionary. The list always has arguments --arg1_name, arg1_value, - -arg2_name, arg2_value... Each - argument name has as prefix '-' if it is short (its length == 1) or '--' if it is long (its length>1) + Generate repeatable dotted CLI configuration assignments from a dictionary. :param str configuration: The dictionary with the configuration :param str group_name: The name of the group that is currently being created """ conf_as_list = [] for argument_name, argument_value in configuration.items(): - prefix = '--' - if len(argument_name) == 1: - prefix = '-' - - if not isinstance(argument_value, dict): - conf_as_list.append(prefix + argument_name) - conf_as_list.append(str(argument_value)) - + argument_path = '.'.join(filter(None, (group_name, argument_name))) + if isinstance(argument_value, dict): + conf_as_list.extend(generate_cli_configuration_from_dictionary( + configuration=argument_value, + group_name=argument_path, + )) else: - - if 'type' in argument_value: - conf_as_list.append(group_name) - conf_as_list.append(argument_value['type']) - argument_value.pop('type') - conf_as_list.append('--name') - conf_as_list.append(argument_name) - else: - group_name = prefix + argument_name - - conf_as_list.extend(generate_cli_configuration_from_dictionary(configuration=argument_value, - group_name=group_name)) + conf_as_list.extend(('-C', f'{argument_path}={argument_value}')) return conf_as_list diff --git a/tests/utils/cli/csv_input_output_stream_mode_enabled_configuration.json b/tests/utils/cli/csv_input_output_stream_mode_enabled_configuration.json index 0c0558bc..74e87e66 100644 --- a/tests/utils/cli/csv_input_output_stream_mode_enabled_configuration.json +++ b/tests/utils/cli/csv_input_output_stream_mode_enabled_configuration.json @@ -5,16 +5,14 @@ "puller": { "model": "HWPCReport", "type": "csv", - "files": ["/tmp/rapl.csv", "/tmp/msr.csv"], - "name": "my_puller" + "files": ["/tmp/rapl.csv", "/tmp/msr.csv"] } }, "output": { "pusher_power": { "type": "csv", "model": "PowerReport", - "directory": "/tmp/formula_results", - "name": "my_pusher" + "directory": "/tmp/formula_results" } } -} \ No newline at end of file +} diff --git a/tests/utils/cli/k8s_pre_processor_complete_configuration.json b/tests/utils/cli/k8s_pre_processor_complete_configuration.json index 151bc026..f81cb817 100644 --- a/tests/utils/cli/k8s_pre_processor_complete_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_complete_configuration.json @@ -18,7 +18,7 @@ }, "pre-processor": { "my_processor": { - "type": "k8s", + "type": "kubernetes", "api-mode": "manual", "puller": "one_puller" } diff --git a/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json index 5fd5213a..06c7787c 100644 --- a/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json @@ -18,7 +18,7 @@ }, "pre-processor": { "my_processor": { - "type": "k8s", + "type": "kubernetes", "api-mode": "manual", "puller": "non_existent_puller" } diff --git a/tests/utils/cli/several_inputs_outputs_stream_mode_enabled_configuration.json b/tests/utils/cli/several_inputs_outputs_stream_mode_enabled_configuration.json index 6010e4b8..11a8ec00 100644 --- a/tests/utils/cli/several_inputs_outputs_stream_mode_enabled_configuration.json +++ b/tests/utils/cli/several_inputs_outputs_stream_mode_enabled_configuration.json @@ -5,15 +5,13 @@ "puller2": { "model": "HWPCReport", "type": "csv", - "files": ["/tmp/rapl.csv", "/tmp/msr.csv"], - "name": "puller2" + "files": ["/tmp/rapl.csv", "/tmp/msr.csv"] }, "puller3": { "model": "HWPCReport", "type": "socket", "host": "localhost", - "port": 1111, - "name": "puller3" + "port": 1111 }, "puller4": { "model": "HWPCReport", From 396afb1d681aaaf33549701e05f85cb4b0d9c275 Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Fri, 4 Sep 2026 15:06:04 +0200 Subject: [PATCH 4/5] refactor(cli): Simplify actor generators Make `Generator` and `DBActorGenerator` generic over their actor and database factory types. Centralize actor, database, report and processor factory resolution. Remove unused registry removal methods, duplicate constants and custom "alread-used" exceptions. Expand behavior-level generator coverage. --- src/powerapi/cli/generator.py | 416 +++++++++++--------- src/powerapi/exception.py | 238 ----------- tests/unit/cli/conftest.py | 12 - tests/unit/cli/test_generator.py | 210 ++++++++-- tests/unit/cli/test_generator_clickhouse.py | 14 - tests/unit/cli/test_generator_influxdb2.py | 14 - tests/unit/cli/test_generator_k8s.py | 18 +- tests/unit/cli/test_generator_mongodb.py | 28 -- tests/unit/cli/test_generator_openstack.py | 6 +- tests/unit/cli/test_generator_prometheus.py | 14 - 10 files changed, 411 insertions(+), 559 deletions(-) diff --git a/src/powerapi/cli/generator.py b/src/powerapi/cli/generator.py index 2c938be8..cf7a9781 100644 --- a/src/powerapi/cli/generator.py +++ b/src/powerapi/cli/generator.py @@ -32,185 +32,176 @@ from powerapi.actor import Actor, ActorProxy from powerapi.database.driver import ReadableDatabaseFactory, WritableDatabaseFactory -from powerapi.exception import PowerAPIException, ModelNameAlreadyUsed, DatabaseNameDoesNotExist, ModelNameDoesNotExist, \ - DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed +from powerapi.exception import ConfigurationError, PowerAPIException from powerapi.filter import ReportFilter from powerapi.processor.processor_actor import ProcessorActor from powerapi.puller import PullerActor from powerapi.pusher import PusherActor -from powerapi.report import HWPCReport, PowerReport, Report, FormulaReport +from powerapi.report import FormulaReport, HWPCReport, PowerReport, Report from powerapi.utils.metadata import build_metadata_mapping COMPONENT_TYPE_KEY = 'type' COMPONENT_MODEL_KEY = 'model' -COMPONENT_DB_NAME_KEY = 'db' -COMPONENT_DB_COLLECTION_KEY = 'collection' -COMPONENT_DB_MANAGER_KEY = 'db_manager' -COMPONENT_DB_MAX_BUFFER_SIZE_KEY = 'max_buffer_size' -COMPONENT_URI_KEY = 'uri' ACTOR_NAME_KEY = 'actor_name' -REGEXP_KEY = 'regexp' - -K8S_API_MODE_KEY = 'api-mode' -K8S_API_KEY_KEY = 'api-key' -K8S_API_HOST_KEY = 'api-host' - -LISTENER_ACTOR_KEY = 'listener_actor' GENERAL_CONF_STREAM_MODE_KEY = 'stream' GENERAL_CONF_VERBOSE_KEY = 'verbose' +_NON_STREAMING_INPUT_TYPES = frozenset(('csv', 'json')) -class Generator: - """ - Generate an actor class and actor start message from config dict. - The config dict has the following structure: - { - "arg1_key": value, - "arg2_key": value - ... - "component_group_name1":{ - "arg1_cpn1_key": value, - "arg2_cpn1_key": value, - ... - } - component_group_name2:{ - ... - } - ... - } +class Generator[ActorT: Actor]: + """ + Generate actors for one configured component group. """ - def __init__(self, component_group_name): + def __init__(self, component_group_name: str): + """ + Initialize a generator for a component group. + :param component_group_name: Name of the component group to generate. + """ self.component_group_name = component_group_name - def generate(self, main_config: dict) -> dict[str, Actor]: + def generate(self, main_config: dict) -> dict[str, ActorT]: """ - Generate an actor class and actor start message from config dict + Generate every actor configured in the component group. + :param main_config: Canonical PowerAPI configuration. + :return: Generated actors indexed by component name. + :raises PowerAPIException: If the component group is missing or a component configuration is invalid. """ if self.component_group_name not in main_config: - raise PowerAPIException(f'Configuration error : Component {self.component_group_name} group is unknown') + raise PowerAPIException(f'Configuration error: Component "{self.component_group_name}" is not defined') actors = {} for component_name, component_config in main_config[self.component_group_name].items(): - try: - actors[component_name] = self._gen_actor(component_config, main_config, component_name) - except KeyError as exn: - raise PowerAPIException(f'Configuration error: Missing "{exn.args[0]}" argument for {component_name} component') from exn - except ValueError as exn: - raise PowerAPIException(f'Configuration error: Invalid parameter for {component_name} component: {exn.args[0]}') from exn + actors[component_name] = self._gen_actor(component_config, main_config, component_name) return actors - def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> Actor: + def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> ActorT: + """ + Generate one actor from its component configuration. + :param component_config: Canonical component configuration. + :param main_config: Canonical PowerAPI configuration. + :param component_name: Name of the component to generate. + :return: Generated actor. + """ raise NotImplementedError() -class BaseGenerator(Generator): +class DBActorGenerator[ActorT: Actor, DBFactoryT: ReadableDatabaseFactory | WritableDatabaseFactory](Generator[ActorT]): """ - Generate an Actor and Start message from config + Resolve database factories before generating database-backed actors. """ def __init__(self, component_group_name: str): - Generator.__init__(self, component_group_name) + """ + Initialize a database-backed actor generator. + :param component_group_name: Name of the component group to generate. + """ + super().__init__(component_group_name) self.report_classes: dict[str, type[Report]] = { 'HWPCReport': HWPCReport, 'PowerReport': PowerReport, 'FormulaReport': FormulaReport, } + self.database_factories: dict[str, Callable[[dict], DBFactoryT]] = {} - def _gen_actor(self, component_config: dict, main_config: dict, component_name: str): - model = self._get_report_class(component_config[COMPONENT_MODEL_KEY], component_config) - component_config[COMPONENT_MODEL_KEY] = model - - actor = self._actor_factory(component_name, main_config, component_config) - return actor - - def _get_report_class(self, model_name: str, component_config: dict): - if model_name not in self.report_classes: - raise PowerAPIException(f'Configuration error: model type {model_name} unknown') - - return self.report_classes[component_config[COMPONENT_MODEL_KEY]] - - def _actor_factory(self, actor_name: str, main_config: dict, component_config: dict): - raise NotImplementedError - - -class DBActorGenerator(BaseGenerator): - """ - ActorGenerator that initialise the start message with a database from config - """ - - def __init__(self, component_group_name: str): - super().__init__(component_group_name) - - self.db_factory: dict[str, Callable[[dict], ReadableDatabaseFactory | WritableDatabaseFactory]] = {} - - def remove_report_class(self, model_name: str): - """ - remove a Model from generator + def _get_report_class(self, model_name: str) -> type[Report]: """ - if model_name not in self.report_classes: - raise ModelNameDoesNotExist(model_name) - - del self.report_classes[model_name] - - def remove_db_factory(self, database_name: str): + Resolve a configured report model name. + :param model_name: Registered report model name. + :return: Report class registered for the configured model. + :raises PowerAPIException: If the report model is unknown. """ - remove a database from generator - """ - if database_name not in self.db_factory: - raise DatabaseNameDoesNotExist(database_name) - - del self.db_factory[database_name] + try: + return self.report_classes[model_name] + except KeyError as error: + raise PowerAPIException(f'Configuration error: Unknown report model "{model_name}"') from error def add_report_class(self, model_name: str, report_class: type[Report]): """ - add a report class to generator + Register a report class. + :param model_name: Name identifying the report model. + :param report_class: Report class associated with the model name. + :raises ValueError: If the report model is already registered. """ if model_name in self.report_classes: - raise ModelNameAlreadyUsed(model_name) + raise ValueError(f'Report model "{model_name}" is already registered') self.report_classes[model_name] = report_class - def add_db_factory(self, db_name: str, db_factory_function: Callable[[dict], ReadableDatabaseFactory | WritableDatabaseFactory]): + def add_db_factory(self, db_name: str, db_factory_function: Callable[[dict], DBFactoryT]): """ - add a database to generator + Register a database factory. + :param db_name: Database type handled by the factory. + :param db_factory_function: Function creating a database factory from component configuration. + :raises ValueError: If the database type is already registered. """ - if db_name in self.db_factory: - raise DatabaseNameAlreadyUsed(db_name) + if db_name in self.database_factories: + raise ValueError(f'Database type "{db_name}" is already registered') - self.db_factory[db_name] = db_factory_function + self.database_factories[db_name] = db_factory_function + + def _create_database_factory(self, db_name: str, component_config: dict) -> DBFactoryT: + """ + Create a database factory for a component. + :param db_name: Registered database type. + :param component_config: Canonical component configuration. + :return: Configured readable or writable database factory. + :raises PowerAPIException: If the database type is unknown or its optional dependencies are unavailable. + """ + try: + factory = self.database_factories[db_name] + except KeyError as error: + raise PowerAPIException(f'Configuration error: Invalid database type: {db_name}') from error - def _generate_db(self, db_name: str, component_config: dict): try: - return self.db_factory[db_name](component_config) - except KeyError as exn: - raise PowerAPIException('Configuration error: Invalid database type: %s', db_name) from exn - except ImportError as exn: - raise PowerAPIException('Dependencies for %s database are not installed', db_name) from exn + return factory(component_config) + except ImportError as error: + raise PowerAPIException(f'Dependencies for {db_name} database are not installed') from error + + def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> ActorT: + """ + Resolve the report model and database factory before generating an actor. + :param component_config: Canonical component configuration. + :param main_config: Canonical PowerAPI configuration. + :param component_name: Name of the component to generate. + :return: Generated database-backed actor. + :raises PowerAPIException: If the report model or database type is unknown or a dependency is unavailable. + """ + factory_config = dict(component_config) + factory_config[COMPONENT_MODEL_KEY] = self._get_report_class(component_config[COMPONENT_MODEL_KEY]) + database_factory = self._create_database_factory( + component_config[COMPONENT_TYPE_KEY], + factory_config, + ) - def _gen_actor(self, component_config: dict, main_config: dict, component_name: str): - model = self._get_report_class(component_config[COMPONENT_MODEL_KEY], component_config) - component_config[COMPONENT_MODEL_KEY] = model - database_manager = self._generate_db(component_config[COMPONENT_TYPE_KEY], component_config) - component_config[COMPONENT_DB_MANAGER_KEY] = database_manager + return self._actor_factory(component_name, main_config, database_factory) - actor = self._actor_factory(component_name, main_config, component_config) - return actor + def _actor_factory(self, actor_name: str, main_config: dict, database_factory: DBFactoryT) -> ActorT: + """ + Create a database-backed actor from a resolved component configuration. + :param actor_name: Name assigned to the actor. + :param main_config: Canonical PowerAPI configuration. + :param database_factory: Configured database factory. + :return: Generated actor. + """ + raise NotImplementedError -class PullerGenerator(DBActorGenerator): +class PullerGenerator(DBActorGenerator[PullerActor, ReadableDatabaseFactory]): """ - Generate Puller Actor class and Puller start message from config + Generate puller actors from input component configurations. """ @staticmethod def _csv_input_database_factory(conf: dict) -> ReadableDatabaseFactory: """ - CSV Input database factory method. + Create a CSV input database factory. + :param conf: Canonical CSV input configuration. + :return: Configured CSV input factory. """ from powerapi.database.csv.driver import CSVInputFactory return CSVInputFactory(conf['model'], conf['files']) @@ -218,7 +209,9 @@ def _csv_input_database_factory(conf: dict) -> ReadableDatabaseFactory: @staticmethod def _json_input_database_factory(conf: dict) -> ReadableDatabaseFactory: """ - JSON Input database factory method. + Create a JSON input database factory. + :param conf: Canonical JSON input configuration. + :return: Configured JSON input factory. """ from powerapi.database.json.driver import JsonInputFactory return JsonInputFactory(conf['model'], conf['filepath'], conf['compression']) @@ -226,7 +219,9 @@ def _json_input_database_factory(conf: dict) -> ReadableDatabaseFactory: @staticmethod def _socket_database_factory(conf: dict) -> ReadableDatabaseFactory: """ - Socket Input database factory method. + Create a socket input database factory. + :param conf: Canonical socket input configuration. + :return: Configured socket input factory. """ from powerapi.database.socket.driver import SocketInputFactory return SocketInputFactory(conf['model'], conf['host'], conf['port']) @@ -234,14 +229,17 @@ def _socket_database_factory(conf: dict) -> ReadableDatabaseFactory: @staticmethod def _mongodb_database_factory(conf: dict) -> ReadableDatabaseFactory: """ - MongoDB Input database factory method. + Create a MongoDB input database factory. + :param conf: Canonical MongoDB input configuration. + :return: Configured MongoDB input factory. """ from powerapi.database.mongodb.driver import MongodbInputFactory return MongodbInputFactory(conf['model'], conf['uri'], conf['db'], conf['collection']) def __init__(self, report_filter: ReportFilter): """ - :param report_filter: Report filter to apply for incoming reports + Initialize a puller generator with the built-in input types. + :param report_filter: Report filter applied to incoming reports. """ super().__init__('input') @@ -252,29 +250,45 @@ def __init__(self, report_filter: ReportFilter): self.add_db_factory('socket', self._socket_database_factory) self.add_db_factory('mongodb', self._mongodb_database_factory) - def _actor_factory(self, actor_name: str, main_config, component_config: dict) -> PullerActor: + def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> PullerActor: + """ + Generate a puller after checking that its input supports the configured execution mode. + :param component_config: Canonical input component configuration. + :param main_config: Canonical PowerAPI configuration. + :param component_name: Name of the input component. + :return: Configured puller actor. + :raises ConfigurationError: If stream mode is enabled for a non-streaming input type. + """ + input_type = component_config[COMPONENT_TYPE_KEY] + if main_config[GENERAL_CONF_STREAM_MODE_KEY] and input_type in _NON_STREAMING_INPUT_TYPES: + raise ConfigurationError(f'Stream mode cannot be used with a {input_type} input', GENERAL_CONF_STREAM_MODE_KEY) + + return super()._gen_actor(component_config, main_config, component_name) + + def _actor_factory(self, actor_name: str, main_config: dict, database_factory: ReadableDatabaseFactory) -> PullerActor: """ - Actor factory method. - :param actor_name: Name of the actor - :param main_config: Global configuration - :param component_config: Actor configuration - :return: Configured Puller actor + Create a puller actor. + :param actor_name: Name assigned to the actor. + :param main_config: Canonical PowerAPI configuration. + :param database_factory: Configured readable database factory. + :return: Configured puller actor. """ - database = component_config[COMPONENT_DB_MANAGER_KEY] stream_mode = main_config[GENERAL_CONF_STREAM_MODE_KEY] logging_level = logging.DEBUG if main_config[GENERAL_CONF_VERBOSE_KEY] else logging.WARNING - return PullerActor(actor_name, database, self.report_filter, stream_mode, level_logger=logging_level) + return PullerActor(actor_name, database_factory, self.report_filter, stream_mode, level_logger=logging_level) -class PusherGenerator(DBActorGenerator): +class PusherGenerator(DBActorGenerator[PusherActor, WritableDatabaseFactory]): """ - Generate Pusher actor and Pusher start message from config + Generate pusher actors from output component configurations. """ @staticmethod def _csv_output_database_factory(conf: dict) -> WritableDatabaseFactory: """ - CSV Output database factory method. + Create a CSV output database factory. + :param conf: Canonical CSV output configuration. + :return: Configured CSV output factory. """ from powerapi.database.csv.driver import CSVOutputFactory return CSVOutputFactory(conf['model'], conf['directory']) @@ -282,7 +296,9 @@ def _csv_output_database_factory(conf: dict) -> WritableDatabaseFactory: @staticmethod def _json_output_database_factory(conf: dict) -> WritableDatabaseFactory: """ - JSON Output database factory method. + Create a JSON output database factory. + :param conf: Canonical JSON output configuration. + :return: Configured JSON output factory. """ from powerapi.database.json.driver import JsonOutputFactory return JsonOutputFactory(conf['model'], conf['filepath'], conf['compression']) @@ -290,7 +306,9 @@ def _json_output_database_factory(conf: dict) -> WritableDatabaseFactory: @staticmethod def _mongodb_database_factory(conf: dict) -> WritableDatabaseFactory: """ - MongoDB Output database factory method. + Create a MongoDB output database factory. + :param conf: Canonical MongoDB output configuration. + :return: Configured MongoDB output factory. """ from powerapi.database.mongodb.driver import MongodbOutputFactory return MongodbOutputFactory(conf['model'], conf['uri'], conf['db'], conf['collection']) @@ -298,7 +316,9 @@ def _mongodb_database_factory(conf: dict) -> WritableDatabaseFactory: @staticmethod def _influxdb2_database_factory(conf: dict) -> WritableDatabaseFactory: """ - InfluxDB2 database factory method. + Create an InfluxDB 2 output database factory. + :param conf: Canonical InfluxDB 2 output configuration. + :return: Configured InfluxDB 2 output factory. """ from powerapi.database.influxdb2.driver import InfluxDB2OutputFactory return InfluxDB2OutputFactory(conf['model'], conf['uri'], conf['org'], conf['bucket'], conf['token']) @@ -306,7 +326,9 @@ def _influxdb2_database_factory(conf: dict) -> WritableDatabaseFactory: @staticmethod def _prometheus_database_factory(conf: dict) -> WritableDatabaseFactory: """ - Prometheus database factory method. + Create a Prometheus output database factory. + :param conf: Canonical Prometheus output configuration. + :return: Configured Prometheus output factory. """ from powerapi.database.prometheus.driver import PrometheusOutputFactory return PrometheusOutputFactory(conf['model'], conf['addr'], conf['port'], conf.get('tags', [])) @@ -314,12 +336,17 @@ def _prometheus_database_factory(conf: dict) -> WritableDatabaseFactory: @staticmethod def _clickhouse_database_factory(conf: dict) -> WritableDatabaseFactory: """ - ClickHouse output database factory method. + Create a ClickHouse output database factory. + :param conf: Canonical ClickHouse output configuration. + :return: Configured ClickHouse output factory. """ from powerapi.database.clickhouse.driver import ClickHouseOutputFactory return ClickHouseOutputFactory(conf['model'], conf['host'], conf['port'], conf['username'], conf['password'], conf['database']) def __init__(self): + """ + Initialize a pusher generator with the built-in output types. + """ super().__init__('output') self.add_db_factory('csv', self._csv_output_database_factory) @@ -329,24 +356,24 @@ def __init__(self): self.add_db_factory('prometheus', self._prometheus_database_factory) self.add_db_factory('clickhouse', self._clickhouse_database_factory) - def _actor_factory(self, actor_name: str, main_config: dict, component_config: dict) -> PusherActor: + def _actor_factory(self, actor_name: str, main_config: dict, database_factory: WritableDatabaseFactory) -> PusherActor: """ - Actor factory method. - :param actor_name: Name of the actor - :param main_config: Global configuration - :param component_config: Actor configuration - :return: Configured Pusher actor + Create a pusher actor. + :param actor_name: Name assigned to the actor. + :param main_config: Canonical PowerAPI configuration. + :param database_factory: Configured writable database factory. + :return: Configured pusher actor. """ - database = component_config[COMPONENT_DB_MANAGER_KEY] level_logger = logging.DEBUG if main_config[GENERAL_CONF_VERBOSE_KEY] else logging.WARNING - return PusherActor(actor_name, database, logger_level=level_logger) + return PusherActor(actor_name, database_factory, logger_level=level_logger) - def generate_report_mapping(self, main_config: dict, actors: dict[str, Actor]) -> dict[type[Report], list[ActorProxy]]: + def generate_report_mapping(self, main_config: dict, actors: dict[str, PusherActor]) -> dict[type[Report], list[ActorProxy]]: """ - Generate the report type to pusher actor mapping. - :param main_config: Main configuration - :param actors: Dictionary of actors (result of the `generate` method) - :return: Dictionary mapping the report type to actors that should process it + Map report types to generated pusher proxies. + :param main_config: Canonical PowerAPI configuration. + :param actors: Generated pusher actors indexed by component name. + :return: Pusher proxies indexed by report type. + :raises PowerAPIException: If the output group or a configured actor is missing. """ if self.component_group_name not in main_config: raise PowerAPIException(f'Configuration error: Component "{self.component_group_name}" is not defined') @@ -355,67 +382,73 @@ def generate_report_mapping(self, main_config: dict, actors: dict[str, Actor]) - for component_name, component_config in main_config[self.component_group_name].items(): try: actor_proxy = actors[component_name].get_proxy() - report_type_to_actor.setdefault(component_config[COMPONENT_MODEL_KEY], []).append(actor_proxy) - except KeyError as exn: - raise PowerAPIException(f'Actor "{component_name}" is not defined') from exn + except KeyError as error: + raise PowerAPIException(f'Actor "{component_name}" is not defined') from error + + report_type = self._get_report_class(component_config[COMPONENT_MODEL_KEY]) + report_type_to_actor.setdefault(report_type, []).append(actor_proxy) return report_type_to_actor -class ProcessorGenerator(Generator): +class ProcessorGenerator(Generator[ProcessorActor]): """ Generator that initializes the processor actor(s) from the configuration. """ def __init__(self, component_group_name: str): """ - :param component_group_name: Name of the component group + Initialize a processor generator. + :param component_group_name: Name of the component group to generate. """ super().__init__(component_group_name) - self.processor_factory: dict[str, Callable[[dict], ProcessorActor]] = {} + self.processor_factories: dict[str, Callable[[dict], ProcessorActor]] = {} - def remove_processor_factory(self, processor_type: str) -> None: + def add_processor_factory(self, processor_type: str, processor_factory_function: Callable[[dict], ProcessorActor]) -> None: """ - Remove the given processor actor factory from the generator. - :param processor_type: Processor type name + Register a processor actor factory. + :param processor_type: Processor type handled by the factory. + :param processor_factory_function: Function creating a processor actor from component configuration. + :raises ValueError: If the processor type is already registered. """ - if processor_type not in self.processor_factory: - raise ProcessorTypeDoesNotExist(processor_type) + if processor_type in self.processor_factories: + raise ValueError(f'Processor type "{processor_type}" is already registered') - del self.processor_factory[processor_type] + self.processor_factories[processor_type] = processor_factory_function - def add_processor_factory(self, processor_type: str, processor_factory_function: Callable) -> None: + def _create_processor(self, processor_name: str, component_config: dict) -> ProcessorActor: """ - Add the given processor actor factory to the generator. - :param processor_type: Processor type name - :param processor_factory_function: Factory method used to generate the processor actors + Create a processor actor for a component. + :param processor_name: Registered processor type. + :param component_config: Resolved processor component configuration. + :return: Configured processor actor. + :raises PowerAPIException: If the processor type is unknown or its optional dependencies are unavailable. """ - if processor_type in self.processor_factory: - raise ProcessorTypeAlreadyUsed(processor_type) - - self.processor_factory[processor_type] = processor_factory_function + try: + factory = self.processor_factories[processor_name] + except KeyError as error: + raise PowerAPIException(f'Configuration error: Invalid processor type: {processor_name}') from error - def _generate_processor(self, processor_name: str, component_config: dict) -> ProcessorActor: try: - return self.processor_factory[processor_name](component_config) - except KeyError as exn: - raise PowerAPIException('Configuration error: Invalid processor type: %s', processor_name) from exn - except ImportError as exn: - raise PowerAPIException('Dependencies for %s processor are not installed', processor_name) from exn + return factory(component_config) + except ImportError as error: + raise PowerAPIException(f'Dependencies for {processor_name} processor are not installed') from error def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> ProcessorActor: """ - Helper method to generate a processor actor from the given configuration. - :param component_config: Configuration of the processor component - :param main_config: Global configuration - :param component_name: Name of the processor actor to generate - :return: Processor actor + Add shared processor settings and generate one processor actor. + :param component_config: Canonical component configuration. + :param main_config: Canonical PowerAPI configuration. + :param component_name: Name of the processor actor to generate. + :return: Configured processor actor. + :raises PowerAPIException: If the processor type is unknown or its optional dependencies are unavailable. """ + runtime_config = dict(component_config) processor_actor_type = component_config[COMPONENT_TYPE_KEY] - component_config[ACTOR_NAME_KEY] = component_name - component_config[GENERAL_CONF_VERBOSE_KEY] = main_config[GENERAL_CONF_VERBOSE_KEY] - return self._generate_processor(processor_actor_type, component_config) + runtime_config[ACTOR_NAME_KEY] = component_name + runtime_config[GENERAL_CONF_VERBOSE_KEY] = main_config[GENERAL_CONF_VERBOSE_KEY] + return self._create_processor(processor_actor_type, runtime_config) class PreProcessorGenerator(ProcessorGenerator): @@ -424,6 +457,9 @@ class PreProcessorGenerator(ProcessorGenerator): """ def __init__(self): + """ + Initialize a pre-processor generator with the built-in processor types. + """ super().__init__('pre-processor') self.add_processor_factory('kubernetes', self._k8s_pre_processor_factory) @@ -432,16 +468,16 @@ def __init__(self): @staticmethod def _k8s_pre_processor_factory(processor_config: dict) -> ProcessorActor: """ - Kubernetes pre-processor actor factory. - :param processor_config: Pre-Processor configuration - :return: Configured Kubernetes pre-processor actor + Create a Kubernetes pre-processor actor. + :param processor_config: Resolved Kubernetes pre-processor configuration. + :return: Configured Kubernetes pre-processor actor. """ from powerapi.processor.pre.k8s.actor import KubernetesPreProcessorActor from powerapi.processor.pre.k8s.monitor_agent import KubernetesMonitorConfig - api_mode = processor_config[K8S_API_MODE_KEY] - api_host = processor_config.get(K8S_API_HOST_KEY, None) - api_key = processor_config.get(K8S_API_KEY_KEY, None) + api_mode = processor_config['api-mode'] + api_host = processor_config.get('api-host') + api_key = processor_config.get('api-key') label_mapping = build_metadata_mapping(processor_config.get('labels', []), prefix='k8s_pod_label_') monitor_config = KubernetesMonitorConfig(api_mode, api_host, api_key, label_mapping) @@ -452,12 +488,14 @@ def _k8s_pre_processor_factory(processor_config: dict) -> ProcessorActor: @staticmethod def _openstack_pre_processor_factory(processor_config: dict) -> ProcessorActor: """ - OpenStack pre-processor actor factory. - :param processor_config: Pre-Processor configuration - :return: Configured OpenStack pre-processor actor + Create an OpenStack pre-processor actor. + :param processor_config: Resolved OpenStack pre-processor configuration. + :return: Configured OpenStack pre-processor actor. """ from powerapi.processor.pre.openstack.actor import OpenStackPreProcessorActor - from powerapi.processor.pre.openstack.monitor_agent import OpenStackMonitorConfig + from powerapi.processor.pre.openstack.monitor_agent import ( + OpenStackMonitorConfig, + ) api_polling_interval = processor_config['polling-interval'] metadata_mapping = build_metadata_mapping(processor_config.get('metadata', []), prefix='openstack_metadata_') diff --git a/src/powerapi/exception.py b/src/powerapi/exception.py index 51a69895..cac1e627 100644 --- a/src/powerapi/exception.py +++ b/src/powerapi/exception.py @@ -81,244 +81,6 @@ def __init__(self, argument_name: str): self.argument_name = argument_name -class NoNameSpecifiedForSubgroupException(ParserException): - """ - Exception raised when attempting to parse substring thant describe a component which not contains the component name - """ - - -class SubgroupAlreadyExistException(ParserException): - """ - Exception raised when attempting to parse a substring to create a component with a name that already exist - """ - - -class SubgroupDoesNotExistException(ParserException): - """ - Exception raised when attempting to add arguments to a subgroup that does not exist - """ - - -class SubgroupParserWithoutNameArgumentException(PowerAPIException): - """ - Exception raised when a subparser without argument name is added to a parser - """ - - -class TooManyArgumentNamesException(ParserException): - """ - Exception raised when attemtping to add an argument with too much names - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - - -class AlreadyAddedArgumentException(ParserException): - """ - Exception raised when attempting to add an argument to a parser that already - have this argument - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Parser already contain an argument ' + argument_name - - -class AlreadyAddedSubparserException(ParserException): - """ - Exception raised when attempting to add a parser that already exists """ - - def __init__(self, parser_name: str): - ParserException.__init__(self, parser_name) - self.msg = 'Parser already contain SubParser with name ' + parser_name - - -class AlreadyAddedSubgroupException(ParserException): - """ - Exception raised when attempting to add a subgroup that already exists """ - - def __init__(self, subgroup_name: str): - ParserException.__init__(self, subgroup_name) - self.msg = 'Parser already contain Subgroup with name ' + subgroup_name - - -class MissingArgumentException(ParserException): - """ - Exception raised when a mandatory argument is missing - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Argument with name(s) ' + argument_name + ' is missing' - - -class RepeatedArgumentException(ParserException): - """ - Exception raised when an argument is repeated several times in a configuration - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Argument with name(s) ' + argument_name + ' is repeated' - - -class MissingValueException(ParserException): - """ - Exception raised when an argument that require a value is caught without - its value - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Argument ' + argument_name + ' require a value' - - -class UnknownArgException(ParserException): - """ - Exception raised when the parser catch an argument that it can't handle - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Unknown argument ' + argument_name - - -class BadTypeException(ParserException): - """ - Exception raised when an argument is parsed with a value of an incorrect type - """ - - def __init__(self, argument_name: str, arg_type: type): - ParserException.__init__(self, argument_name) - self.msg = argument_name + " expect " + arg_type.__name__ - - -class BadContextException(ParserException): - """ - Exception raised when the parser catch an argument that it can't handle in - the current context - """ - - def __init__(self, argument_name: str, context_list: list): - ParserException.__init__(self, argument_name) - self.context_list = context_list - self.msg = 'argument ' + argument_name + 'not used in the correct context\nUse it with the following arguments :' - for main_arg_name, context_name in context_list: - self.msg += '\n --' + main_arg_name + ' ' + context_name - - -class NotAllowedArgumentValueException(PowerAPIException): - """ - This exception happens when the configuration define an argument value that is incompatible with other arguments' - values - """ - - -class FileDoesNotExistException(PowerAPIExceptionWithMessage): - """ - This exception happens when the configuration define a input file that does not exist or is not accessible - """ - - def __init__(self, file_name: str): - PowerAPIExceptionWithMessage.__init__(self, "The File " + file_name + " does not exist or is not accessible") - self.file_name = file_name - - -class SameLengthArgumentNamesException(ParserException): - """ - Exception raised when attempting to add an argument with names that have the same length - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - - -class ModelNameAlreadyUsed(PowerAPIException): - """ - Exception raised when attempting to add to a DBActorGenerator a model factory with a name already bound to another - model factory in the DBActorGenerator - """ - - def __init__(self, model_name: str): - PowerAPIException.__init__(self) - self.model_name = model_name - - -class DatabaseNameDoesNotExist(PowerAPIException): - """ - Exception raised when attempting to remove to a DBActorGenerator a database factory with a name that is not bound to - another database factory in the DBActorGenerator - """ - - def __init__(self, database_name: str): - PowerAPIException.__init__(self) - self.database_name = database_name - - -class DatabaseNameAlreadyUsed(PowerAPIException): - """ - Exception raised when attempting to add to a DBActorGenerator a database factory with a name already bound to - another database factory in the DBActorGenerator - """ - - def __init__(self, database_name: str): - PowerAPIException.__init__(self) - self.database_name = database_name - - -class ModelNameDoesNotExist(PowerAPIException): - """ - Exception raised when attempting to remove to a DBActorGenerator a model factory with a name that is not bound to - another model factory in the DBActorGenerator - """ - - def __init__(self, model_name: str): - PowerAPIException.__init__(self) - self.model_name = model_name - - -class InvalidPrefixException(PowerAPIException): - """ - Exception raised when attempting to add a new prefix that is a prefix of an existing one or - vice-versa - """ - - def __init__(self, existing_prefix: str, new_prefix: str): - PowerAPIException.__init__(self) - self.new_prefix = new_prefix - self.existing_prefix = existing_prefix - self.msg = "The new prefix " + self.new_prefix + " has a conflict with the existing prefix " \ - + self.existing_prefix - - -class ProcessorTypeDoesNotExist(PowerAPIException): - """ - Exception raised when attempting to remove to a ProcessorActorGenerator a processor factory with a type that is not - bound to a processor factory - """ - - def __init__(self, processor_type: str): - PowerAPIException.__init__(self) - self.processor_type = processor_type - - -class ProcessorTypeAlreadyUsed(PowerAPIException): - """ - Exception raised when attempting to add to a ProcessorActorGenerator a processor factory with a type already bound - to another processor factory - """ - - def __init__(self, processor_type: str): - PowerAPIException.__init__(self) - self.processor_type = processor_type - - class UnsupportedActorTypeException(ParserException): """ Exception raised when the binding manager do not support an actor type diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 4107c81b..ca72e00e 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -40,18 +40,6 @@ def several_inputs_outputs_stream_config(): return load_configuration_from_json_file('several_inputs_outputs_stream_mode_enabled_configuration.json') -@pytest.fixture -def several_inputs_outputs_stream_socket_without_some_arguments_config(several_inputs_outputs_stream_config): - """ - Configuration with a socket input missing a required argument. - """ - for current_input in several_inputs_outputs_stream_config['input'].values(): - if current_input['type'] == 'socket': - current_input.pop('port') - - return several_inputs_outputs_stream_config - - @pytest.fixture def several_inputs_outputs_postmortem_config(several_inputs_outputs_stream_config): """ diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 37b64ffe..8d6566a2 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -27,23 +27,36 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +from copy import deepcopy + import pytest -from powerapi.cli.generator import ModelNameDoesNotExist -from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, PreProcessorGenerator +from powerapi.cli.generator import ( + PreProcessorGenerator, + PullerGenerator, + PusherGenerator, +) from powerapi.database.csv.driver import CSVInputFactory, CSVOutputFactory from powerapi.database.json.driver import JsonInputFactory, JsonOutputFactory from powerapi.database.socket.driver import SocketInputFactory -from powerapi.exception import PowerAPIException +from powerapi.exception import ConfigurationError, PowerAPIException from powerapi.filter import BroadcastReportFilter from powerapi.puller import PullerActor from powerapi.pusher import PusherActor -from powerapi.report import PowerReport, FormulaReport +from powerapi.report import FormulaReport, HWPCReport, PowerReport + + +def _unavailable_factory(_: dict): + """ + Simulate a component factory whose optional dependency is unavailable. + """ + raise ImportError def test_generate_puller_from_empty_config_dict_raise_an_exception(): """ - Test that PullerGenerator raises a PowerAPIException when there is no input argument + Test that PullerGenerator raises a PowerAPIException when there is no input argument. """ conf = {} generator = PullerGenerator(BroadcastReportFilter()) @@ -52,16 +65,32 @@ def test_generate_puller_from_empty_config_dict_raise_an_exception(): generator.generate(conf) -def test_generate_several_pullers_from_config(several_inputs_outputs_stream_config): +@pytest.mark.parametrize('input_type', ['csv', 'json']) +def test_generate_file_puller_in_stream_mode_raise_an_exception(several_inputs_outputs_stream_config, input_type): + """ + Test that PullerGenerator rejects input types that do not support stream mode. + """ + config = deepcopy(several_inputs_outputs_stream_config) + config['input'] = {name: value for name, value in config['input'].items() if value['type'] == input_type} + generator = PullerGenerator(BroadcastReportFilter()) + + with pytest.raises(ConfigurationError) as raised_exception: + generator.generate(config) + + assert raised_exception.value.path == 'stream' + assert raised_exception.value.reason == f'Stream mode cannot be used with a {input_type} input' + + +def test_generate_several_pullers_from_config(several_inputs_outputs_postmortem_config): """ - Test that several inputs are correctly used to generate the related actors + Test that several inputs are correctly used to generate the related actors. """ generator = PullerGenerator(BroadcastReportFilter()) - pullers = generator.generate(several_inputs_outputs_stream_config) + pullers = generator.generate(several_inputs_outputs_postmortem_config) - assert len(pullers) == len(several_inputs_outputs_stream_config['input']) + assert len(pullers) == len(several_inputs_outputs_postmortem_config['input']) - for puller_name, current_puller_infos in several_inputs_outputs_stream_config['input'].items(): + for puller_name, current_puller_infos in several_inputs_outputs_postmortem_config['input'].items(): assert puller_name in pullers assert isinstance(pullers[puller_name], PullerActor) @@ -81,55 +110,113 @@ def test_generate_several_pullers_from_config(several_inputs_outputs_stream_conf pytest.fail(f'Unsupported puller type: {current_puller_infos["type"]}') -def test_generate_puller_raise_exception_when_missing_arguments_in_socket_input( - several_inputs_outputs_stream_socket_without_some_arguments_config): +def test_generate_streaming_puller_preserves_runtime_settings(several_inputs_outputs_stream_config): + """ + Test that a streaming puller receives its filter, stream mode, and logging level. + """ + config = deepcopy(several_inputs_outputs_stream_config) + config['input'] = {'puller3': config['input']['puller3']} + report_filter = BroadcastReportFilter() + + puller = PullerGenerator(report_filter).generate(config)['puller3'] + + assert puller.report_filter is report_filter + assert puller.stream_mode is True + assert puller.logging_level == logging.DEBUG + + +def test_generate_puller_with_registered_report_model(several_inputs_outputs_postmortem_config): """ - Test that PullerGenerator raise a PowerAPIException when some arguments are missing for socket input + Test that a registered report model is resolved when generating a puller. """ + config = deepcopy(several_inputs_outputs_postmortem_config) + config['input'] = {'puller2': config['input']['puller2']} + config['input']['puller2']['model'] = 'CustomReport' generator = PullerGenerator(BroadcastReportFilter()) + generator.add_report_class('CustomReport', HWPCReport) - with pytest.raises(PowerAPIException): - generator.generate(several_inputs_outputs_stream_socket_without_some_arguments_config) + puller = generator.generate(config)['puller2'] + + assert puller.database_factory.report_type is HWPCReport -def test_remove_model_factory_that_does_not_exist_on_a_DBActorGenerator_must_raise_ModelNameDoesNotExist(): +def test_register_existing_report_model_raises_value_error(): """ - Test that an exception is raised when a model factory that does not exist is erased + Test that a report model cannot be registered more than once. """ - generator = DBActorGenerator('input') - num_report_classes = len(generator.report_classes) + generator = PullerGenerator(BroadcastReportFilter()) - with pytest.raises(ModelNameDoesNotExist): - generator.remove_report_class('model') + with pytest.raises(ValueError, match='Report model "HWPCReport" is already registered'): + generator.add_report_class('HWPCReport', HWPCReport) - assert len(generator.report_classes) == num_report_classes +def test_register_existing_database_type_raises_value_error(): + """ + Test that a database type cannot be registered more than once. + """ + generator = PullerGenerator(BroadcastReportFilter()) + + with pytest.raises(ValueError, match='Database type "csv" is already registered'): + generator.add_db_factory('csv', generator.database_factories['csv']) -def test_remove_hwpc_report_model_and_generate_puller_from_a_config_using_model(several_inputs_outputs_stream_config): + +def test_generate_puller_with_unknown_report_model_raises_an_exception(several_inputs_outputs_postmortem_config): """ PullerGenerator should raise an exception when the model of an input is not defined. """ + config = deepcopy(several_inputs_outputs_postmortem_config) + next(iter(config['input'].values()))['model'] = 'UnknownReport' generator = PullerGenerator(BroadcastReportFilter()) - generator.remove_report_class('HWPCReport') - with pytest.raises(PowerAPIException): - _ = generator.generate(several_inputs_outputs_stream_config) + with pytest.raises(PowerAPIException, match='Configuration error: Unknown report model "UnknownReport"'): + generator.generate(config) -def test_remove_csv_database_factory_and_generate_puller_from_a_config_using_type(several_inputs_outputs_stream_config): +def test_generate_puller_with_unknown_database_type_raises_an_exception(several_inputs_outputs_postmortem_config): """ PullerGenerator should raise an exception when the database of an input is not defined. """ + config = deepcopy(several_inputs_outputs_postmortem_config) + next(iter(config['input'].values()))['type'] = 'unknown' generator = PullerGenerator(BroadcastReportFilter()) - generator.remove_db_factory('csv') - with pytest.raises(PowerAPIException): - _ = generator.generate(several_inputs_outputs_stream_config) + with pytest.raises(PowerAPIException, match='Configuration error: Invalid database type: unknown'): + generator.generate(config) + + +def test_generate_puller_with_unavailable_database_dependency_raises_an_exception(several_inputs_outputs_postmortem_config): + """ + Test that a missing database dependency is reported as a configuration error. + """ + config = deepcopy(several_inputs_outputs_postmortem_config) + config['input'] = {'puller2': config['input']['puller2']} + config['input']['puller2']['type'] = 'unavailable' + generator = PullerGenerator(BroadcastReportFilter()) + generator.add_db_factory('unavailable', _unavailable_factory) + with pytest.raises(PowerAPIException, match='Dependencies for unavailable database are not installed'): + generator.generate(config) -def test_generate_pusher_from_empty_config_dict_raise_an_exception(): + +def test_generate_does_not_modify_configuration(several_inputs_outputs_postmortem_config): """ - Test that PusherGenerator raise an exception when there is no output argument + Test that generating pullers and pushers repeatedly preserves the canonical configuration. + """ + expected = deepcopy(several_inputs_outputs_postmortem_config) + puller_generator = PullerGenerator(BroadcastReportFilter()) + pusher_generator = PusherGenerator() + + puller_generator.generate(several_inputs_outputs_postmortem_config) + pusher_generator.generate(several_inputs_outputs_postmortem_config) + puller_generator.generate(several_inputs_outputs_postmortem_config) + pusher_generator.generate(several_inputs_outputs_postmortem_config) + + assert several_inputs_outputs_postmortem_config == expected + + +def test_generate_pusher_from_empty_config_dict_raises_an_exception(): + """ + Test that PusherGenerator raises an exception when there is no output argument. """ conf = {} generator = PusherGenerator() @@ -140,7 +227,7 @@ def test_generate_pusher_from_empty_config_dict_raise_an_exception(): def test_generate_several_pushers_from_config(several_inputs_outputs_stream_config): """ - Test that several outputs are correctly used to generate the related actors + Test that several outputs are correctly used to generate the related actors. """ generator = PusherGenerator() @@ -181,12 +268,65 @@ def test_generate_pusher_report_type_to_actor_mapping(single_input_multiple_outp assert [proxy.actor_type for proxy in report_mapping[FormulaReport]] == [PusherActor] -def test_generate_pre_processor_from_empty_config_dict_raise_an_exception(): +def test_generate_pusher_report_mapping_without_output_group_raises_an_exception(): + """ + Test that report mapping requires the output component group. + """ + generator = PusherGenerator() + + with pytest.raises(PowerAPIException, match='Configuration error: Component "output" is not defined'): + generator.generate_report_mapping({}, {}) + + +def test_generate_pusher_report_mapping_with_missing_actor_raises_an_exception(): + """ + Test that report mapping rejects an output without a generated actor. + """ + config = {'output': {'missing': {'model': 'PowerReport'}}} + generator = PusherGenerator() + + with pytest.raises(PowerAPIException, match='Actor "missing" is not defined'): + generator.generate_report_mapping(config, {}) + + +def test_generate_pre_processor_from_empty_config_dict_raises_an_exception(): """ - Test that PreProcessGenerator raise an exception when there is no processor argument + Test that PreProcessGenerator raises an exception when there is no processor argument. """ conf = {} generator = PreProcessorGenerator() with pytest.raises(PowerAPIException): generator.generate(conf) + + +def test_register_existing_processor_type_raises_value_error(): + """ + Test that a processor type cannot be registered more than once. + """ + generator = PreProcessorGenerator() + + with pytest.raises(ValueError, match='Processor type "kubernetes" is already registered'): + generator.add_processor_factory('kubernetes', generator.processor_factories['kubernetes']) + + +def test_generate_unknown_processor_type_raises_an_exception(): + """ + Test that generating an unknown processor type raises a configuration error. + """ + config = {'verbose': False, 'pre-processor': {'processor': {'type': 'unknown'}}} + + with pytest.raises(PowerAPIException, match='Configuration error: Invalid processor type: unknown'): + PreProcessorGenerator().generate(config) + + +def test_generate_processor_with_unavailable_dependency_raises_an_exception(): + """ + Test that a missing processor dependency is reported as a configuration error. + """ + config = {'verbose': False, 'pre-processor': {'processor': {'type': 'unavailable'}}} + generator = PreProcessorGenerator() + generator.add_processor_factory('unavailable', _unavailable_factory) + + with pytest.raises(PowerAPIException, match='Dependencies for unavailable processor are not installed'): + generator.generate(config) diff --git a/tests/unit/cli/test_generator_clickhouse.py b/tests/unit/cli/test_generator_clickhouse.py index ca6746df..e0d347eb 100644 --- a/tests/unit/cli/test_generator_clickhouse.py +++ b/tests/unit/cli/test_generator_clickhouse.py @@ -29,7 +29,6 @@ import pytest from powerapi.cli.generator import PusherGenerator -from powerapi.exception import PowerAPIException from powerapi.pusher import PusherActor pytest.importorskip('powerapi.database.clickhouse.driver') # The ClickHouse driver requires external dependencies to work. @@ -81,16 +80,3 @@ def test_pusher_generator_with_valid_clickhouse_config(clickhouse_config): assert db_factory.username == expected_db_attributes['username'] assert db_factory.password == expected_db_attributes['password'] assert db_factory.database_name == expected_db_attributes['database'] - - -@pytest.mark.parametrize('missing_arg', ['model', 'host', 'port', 'username', 'password', 'database']) -def test_pusher_generator_with_missing_arguments_in_clickhouse_config(clickhouse_config, missing_arg): - """ - PusherGenerator should raise an exception when a required argument is missing from the ClickHouse config. - """ - generator = PusherGenerator() - - clickhouse_config['output']['pytest-clickhouse-pusher'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(clickhouse_config) diff --git a/tests/unit/cli/test_generator_influxdb2.py b/tests/unit/cli/test_generator_influxdb2.py index 4acc96ce..3b11b968 100644 --- a/tests/unit/cli/test_generator_influxdb2.py +++ b/tests/unit/cli/test_generator_influxdb2.py @@ -29,7 +29,6 @@ import pytest from powerapi.cli.generator import PusherGenerator -from powerapi.exception import PowerAPIException from powerapi.pusher import PusherActor pytest.importorskip("powerapi.database.influxdb2.driver") # The InfluxDB2 driver requires external dependencies to work. @@ -79,16 +78,3 @@ def test_pusher_generator_with_valid_influxdb2_config(influxdb2_config): assert db_factory.org == expected_db_attributes['org'] assert db_factory.token == expected_db_attributes['token'] assert db_factory.bucket == expected_db_attributes['bucket'] - - -@pytest.mark.parametrize('missing_arg', ['model', 'uri', 'org', 'token', 'bucket']) -def test_pusher_generator_with_missing_arguments_in_influxdb2_config(influxdb2_config, missing_arg): - """ - PusherGenerator should raise an exception when a required argument is missing from the InfluxDB2 config. - """ - generator = PusherGenerator() - - influxdb2_config['output']['pytest-influxdb2-pusher'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(influxdb2_config) diff --git a/tests/unit/cli/test_generator_k8s.py b/tests/unit/cli/test_generator_k8s.py index 9726caeb..32b6e7e5 100644 --- a/tests/unit/cli/test_generator_k8s.py +++ b/tests/unit/cli/test_generator_k8s.py @@ -31,7 +31,6 @@ pytest.importorskip('kubernetes') from powerapi.cli.generator import PreProcessorGenerator -from powerapi.exception import PowerAPIException from powerapi.processor.pre.k8s.actor import KubernetesPreProcessorActor @@ -49,6 +48,7 @@ def k8s_processor_config(): 'api-mode': 'manual', 'api-host': 'https://127.0.0.1:36599', 'api-key': 'pytest-token-powerapi', + 'labels': ['app.kubernetes.io/name'], 'puller': 'pytest-json-puller' } } @@ -72,16 +72,6 @@ def test_preprocessor_generator_with_valid_k8s_config(k8s_processor_config): assert preprocessor.monitor_config.api_mode == expected_preprocessor_attributes['api-mode'] assert preprocessor.monitor_config.api_key == expected_preprocessor_attributes['api-key'] assert preprocessor.monitor_config.api_host == expected_preprocessor_attributes['api-host'] - - -@pytest.mark.parametrize('missing_arg', ['api-mode']) -def test_preprocessor_generator_with_missing_arguments_in_k8s_config(k8s_processor_config, missing_arg): - """ - PreProcessorGenerator should raise an exception when a required argument is missing from the MongoDB config. - """ - generator = PreProcessorGenerator() - - k8s_processor_config['pre-processor']['pytest-k8s-preprocessor'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(k8s_processor_config) + assert preprocessor.monitor_config.label_mapping == { + 'app.kubernetes.io/name': 'k8s_pod_label_app_kubernetes_io_name', + } diff --git a/tests/unit/cli/test_generator_mongodb.py b/tests/unit/cli/test_generator_mongodb.py index 22d524f9..a7f2803c 100644 --- a/tests/unit/cli/test_generator_mongodb.py +++ b/tests/unit/cli/test_generator_mongodb.py @@ -29,7 +29,6 @@ import pytest from powerapi.cli.generator import PusherGenerator, PullerGenerator -from powerapi.exception import PowerAPIException from powerapi.filter import BroadcastReportFilter from powerapi.puller import PullerActor from powerapi.pusher import PusherActor @@ -89,20 +88,6 @@ def test_puller_generator_with_valid_mongodb_config(mongodb_config): assert db_factory.database_name == expected_db_attributes['db'] assert db_factory.collection_name == expected_db_attributes['collection'] - -@pytest.mark.parametrize('missing_arg', ['model', 'uri']) -def test_puller_generator_with_missing_arguments_in_mongodb_config(mongodb_config, missing_arg): - """ - PullerGenerator should raise an exception when a required argument is missing from the MongoDB config. - """ - generator = PullerGenerator(BroadcastReportFilter()) - - mongodb_config['input']['pytest-mongodb-puller'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(mongodb_config) - - def test_pusher_generator_with_valid_mongodb_config(mongodb_config): """ PusherGenerator should generate a PusherActor with a MongoDB database driver. @@ -123,16 +108,3 @@ def test_pusher_generator_with_valid_mongodb_config(mongodb_config): assert db_factory.uri == expected_db_attributes['uri'] assert db_factory.database_name == expected_db_attributes['db'] assert db_factory.collection_name == expected_db_attributes['collection'] - - -@pytest.mark.parametrize('missing_arg', ['model', 'uri', 'db', 'collection']) -def test_pusher_generator_with_missing_arguments_in_mongodb_config(mongodb_config, missing_arg): - """ - PusherGenerator should raise an exception when a required argument is missing from the MongoDB config. - """ - generator = PusherGenerator() - - mongodb_config['output']['pytest-mongodb-pusher'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(mongodb_config) diff --git a/tests/unit/cli/test_generator_openstack.py b/tests/unit/cli/test_generator_openstack.py index 53dc6848..633cdaf2 100644 --- a/tests/unit/cli/test_generator_openstack.py +++ b/tests/unit/cli/test_generator_openstack.py @@ -46,7 +46,8 @@ def openstack_config(): 'pytest-openstack-preprocessor': { 'type': 'openstack', 'puller': 'pytest-json-puller', - 'polling-interval': 10.0 + 'polling-interval': 10.0, + 'metadata': ['environment'], } } } @@ -67,3 +68,6 @@ def test_preprocessor_generator_with_valid_openstack_config(openstack_config): expected_preprocessor_attributes = openstack_config['pre-processor']['pytest-openstack-preprocessor'] assert preprocessor.monitor_config.polling_interval == expected_preprocessor_attributes['polling-interval'] + assert preprocessor.monitor_config.metadata_mapping == { + 'environment': 'openstack_metadata_environment', + } diff --git a/tests/unit/cli/test_generator_prometheus.py b/tests/unit/cli/test_generator_prometheus.py index 4c1b360b..b6131adb 100644 --- a/tests/unit/cli/test_generator_prometheus.py +++ b/tests/unit/cli/test_generator_prometheus.py @@ -29,7 +29,6 @@ import pytest from powerapi.cli.generator import PusherGenerator -from powerapi.exception import PowerAPIException from powerapi.pusher import PusherActor pytest.importorskip("powerapi.database.prometheus.driver") # The Prometheus driver requires external dependencies to work. @@ -76,16 +75,3 @@ def test_pusher_generator_with_valid_prometheus_config(prometheus_config): assert db_factory.listen_addr == expected_db_attributes['addr'] assert db_factory.listen_port == expected_db_attributes['port'] assert set(db_factory.tags) == {'powerapi_example_tag1', 'powerapi_example_tag2'} - - -@pytest.mark.parametrize('missing_arg', ['model', 'addr', 'port']) -def test_pusher_generator_with_missing_arguments_in_mongodb_config(prometheus_config, missing_arg): - """ - PusherGenerator should raise an exception when a required argument is missing from the Prometheus config. - """ - generator = PusherGenerator() - - prometheus_config['output']['pytest-prometheus-pusher'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(prometheus_config) From 315e969a20342873e782668d7ab5faa521d8c208 Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Fri, 4 Sep 2026 15:15:48 +0200 Subject: [PATCH 5/5] refactor(config)!: Rename the `cli` package to `config` BREAKING CHANGE: Modules previously imported from `powerapi.cli` must now be imported from `powerapi.config`. --- src/powerapi/{cli => config}/__init__.py | 0 src/powerapi/{cli => config}/_utils.py | 0 src/powerapi/{cli => config}/binding_manager.py | 0 src/powerapi/{cli => config}/cli_parser.py | 2 +- src/powerapi/{cli => config}/common_cli_parsing_manager.py | 4 ++-- src/powerapi/{cli => config}/config_loader.py | 2 +- src/powerapi/{cli => config}/config_parser.py | 0 src/powerapi/{cli => config}/generator.py | 0 src/powerapi/{cli => config}/parsing_manager.py | 6 +++--- tests/unit/{cli => config}/__init__.py | 0 tests/unit/{cli => config}/conftest.py | 0 tests/unit/{cli => config}/test_binding_manager.py | 4 ++-- tests/unit/{cli => config}/test_cli_parser.py | 6 +++--- .../unit/{cli => config}/test_common_cli_parsing_manager.py | 4 ++-- tests/unit/{cli => config}/test_config_parser.py | 4 ++-- tests/unit/{cli => config}/test_generator.py | 2 +- tests/unit/{cli => config}/test_generator_clickhouse.py | 2 +- tests/unit/{cli => config}/test_generator_influxdb2.py | 2 +- tests/unit/{cli => config}/test_generator_k8s.py | 2 +- tests/unit/{cli => config}/test_generator_mongodb.py | 2 +- tests/unit/{cli => config}/test_generator_openstack.py | 2 +- tests/unit/{cli => config}/test_generator_prometheus.py | 2 +- tests/unit/{cli => config}/test_parsing_manager.py | 6 +++--- tests/unit/{cli => config}/test_utils.py | 2 +- 24 files changed, 27 insertions(+), 27 deletions(-) rename src/powerapi/{cli => config}/__init__.py (100%) rename src/powerapi/{cli => config}/_utils.py (100%) rename src/powerapi/{cli => config}/binding_manager.py (100%) rename src/powerapi/{cli => config}/cli_parser.py (99%) rename src/powerapi/{cli => config}/common_cli_parsing_manager.py (99%) rename src/powerapi/{cli => config}/config_loader.py (99%) rename src/powerapi/{cli => config}/config_parser.py (100%) rename src/powerapi/{cli => config}/generator.py (100%) rename src/powerapi/{cli => config}/parsing_manager.py (97%) rename tests/unit/{cli => config}/__init__.py (100%) rename tests/unit/{cli => config}/conftest.py (100%) rename tests/unit/{cli => config}/test_binding_manager.py (98%) rename tests/unit/{cli => config}/test_cli_parser.py (98%) rename tests/unit/{cli => config}/test_common_cli_parsing_manager.py (98%) rename tests/unit/{cli => config}/test_config_parser.py (99%) rename tests/unit/{cli => config}/test_generator.py (99%) rename tests/unit/{cli => config}/test_generator_clickhouse.py (98%) rename tests/unit/{cli => config}/test_generator_influxdb2.py (98%) rename tests/unit/{cli => config}/test_generator_k8s.py (98%) rename tests/unit/{cli => config}/test_generator_mongodb.py (98%) rename tests/unit/{cli => config}/test_generator_openstack.py (98%) rename tests/unit/{cli => config}/test_generator_prometheus.py (98%) rename tests/unit/{cli => config}/test_parsing_manager.py (97%) rename tests/unit/{cli => config}/test_utils.py (97%) diff --git a/src/powerapi/cli/__init__.py b/src/powerapi/config/__init__.py similarity index 100% rename from src/powerapi/cli/__init__.py rename to src/powerapi/config/__init__.py diff --git a/src/powerapi/cli/_utils.py b/src/powerapi/config/_utils.py similarity index 100% rename from src/powerapi/cli/_utils.py rename to src/powerapi/config/_utils.py diff --git a/src/powerapi/cli/binding_manager.py b/src/powerapi/config/binding_manager.py similarity index 100% rename from src/powerapi/cli/binding_manager.py rename to src/powerapi/config/binding_manager.py diff --git a/src/powerapi/cli/cli_parser.py b/src/powerapi/config/cli_parser.py similarity index 99% rename from src/powerapi/cli/cli_parser.py rename to src/powerapi/config/cli_parser.py index 8567d348..c6de36c9 100644 --- a/src/powerapi/cli/cli_parser.py +++ b/src/powerapi/config/cli_parser.py @@ -29,7 +29,7 @@ import argparse from dataclasses import dataclass -from powerapi.cli.config_parser import ( +from powerapi.config.config_parser import ( ArgumentDefinition, ComponentGroupSchema, ComponentSchema, diff --git a/src/powerapi/cli/common_cli_parsing_manager.py b/src/powerapi/config/common_cli_parsing_manager.py similarity index 99% rename from src/powerapi/cli/common_cli_parsing_manager.py rename to src/powerapi/config/common_cli_parsing_manager.py index a7cad558..a24c630a 100644 --- a/src/powerapi/cli/common_cli_parsing_manager.py +++ b/src/powerapi/config/common_cli_parsing_manager.py @@ -27,8 +27,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.cli.config_parser import ComponentSchema -from powerapi.cli.parsing_manager import ConfigurationParsingManager +from powerapi.config.config_parser import ComponentSchema +from powerapi.config.parsing_manager import ConfigurationParsingManager def generate_env_prefix(*components: str, root_prefix: str = 'POWERAPI') -> str: diff --git a/src/powerapi/cli/config_loader.py b/src/powerapi/config/config_loader.py similarity index 99% rename from src/powerapi/cli/config_loader.py rename to src/powerapi/config/config_loader.py index a5f0b8a5..b9613e46 100644 --- a/src/powerapi/cli/config_loader.py +++ b/src/powerapi/config/config_loader.py @@ -30,7 +30,7 @@ import os from collections.abc import Iterable -from powerapi.cli.config_parser import ( +from powerapi.config.config_parser import ( ComponentGroupSchema, ConfigurationSchema, ) diff --git a/src/powerapi/cli/config_parser.py b/src/powerapi/config/config_parser.py similarity index 100% rename from src/powerapi/cli/config_parser.py rename to src/powerapi/config/config_parser.py diff --git a/src/powerapi/cli/generator.py b/src/powerapi/config/generator.py similarity index 100% rename from src/powerapi/cli/generator.py rename to src/powerapi/config/generator.py diff --git a/src/powerapi/cli/parsing_manager.py b/src/powerapi/config/parsing_manager.py similarity index 97% rename from src/powerapi/cli/parsing_manager.py rename to src/powerapi/config/parsing_manager.py index dc229362..a4a378a6 100644 --- a/src/powerapi/cli/parsing_manager.py +++ b/src/powerapi/config/parsing_manager.py @@ -30,9 +30,9 @@ import sys from typing import Any -from powerapi.cli.cli_parser import CLIArgumentParser -from powerapi.cli.config_loader import EnvironmentConfigLoader, JSONConfigLoader -from powerapi.cli.config_parser import ( +from powerapi.config.cli_parser import CLIArgumentParser +from powerapi.config.config_loader import EnvironmentConfigLoader, JSONConfigLoader +from powerapi.config.config_parser import ( ComponentSchema, ConfigurationSchema, ConfigurationSectionSchema, diff --git a/tests/unit/cli/__init__.py b/tests/unit/config/__init__.py similarity index 100% rename from tests/unit/cli/__init__.py rename to tests/unit/config/__init__.py diff --git a/tests/unit/cli/conftest.py b/tests/unit/config/conftest.py similarity index 100% rename from tests/unit/cli/conftest.py rename to tests/unit/config/conftest.py diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/config/test_binding_manager.py similarity index 98% rename from tests/unit/cli/test_binding_manager.py rename to tests/unit/config/test_binding_manager.py index 97b8f7e2..738cac3a 100644 --- a/tests/unit/cli/test_binding_manager.py +++ b/tests/unit/config/test_binding_manager.py @@ -32,8 +32,8 @@ import pytest from powerapi.actor import ActorProxy -from powerapi.cli.binding_manager import PreProcessorBindingManager -from powerapi.cli.generator import PreProcessorGenerator, PullerGenerator +from powerapi.config.binding_manager import PreProcessorBindingManager +from powerapi.config.generator import PreProcessorGenerator, PullerGenerator from powerapi.dispatcher import DispatcherActor from powerapi.exception import UnexistingActorException, UnsupportedActorTypeException, TargetActorAlreadyUsed from powerapi.filter import BroadcastReportFilter diff --git a/tests/unit/cli/test_cli_parser.py b/tests/unit/config/test_cli_parser.py similarity index 98% rename from tests/unit/cli/test_cli_parser.py rename to tests/unit/config/test_cli_parser.py index 8da90f52..269f787b 100644 --- a/tests/unit/cli/test_cli_parser.py +++ b/tests/unit/config/test_cli_parser.py @@ -30,9 +30,9 @@ import pytest -from powerapi.cli import cli_parser -from powerapi.cli.cli_parser import CLIArgumentParser, CLIParseException -from powerapi.cli.config_parser import ( +from powerapi.config import cli_parser +from powerapi.config.cli_parser import CLIArgumentParser, CLIParseException +from powerapi.config.config_parser import ( ComponentSchema, ConfigurationSchema, ConfigurationSectionSchema, diff --git a/tests/unit/cli/test_common_cli_parsing_manager.py b/tests/unit/config/test_common_cli_parsing_manager.py similarity index 98% rename from tests/unit/cli/test_common_cli_parsing_manager.py rename to tests/unit/config/test_common_cli_parsing_manager.py index 7134f5a1..5e445c28 100644 --- a/tests/unit/cli/test_common_cli_parsing_manager.py +++ b/tests/unit/config/test_common_cli_parsing_manager.py @@ -28,8 +28,8 @@ import pytest -from powerapi.cli.cli_parser import CLIParseException -from powerapi.cli.common_cli_parsing_manager import ( +from powerapi.config.cli_parser import CLIParseException +from powerapi.config.common_cli_parsing_manager import ( CommonCLIParsingManager, PreProcessorSchema, PullerSchema, diff --git a/tests/unit/cli/test_config_parser.py b/tests/unit/config/test_config_parser.py similarity index 99% rename from tests/unit/cli/test_config_parser.py rename to tests/unit/config/test_config_parser.py index 4f66d26a..48fe2465 100644 --- a/tests/unit/cli/test_config_parser.py +++ b/tests/unit/config/test_config_parser.py @@ -30,8 +30,8 @@ import pytest -from powerapi.cli.config_loader import EnvironmentConfigLoader, JSONConfigLoader -from powerapi.cli.config_parser import ( +from powerapi.config.config_loader import EnvironmentConfigLoader, JSONConfigLoader +from powerapi.config.config_parser import ( ComponentSchema, ConfigurationSchema, ConfigurationSectionSchema, diff --git a/tests/unit/cli/test_generator.py b/tests/unit/config/test_generator.py similarity index 99% rename from tests/unit/cli/test_generator.py rename to tests/unit/config/test_generator.py index 8d6566a2..74b56c5a 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/config/test_generator.py @@ -32,7 +32,7 @@ import pytest -from powerapi.cli.generator import ( +from powerapi.config.generator import ( PreProcessorGenerator, PullerGenerator, PusherGenerator, diff --git a/tests/unit/cli/test_generator_clickhouse.py b/tests/unit/config/test_generator_clickhouse.py similarity index 98% rename from tests/unit/cli/test_generator_clickhouse.py rename to tests/unit/config/test_generator_clickhouse.py index e0d347eb..2f8551b2 100644 --- a/tests/unit/cli/test_generator_clickhouse.py +++ b/tests/unit/config/test_generator_clickhouse.py @@ -28,7 +28,7 @@ import pytest -from powerapi.cli.generator import PusherGenerator +from powerapi.config.generator import PusherGenerator from powerapi.pusher import PusherActor pytest.importorskip('powerapi.database.clickhouse.driver') # The ClickHouse driver requires external dependencies to work. diff --git a/tests/unit/cli/test_generator_influxdb2.py b/tests/unit/config/test_generator_influxdb2.py similarity index 98% rename from tests/unit/cli/test_generator_influxdb2.py rename to tests/unit/config/test_generator_influxdb2.py index 3b11b968..90e30a03 100644 --- a/tests/unit/cli/test_generator_influxdb2.py +++ b/tests/unit/config/test_generator_influxdb2.py @@ -28,7 +28,7 @@ import pytest -from powerapi.cli.generator import PusherGenerator +from powerapi.config.generator import PusherGenerator from powerapi.pusher import PusherActor pytest.importorskip("powerapi.database.influxdb2.driver") # The InfluxDB2 driver requires external dependencies to work. diff --git a/tests/unit/cli/test_generator_k8s.py b/tests/unit/config/test_generator_k8s.py similarity index 98% rename from tests/unit/cli/test_generator_k8s.py rename to tests/unit/config/test_generator_k8s.py index 32b6e7e5..7a92b140 100644 --- a/tests/unit/cli/test_generator_k8s.py +++ b/tests/unit/config/test_generator_k8s.py @@ -30,7 +30,7 @@ pytest.importorskip('kubernetes') -from powerapi.cli.generator import PreProcessorGenerator +from powerapi.config.generator import PreProcessorGenerator from powerapi.processor.pre.k8s.actor import KubernetesPreProcessorActor diff --git a/tests/unit/cli/test_generator_mongodb.py b/tests/unit/config/test_generator_mongodb.py similarity index 98% rename from tests/unit/cli/test_generator_mongodb.py rename to tests/unit/config/test_generator_mongodb.py index a7f2803c..1c0b1411 100644 --- a/tests/unit/cli/test_generator_mongodb.py +++ b/tests/unit/config/test_generator_mongodb.py @@ -28,7 +28,7 @@ import pytest -from powerapi.cli.generator import PusherGenerator, PullerGenerator +from powerapi.config.generator import PusherGenerator, PullerGenerator from powerapi.filter import BroadcastReportFilter from powerapi.puller import PullerActor from powerapi.pusher import PusherActor diff --git a/tests/unit/cli/test_generator_openstack.py b/tests/unit/config/test_generator_openstack.py similarity index 98% rename from tests/unit/cli/test_generator_openstack.py rename to tests/unit/config/test_generator_openstack.py index 633cdaf2..f8fab8b4 100644 --- a/tests/unit/cli/test_generator_openstack.py +++ b/tests/unit/config/test_generator_openstack.py @@ -30,7 +30,7 @@ pytest.importorskip('openstack') -from powerapi.cli.generator import PreProcessorGenerator +from powerapi.config.generator import PreProcessorGenerator from powerapi.processor.pre.openstack.actor import OpenStackPreProcessorActor diff --git a/tests/unit/cli/test_generator_prometheus.py b/tests/unit/config/test_generator_prometheus.py similarity index 98% rename from tests/unit/cli/test_generator_prometheus.py rename to tests/unit/config/test_generator_prometheus.py index b6131adb..f6b1d10a 100644 --- a/tests/unit/cli/test_generator_prometheus.py +++ b/tests/unit/config/test_generator_prometheus.py @@ -28,7 +28,7 @@ import pytest -from powerapi.cli.generator import PusherGenerator +from powerapi.config.generator import PusherGenerator from powerapi.pusher import PusherActor pytest.importorskip("powerapi.database.prometheus.driver") # The Prometheus driver requires external dependencies to work. diff --git a/tests/unit/cli/test_parsing_manager.py b/tests/unit/config/test_parsing_manager.py similarity index 97% rename from tests/unit/cli/test_parsing_manager.py rename to tests/unit/config/test_parsing_manager.py index c19ade09..db33be67 100644 --- a/tests/unit/cli/test_parsing_manager.py +++ b/tests/unit/config/test_parsing_manager.py @@ -32,9 +32,9 @@ import pytest -from powerapi.cli.cli_parser import CLIParseException -from powerapi.cli.config_parser import ComponentSchema, ConfigurationSectionSchema -from powerapi.cli.parsing_manager import ConfigurationParsingManager +from powerapi.config.cli_parser import CLIParseException +from powerapi.config.config_parser import ComponentSchema, ConfigurationSectionSchema +from powerapi.config.parsing_manager import ConfigurationParsingManager from powerapi.exception import ConfigurationError diff --git a/tests/unit/cli/test_utils.py b/tests/unit/config/test_utils.py similarity index 97% rename from tests/unit/cli/test_utils.py rename to tests/unit/config/test_utils.py index f0fc8bef..a229b0b6 100644 --- a/tests/unit/cli/test_utils.py +++ b/tests/unit/config/test_utils.py @@ -28,7 +28,7 @@ import pytest -from powerapi.cli._utils import merge_dictionaries, string_to_bool, string_to_list +from powerapi.config._utils import merge_dictionaries, string_to_bool, string_to_list @pytest.mark.parametrize('value', [' YES ', 'y', 'true', 't', '1'])