diff --git a/src/powerapi/cli/_utils.py b/src/powerapi/cli/_utils.py deleted file mode 100644 index 20d528eb..00000000 --- a/src/powerapi/cli/_utils.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2023, Inria -# Copyright (c) 2023, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -def find_longest_string_in_list(strings: list[str]) -> str: - """ - Find the longest string from a given list of string. - :param strings: List of strings - """ - return max(strings, key=len) - - -def string_to_bool(value: str) -> bool: - """ - Transforms a string to a boolean according to its content. - :param value: The string to be converted - :return: Boolean value - """ - return value.casefold() in ("yes", "y", "true", "t", "1") - - -def string_to_list(value: str) -> list: - """ - Transforms a comma separated list to a list of strings. - :param value: The string to be converted - :return: List of strings - """ - if value == '': - return [] - - return [v.strip() for v in value.split(',')] - - -def merge_dictionaries(source: dict, destination: dict) -> dict: - """ - Recursively merge the source dictionary into destination. - :param source: Dictionary to be merged - :param destination: Dictionary where the source will be merged to - """ - for key, value in source.items(): - if isinstance(value, dict) and key in destination and isinstance(destination[key], dict): - destination[key] = merge_dictionaries(value, destination[key]) - else: - destination[key] = value - - return destination - - -def get_longest_related_suffix(var: str, suffixes: list) -> str: - """ - Search for the longest suffix of a string variable in a provided list. It returns None if a suffix is not found - :param var: A string for looking its longest suffix - :param suffixes: A list of suffixes - """ - suffix = None - - for current_suffix in suffixes: - if var.endswith(current_suffix): - if suffix is None: - suffix = current_suffix - elif len(current_suffix) > len(suffix): - suffix = current_suffix - - return suffix - - -def to_lower_case_and_replace_separators(strings: list, old_separator: str, new_separator: str) -> list: - """ - Transform all strings in a provided list to lower case and replace a given separator for a new one - :param strings: List of string to be converted to lower case - :param old_separator: The old separator in the list of strings - :param new_separator: The new separator in the list of strings - """ - new_strings = [] - for current_string in strings: - new_strings.append(current_string.lower().replace(old_separator, new_separator)) - - return new_strings diff --git a/src/powerapi/cli/common_cli_parsing_manager.py b/src/powerapi/cli/common_cli_parsing_manager.py deleted file mode 100644 index 86df9594..00000000 --- a/src/powerapi/cli/common_cli_parsing_manager.py +++ /dev/null @@ -1,490 +0,0 @@ -# Copyright (c) 2021, Inria -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from powerapi.cli.config_parser import store_true -from powerapi.cli.parsing_manager import RootConfigParsingManager, SubgroupConfigParsingManager - - -def generate_env_prefix(*components: str, root_prefix: str = 'POWERAPI') -> str: - """ - Generate the environment variable prefix from the given components. - :param components: Additional prefix components. - :param root_prefix: Root namespace for the prefix. - :return: The normalized environment variable prefix. - """ - return '_'.join( - normalized_part.upper() for part in (root_prefix, *components) if (normalized_part := part.strip()) - ) + '_' - - -class PullerConfigParsingManager(SubgroupConfigParsingManager): - """ - Subgroup parser with arguments shared by every puller input. - """ - - def __init__(self, name: str) -> None: - """ - Initialize a puller parser with common actor name and report model arguments. - """ - super().__init__(name) - - self.add_argument( - 'n', 'name', - help_text='Name assigned to this puller actor' - ) - self.add_argument( - 'm', 'model', - help_text='Report type produced by this input source', - default_value='HWPCReport' - ) - - -class PusherConfigParsingManager(SubgroupConfigParsingManager): - """ - Subgroup parser with arguments shared by every pusher output. - """ - - def __init__(self, name: str) -> None: - """ - Initialize a pusher parser with common actor name and report model arguments. - """ - super().__init__(name) - - self.add_argument( - 'n', 'name', - help_text='Name assigned to this pusher actor' - ) - self.add_argument( - 'm', 'model', - help_text='Report type consumed by this output destination', - default_value='PowerReport' - ) - - -class PreProcessorConfigParsingManager(SubgroupConfigParsingManager): - """ - Subgroup parser with arguments shared by every pre-processor. - """ - - def __init__(self, name: str) -> None: - """ - Initialize a pre-processor parser with common actor name and puller binding arguments. - """ - super().__init__(name) - - self.add_argument( - 'n', 'name', - help_text='Name assigned to this pre-processor actor' - ) - self.add_argument( - 'p', 'puller', - help_text='Name of the puller actor this pre-processor receives reports from', - is_mandatory=True, - ) - - -class CommonCLIParsingManager(RootConfigParsingManager): - """ - Root parser that registers PowerAPI's built-in CLI component options. - """ - - def __init__(self) -> None: - """ - Initialize the root parser and register all built-in component parsers. - """ - super().__init__() - - self._register_environment_prefixes() - self._register_subgroups() - self._register_root_arguments() - self._register_input_parsers() - self._register_output_parsers() - self._register_pre_processor_parsers() - - def _register_environment_prefixes(self) -> None: - """ - Register environment variable prefixes accepted by the root parser. - """ - self.add_argument_prefix(generate_env_prefix()) - - def _register_subgroups(self) -> None: - """ - Register top-level component groups accepted by the CLI. - """ - self.add_subgroup( - name='input', - prefix=generate_env_prefix('INPUT'), - help_text='Configure an input source: --input TYPE OPTIONS' - ) - self.add_subgroup( - name='output', - prefix=generate_env_prefix('OUTPUT'), - help_text='Configure an output destination: --output TYPE OPTIONS' - ) - self.add_subgroup( - name='pre-processor', - prefix=generate_env_prefix('PRE_PROCESSOR'), - help_text='Configure a pre-processor: --pre-processor TYPE OPTIONS' - ) - self.add_subgroup( - name='post-processor', - prefix=generate_env_prefix('POST_PROCESSOR'), - help_text='Configure a post-processor: --post-processor TYPE OPTIONS' - ) - - def _register_root_arguments(self) -> None: - """ - Register root-level options that apply to the whole PowerAPI process. - """ - self.add_argument( - 'v', 'verbose', - is_flag=True, - action=store_true, - default_value=False, - help_text='Enable verbose logging', - ) - self.add_argument( - 's', 'stream', - is_flag=True, - action=store_true, - default_value=False, - help_text='Enable stream processing mode', - ) - - def _register_input_parsers(self): - """ - Register all built-in input source parsers. - """ - self._register_mongodb_input_parser() - self._register_socket_input_parser() - self._register_csv_input_parser() - self._register_json_input_parser() - - def _register_mongodb_input_parser(self): - """ - Register the MongoDB input parser. - """ - subparser_mongo_input = PullerConfigParsingManager('mongodb') - - subparser_mongo_input.add_argument( - 'u', 'uri', - help_text='MongoDB connection URI', - is_mandatory=True - ) - subparser_mongo_input.add_argument( - 'd', 'db', - help_text='MongoDB database name', - is_mandatory=True - ) - subparser_mongo_input.add_argument( - 'c', 'collection', - help_text='MongoDB collection name', - is_mandatory=True - ) - - self.add_subgroup_parser('input', subparser_mongo_input) - - def _register_socket_input_parser(self): - """ - Register the Socket input parser. - """ - subparser_socket_input = PullerConfigParsingManager('socket') - - subparser_socket_input.add_argument( - 'h', 'host', - help_text='Host address the socket listens on', - default_value='localhost' - ) - subparser_socket_input.add_argument( - 'p', 'port', - help_text="Port number the socket listens on", - argument_type=int, - default_value=9080, - ) - - self.add_subgroup_parser('input', subparser_socket_input) - - def _register_csv_input_parser(self): - """ - Register the CSV input parser. - """ - subparser_csv_input = PullerConfigParsingManager('csv') - - subparser_csv_input.add_argument( - 'f', 'files', - help_text='Comma-separated list of CSV input files', - argument_type=list, - is_mandatory=True - ) - - self.add_subgroup_parser('input', subparser_csv_input) - - def _register_json_input_parser(self): - """ - Register the JSON input parser. - """ - subparser_json_input = PullerConfigParsingManager('json') - - subparser_json_input.add_argument( - 'f', 'filepath', - help_text='Path to the JSON input file', - is_mandatory=True - ) - subparser_json_input.add_argument( - 'c', 'compression', - help_text='Input compression format: auto, gzip, lzma, or none', - default_value='auto' - ) - - self.add_subgroup_parser('input', subparser_json_input) - - def _register_output_parsers(self): - """ - Register all built-in output destination parsers. - """ - self._register_mongodb_output_parser() - self._register_prometheus_output_parser() - self._register_csv_output_parser() - self._register_json_output_parser() - self._register_influxdb2_output_parser() - self._register_clickhouse_output_parser() - - def _register_mongodb_output_parser(self): - """ - Register the MongoDB output parser. - """ - subparser_mongo_output = PusherConfigParsingManager('mongodb') - - subparser_mongo_output.add_argument( - 'u', 'uri', - help_text='MongoDB connection URI', - is_mandatory=True - ) - subparser_mongo_output.add_argument( - 'd', 'db', - help_text='MongoDB database name', - is_mandatory=True - ) - subparser_mongo_output.add_argument( - 'c', 'collection', - help_text='MongoDB collection name', - is_mandatory=True - ) - - self.add_subgroup_parser('output', subparser_mongo_output) - - def _register_prometheus_output_parser(self): - """ - Register the Prometheus output parser. - """ - subparser_prometheus_output = PusherConfigParsingManager('prometheus') - - subparser_prometheus_output.add_argument( - 'u', 'addr', - help_text='Host address the Prometheus HTTP server listens on', - default_value='localhost' - ) - subparser_prometheus_output.add_argument( - 'p', 'port', - help_text='Port number the Prometheus HTTP server listens on', - argument_type=int, - default_value=8000 - ) - subparser_prometheus_output.add_argument( - 'M', 'metric-name', - help_text='Prometheus metric name to expose', - default_value='power_estimation_watts' - ) - subparser_prometheus_output.add_argument( - 'd', 'metric-description', - help_text='Prometheus metric description', - default_value='Estimated power consumption of the target' - ) - subparser_prometheus_output.add_argument( - 't', 'tags', - help_text='Comma-separated list of report metadata fields exposed as metric labels', - argument_type=list - ) - - self.add_subgroup_parser('output', subparser_prometheus_output) - - def _register_csv_output_parser(self): - """ - Register the CSV output parser. - """ - subparser_csv_output = PusherConfigParsingManager('csv') - - subparser_csv_output.add_argument( - 'd', 'directory', - help_text='Directory where CSV output files are written', - is_mandatory=True - ) - - self.add_subgroup_parser('output', subparser_csv_output) - - def _register_json_output_parser(self): - """ - Register the JSON output parser. - """ - subparser_json_output = PusherConfigParsingManager('json') - - subparser_json_output.add_argument( - 'f', 'filepath', - help_text='Path to the JSON output file', - is_mandatory=True - ) - subparser_json_output.add_argument( - 'c', 'compression', - help_text='Output compression format: auto, gzip, lzma, or none', - default_value='auto' - ) - - self.add_subgroup_parser('output', subparser_json_output) - - def _register_influxdb2_output_parser(self): - """ - Register the InfluxDB 2 output parser. - """ - subparser_influx2_output = PusherConfigParsingManager('influxdb2') - - subparser_influx2_output.add_argument( - 'u', 'uri', - help_text='InfluxDB server URI', - is_mandatory=True - ) - subparser_influx2_output.add_argument( - 'k', 'token', - help_text='InfluxDB API token', - is_mandatory=True - ) - subparser_influx2_output.add_argument( - 'g', 'org', - help_text='InfluxDB organization name', - is_mandatory=True - ) - subparser_influx2_output.add_argument( - 'b', 'bucket', - help_text='InfluxDB bucket name', - is_mandatory=True - ) - - self.add_subgroup_parser('output', subparser_influx2_output) - - def _register_clickhouse_output_parser(self): - """ - Register the ClickHouse output parser. - """ - subparser_clickhouse_output = PusherConfigParsingManager('clickhouse') - - subparser_clickhouse_output.add_argument( - 'h', 'host', - help_text='ClickHouse server host', - is_mandatory=True, - ) - subparser_clickhouse_output.add_argument( - 'p', 'port', - help_text='ClickHouse server port', - argument_type=int, - default_value=8123, - ) - subparser_clickhouse_output.add_argument( - 'u', 'username', - help_text='ClickHouse username', - default_value='default', - ) - subparser_clickhouse_output.add_argument( - 'P', 'password', - help_text='ClickHouse password', - default_value='', - ) - subparser_clickhouse_output.add_argument( - 'd', 'database', - help_text='ClickHouse database name', - default_value='default', - ) - - self.add_subgroup_parser('output', subparser_clickhouse_output) - - def _register_pre_processor_parsers(self): - """ - Register all built-in pre-processor parsers. - """ - self._register_k8s_pre_processor_parser() - self._register_openstack_pre_processor_parser() - - def _register_k8s_pre_processor_parser(self): - """ - Register the Kubernetes pre-processor parser. - """ - subparser_k8s_pre_processor = PreProcessorConfigParsingManager('k8s') - - subparser_k8s_pre_processor.add_argument( - 'a', 'api-mode', - help_text='Kubernetes API access mode: local, manual, or cluster', - default_value='cluster' - ) - - subparser_k8s_pre_processor.add_argument( - 'k', 'api-key', - help_text='Kubernetes bearer token for manual API mode', - ) - - subparser_k8s_pre_processor.add_argument( - 'h', 'api-host', - help_text='Kubernetes API host for manual API mode', - ) - - subparser_k8s_pre_processor.add_argument( - 'l', 'labels', - help_text='Comma-separated list of Kubernetes pod labels added to reports as metadata', - argument_type=list - ) - - self.add_subgroup_parser('pre-processor', subparser_k8s_pre_processor) - - def _register_openstack_pre_processor_parser(self): - """ - Register the OpenStack pre-processor parser. - """ - subparser_openstack_pre_processor = PreProcessorConfigParsingManager('openstack') - - subparser_openstack_pre_processor.add_argument( - 'i', "polling-interval", - help_text='OpenStack API polling interval in seconds', - argument_type=float, - default_value=10.0 - ) - - subparser_openstack_pre_processor.add_argument( - 'm', 'metadata', - help_text='Comma-separated list of OpenStack server metadata fields added to reports', - argument_type=list - ) - - self.add_subgroup_parser('pre-processor', subparser_openstack_pre_processor) diff --git a/src/powerapi/cli/config_parser.py b/src/powerapi/cli/config_parser.py deleted file mode 100644 index 8cd976a1..00000000 --- a/src/powerapi/cli/config_parser.py +++ /dev/null @@ -1,778 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import getopt -import json -import os -import sys -from collections.abc import Callable -from typing import Any - -from powerapi.exception import AlreadyAddedArgumentException, UnknownArgException, \ - MissingValueException, BadContextException, TooManyArgumentNamesException, NoNameSpecifiedForSubgroupException, \ - SubgroupAlreadyExistException, SubgroupParserWithoutNameArgumentException, BadTypeException, \ - MissingArgumentException, SameLengthArgumentNamesException, InvalidPrefixException, RepeatedArgumentException, \ - SubgroupDoesNotExistException, AlreadyAddedSubgroupException -from ._utils import find_longest_string_in_list, string_to_bool, to_lower_case_and_replace_separators, \ - get_longest_related_suffix, string_to_list - - -def store_val(argument_name: str, val: Any, configuration: dict, args: list | None = None) -> (list, dict): - """ - Action that stores the value of the argument on the parser result - """ - if val == '': - val = None - - configuration[argument_name] = val - return args, configuration - - -def store_true(argument_name: str, configuration: dict, val: Any = None, args: list | None = None) -> (list, dict): - """ - Action that stores a True boolean value on the parser result - """ - configuration[argument_name] = True - return args, configuration - - -class ConfigurationArgument: - """ - Argument provided by a formula configuration. - """ - - def __init__(self, names: list, is_flag: bool, default_value: Any, help_text: str, argument_type: type, - is_mandatory: bool, action: Callable | None = None): - self.names = names - self.is_flag = is_flag - self.default_value = default_value - self.help_text = help_text - self.type = argument_type - self.is_mandatory = is_mandatory - self.action = action - - def __eq__(self, arg): - names_ok = True - - for current_name in self.names: - names_ok = current_name in arg.names - - return names_ok and len(self.names) == len(arg.names) and self.is_flag == arg.is_flag and \ - self.type == arg.type and self.default_value == arg.default_value and \ - self.help_text == arg.help_text and self.is_mandatory == arg.is_mandatory - - -class BaseConfigParser: - """ - Base class for configuration parsers. - """ - - def __init__(self): - self.arguments = {} - - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False): - """add an optional argument to the parser that will activate an action - - :param str names: names of the optional argument that will be bind to - the action (could be long or short name) - - :param bool is_flag: True if the argument doesn't require to be followed - by a value - - :param Callable action: action that will be executed when the argument is - caught by the parser. the lambda take 4 parameters - (the name of the argument caught by the parser, - the value attached to this argument, the current - list of arguments that is parsed by the parser and - the parser result) and return a list of token and - a parser result(dict) - - :param default_value: the default value attached to this argument - - :param str help_text: text that describe the argument - - :param type argument_type: type of the value that the argument must catch - - :param bool is_mandatory: True if the argument is required - - :raise AlreadyAddedArgumentException: when attempting to add an - argument that already have been - added to this parser - - """ - for name in names: - if name in self.arguments: - raise AlreadyAddedArgumentException(name) - - argument = ConfigurationArgument(names=list(names), is_flag=is_flag, action=action, - default_value=default_value, help_text=help_text, - argument_type=argument_type, is_mandatory=is_mandatory) - - for name in names: - self.arguments[name] = argument - - def _get_default_arguments_values(self): - default_values = {} - for _, argument in self.arguments.items(): - if argument.default_value is not None: - argument_name = find_longest_string_in_list(argument.names) - if argument_name not in default_values: - default_values[argument_name] = argument.default_value - - return default_values - - def _get_default_argument_value(self, argument_name: str) -> dict: - """ - Return a dict containing the default value for the given argument or an empty dict - if it does not have one or if it does not exist. - :param str argument_name : The name of the argument - """ - default_values_dict = {} - if argument_name in self.arguments: - argument = self.arguments[argument_name] - if argument.default_value is not None: - longest_argument_name = find_longest_string_in_list(argument.names) - default_values_dict[longest_argument_name] = argument.default_value - - return default_values_dict - - def _get_arguments_str(self, indent: str) -> str: - already_added_argument = [] - arguments_str_representation = '' - for _, argument in self.arguments.items(): - if argument not in already_added_argument: - arguments_str_representation += indent + ', '.join( - map(lambda x: '-' + x if len(x) == 1 else '--' + x, argument.names)) - arguments_str_representation += ' : ' + argument.help_text + '\n' - already_added_argument.append(argument) - return arguments_str_representation - - def _unknown_argument_behaviour(self, arg_name: str, val: Any, args: list, configuration: dict): - raise NotImplementedError() - - def _parse(self, args: list, configuration: dict) -> (list, dict): - - while args: - arg, val = args.pop(0) - if arg not in self.arguments: - args.insert(0, (arg, val)) - return self._unknown_argument_behaviour(arg, val, args, configuration) - - argument = self.arguments[arg] - - arg_long_name = find_longest_string_in_list(argument.names) - val = cast_argument_value(arg_long_name, val, argument) - - args, configuration = argument.action(argument_name=arg_long_name, val=val, args=args, - configuration=configuration) - - return args, configuration - - def _get_mandatory_arguments(self) -> list: - """ - Return the list of mandatory arguments - """ - mand_args = [] - for _, argument in self.arguments.items(): - if argument.is_mandatory and argument not in mand_args: - mand_args.append(argument) - return mand_args - - def get_arguments(self): - """ Get the parser arguments """ - return self.arguments - - def get_longest_arguments_names(self) -> list: - """ - Return a list with the longest names of the different arguments - """ - long_arguments_names = [] - for _, argument in self.arguments.items(): - longest_name = find_longest_string_in_list(argument.names) - if longest_name not in long_arguments_names: - long_arguments_names.append(longest_name) - - return long_arguments_names - - def validate(self, conf: dict) -> dict: - """ - Check that mandatory arguments are present in the provided configuration. - Check that arguments are not repeated in the provided configuration. - Check that arguments are recognized by the parser. - It also defines default values if any for arguments that are not defined in the configuration - - """ - # Check that all the mandatory arguments are present - mandatory_args = self._get_mandatory_arguments() - for arg in mandatory_args: - is_present = False - for arg_name in arg.names: - if arg_name in conf: - is_present = True - break - if not is_present: - raise MissingArgumentException(str(arg.names)) - - # Define default values for no defined arguments if they are specified - for argument_name, argument_definition in self.arguments.items(): - is_defined = False - for name in argument_definition.names: - if name in conf: - is_defined = True - break - if not is_defined and argument_definition.default_value is not None: - conf[argument_name] = argument_definition.default_value - - # Check that arguments are not repeated and that they exist - present_arguments = [] - for current_argument_name in conf: - if current_argument_name in self.arguments: - argument = self.arguments[current_argument_name] - if argument in present_arguments: - raise RepeatedArgumentException(argument_name=current_argument_name) - present_arguments.append(argument) - elif current_argument_name != 'type': - raise UnknownArgException(argument_name=current_argument_name) - - return self.normalize_configuration(conf=conf) - - def normalize_configuration(self, conf: dict) -> dict: - """ - Return a configuration dict that has all the arguments' names in the long form. - If an argument does not exist, a UnknownArgException is raised - If an argument is repeated, a RepeatedArgumentException is raised - """ - conf_with_long_names = {} - - for current_argument_name in conf: - if current_argument_name not in self.arguments and current_argument_name != 'type': - raise UnknownArgException(current_argument_name) - - longest_argument_name = 'type' - if current_argument_name != 'type': - current_argument = self.arguments[current_argument_name] - longest_argument_name = find_longest_string_in_list(current_argument.names) - if longest_argument_name in conf_with_long_names: - raise RepeatedArgumentException(argument_name=longest_argument_name) - conf_with_long_names[longest_argument_name] = conf[current_argument_name] - - return conf_with_long_names - - def cast_arguments_values(self, arguments: dict) -> dict: - """ - Cast to the argument type the different values in the provided dictionary. - The dictionary only contains values with basic types (string, int...) - :param dict arguments: The dictionary with the values to cast - """ - for argument_name, argument_value in arguments.items(): - - if argument_name != 'type': - casted_value = cast_argument_value(val=argument_value, arg_name=argument_name, - argument=self.arguments[argument_name]) - arguments[argument_name] = casted_value - - return arguments - - -class SubgroupParserGroup: - """ - Group of subgroup parsers stored in a dictionary. Each subgroup has a name - """ - - def __init__(self, group_name: str, help_text: str = '', prefix: str = ''): - """ - Create a group of subgroup parsers - :param str group_name: name of the subgroup - :param str help_text: Help text related to the subgroup - :param str prefix: Prefix related to the group for parsing environment variables - """ - self.group_name = group_name - self.help_text = help_text - self.subparsers = {} - self.prefix = prefix - - def get_prefix(self) -> str: - """ - Return the group's prefix - """ - return self.prefix - - def contains(self, name: str): - """ - Check if the given name belongs to one of the subparser of the grouo - :param str name: Name to look for - """ - return name in self.subparsers - - def add_subgroup_parser(self, name: str, subparser: BaseConfigParser): - """ - Add a subgroup parser to the group - :param str name: Subgroup parser name - :param BaseConfigParser subparser: subparser to be added - """ - self.subparsers[name] = subparser - - def get_subgroup_parser(self, name: str) -> BaseConfigParser: - """ - Return the subgroup parser with the given name - :param str name: Subparser name - """ - return self.subparsers[name] - - def __iter__(self): - return iter(self.subparsers.items()) - - def get_help(self) -> str: - """ - return help string - """ - help_str = self.group_name + ' details :\n' - for subparser_name, subparser in self.subparsers.items(): - help_str += ' --' + self.group_name + ' ' + subparser_name + ':\n' - help_str += subparser.get_help() - help_str += '\n' - - return help_str - - def get_longest_arguments_names(self) -> list: - """ - Return a list of arguments names from the different parsers that are part of the group - """ - arguments_names = [] - for _, subparser in self.subparsers.items(): - arguments_names.extend(subparser.get_longest_arguments_names()) - return list(set(arguments_names)) - - def get_group_name(self) -> str: - """ - Return the name of the group - """ - return self.group_name - - -class SubgroupConfigParser(BaseConfigParser): - """ - A parser for a subgroup - """ - - def __init__(self, name: str): - """ - Create a subgroup parser with the given name - :param str name: Name of the subgroup parser - """ - BaseConfigParser.__init__(self) - self.name = name - - def _unknown_argument_behaviour(self, arg_name: str, val: Any, args: list, - configuration: dict): - return args, configuration - - def parse(self, token_list: list) -> (list, dict): - """ - Parse the given token list until an unknown argument is caught - - :param list token_list: the token list currently parsed - - :return dict: the result of the parsing - - """ - local_result = BaseConfigParser._get_default_argument_value(self, argument_name='name') - if not token_list: - return token_list, local_result - - return self._parse(token_list, local_result) - - def get_help(self) -> str: - """ - return help string - """ - return self._get_arguments_str(' ') - - -class RootConfigParser(BaseConfigParser): - """ - Root configuration parser. - """ - - def __init__(self, help_arg: bool = True, separator_env_vars_names: str = '_', separator_args_names: str = '-'): - """ - :param bool help_arg: if True, add a -h/--help argument that display help - :param str separator_env_vars_names: separator for the environment variables names - :param str separator_args_names: separator for arguments with composed names - """ - BaseConfigParser.__init__(self) - self.short_arg = '' - self.long_arg = [] - self.subgroup_parsers = {} - - self.arguments_prefix = [] - self.default_separator_env_vars_names = separator_env_vars_names - self.default_separator_args_names = separator_args_names - - self.help_arg = help_arg - if help_arg: - self.add_argument('h', 'help', is_flag=True, argument_type=bool) - - def get_help(self): - """ - return help string - """ - s = 'main arguments:\n' - s += self._get_arguments_str(' ') - s += '\n' - - for _, subparser_group in self.subgroup_parsers.items(): - s += subparser_group.get_help() - - return s - - def parse(self, args: list) -> dict: - """ - :param list args: list that contains the arguments and their values - - :return dict: Dictionary that contains the arguments with its associated values extracted from args - - :raise UnknownArgException: when the parser catch an argument that - this parser can't handle - - :raise BadContextException: when an argument that the parser can't - handle in the current context is caught - - :raise MissingValueException: when an argument that require a value is - caught without its value - - :raise BadTypeException: when an argument is parsed with a value of an - incorrect type - """ - try: - args, _ = getopt.getopt(args, self.short_arg, self.long_arg) - except getopt.GetoptError as exn: - if 'recognized' in exn.msg: - raise UnknownArgException(exn.opt) from exn - if 'requires' in exn.msg: - raise MissingValueException(exn.opt) from exn - - # Remove `-` and `--` prefix in argument name (`--test-arg` to `test-arg`) - args = [(arg[0].lstrip('-'), arg[1]) for arg in args] - - # verify if help argument exists in args - if self.help_arg: - for arg_name, _ in args: - if arg_name in ('h', 'help'): - print(self.get_help()) - sys.exit(0) - - configuration = {} - - args, configuration = self._parse(args, configuration) - - return configuration - - def parse_config_dict(self, file_name: str) -> dict: - """ - Return a configuration dict that has all the arguments' names in the long form. - If an argument does not exist, a UnknownArgException is raised - """ - with open(file_name, encoding='utf-8') as config_file: - conf = json.load(config_file) - return self.normalize_configuration(conf=conf) - - def normalize_configuration(self, conf: dict) -> dict: - """ - Return a configuration dict that has all the arguments' names in the long form. - If an argument does not exist, a UnknownArgException is raised - """ - # We normalize the simple arguments names - conf = BaseConfigParser.normalize_configuration(self, conf=conf) - - # We normalize the groups arguments names - - for argument_name, argument_value in conf.items(): - if isinstance(argument_value, dict): - for group_name, group in argument_value.items(): - group = self.subgroup_parsers[argument_name].get_subgroup_parser(name=group['type']). \ - normalize_configuration(conf=group) - argument_value[group_name] = group - - return conf - - def _unknown_argument_behaviour(self, arg_name: str, val: Any, args: list, - configuration: dict): - good_contexts = [] - for main_arg_name, subparser_group in self.subgroup_parsers.items(): - for subparser_name, subparser in subparser_group: - if arg_name in subparser.arguments: - good_contexts.append((main_arg_name, subparser_name)) - raise BadContextException(arg_name, good_contexts) - - def _add_argument_names(self, names: list, is_flag: bool): - - if len(names) > 2: - raise TooManyArgumentNamesException(names[2]) - - if len(names) > 1 and len(names[0]) == len(names[1]): - raise SameLengthArgumentNamesException(names[1]) - - def add_suffix_to_argument_name_if_required(current_name): - if len(current_name) == 1: - return current_name + ('' if is_flag else ':') - return current_name + ('' if is_flag else '=') - - for name in names: - if len(name) == 1: - self.short_arg += add_suffix_to_argument_name_if_required(name) - else: - self.long_arg.append(add_suffix_to_argument_name_if_required(name)) - - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False): - self._add_argument_names(list(names), is_flag) - BaseConfigParser.add_argument(self, *names, is_flag=is_flag, action=action, default_value=default_value, - help_text=help_text, argument_type=argument_type, is_mandatory=is_mandatory) - - def add_subgroup_parser(self, subgroup_type: str, subgroup_parser: SubgroupConfigParser): - """ - Add a subparser that will be used by the argument *group_name* - The group must contain a name action - :param str subgroup_type: the group type - :param SubgroupConfigParser subgroup_parser: The subgroup parser - :param str help_text: help text related to the parser - - :raise AlreadyAddedArgumentException: when attempting to add an - argument that already have been - added to this parser - :raise SubgroupDoesNotExistException If the group related to the paser does not exist - """ - - if 'name' not in subgroup_parser.arguments: - raise SubgroupParserWithoutNameArgumentException() - if subgroup_type not in self.subgroup_parsers: - raise SubgroupDoesNotExistException(argument_name=subgroup_type) - if self.subgroup_parsers[subgroup_type].contains(subgroup_parser.name): - raise AlreadyAddedArgumentException(subgroup_parser.name) - - self.subgroup_parsers[subgroup_type].add_subgroup_parser(subgroup_parser.name, subgroup_parser) - - for action_name, action in subgroup_parser.arguments.items(): - self._add_argument_names([action_name], action.is_flag) - - def add_subgroup(self, subgroup_type: str, help_text: str = '', prefix: str = ''): - """ - Add a subgroup that will be used by the argument *group_name* - The group must contain a name action - :param str subgroup_type: the subgroup type - :param str help_text: help text related to the subgroup - :param str prefix: the prefix related to the subgroup - - :raise AlreadyAddedSubgroupException is the subgroup already exists - """ - - def _action(argument_name: str, val: Any, args: list, configuration: dict): - if argument_name not in configuration: - configuration[argument_name] = {} - - parser = self.subgroup_parsers[argument_name].get_subgroup_parser(val) - args, parse_result = parser.parse(args) - - if 'name' not in parse_result: - raise NoNameSpecifiedForSubgroupException(subgroup_type) - - subgroup_name = parse_result['name'] - del parse_result['name'] - - if subgroup_name in configuration[argument_name]: - raise SubgroupAlreadyExistException(subgroup_name) - - configuration[argument_name][subgroup_name] = parse_result - configuration[argument_name][subgroup_name]['type'] = parser.name - - return args, configuration - - if subgroup_type not in self.subgroup_parsers: - self.subgroup_parsers[subgroup_type] = SubgroupParserGroup(subgroup_type, help_text=help_text, - prefix=prefix) - self.add_argument(subgroup_type, action=_action, help_text=help_text) - - else: - raise AlreadyAddedSubgroupException(subgroup_type) - - def add_argument_prefix(self, argument_prefix: str): - """ - Add a simple argument prefix to the list if argument_prefix is no prefix of an existing argument prefix or - vice-versa. Otherwise, it raises an InvalidPrefixException - :param argument_prefix: a new argument prefix to be added - """ - for existing_argument_prefix in self.arguments_prefix: - if argument_prefix.startswith(existing_argument_prefix) or \ - existing_argument_prefix.startswith(argument_prefix): - raise InvalidPrefixException(existing_prefix=existing_argument_prefix, new_prefix=argument_prefix) - - self.arguments_prefix.append(argument_prefix) - - def parse_config_environment_variables(self) -> dict: - """ - Parse environment variables to extract a configuration. Then merges the extracted configution with the one - provided as parameter - :param dict current_conf: Configuration to execute the merge - """ - - conf = {} - - for current_environment_var_prefix in self.arguments_prefix: - conf = self._extract_simple_environment_variables_with_prefix( - simple_variables_prefix=current_environment_var_prefix, - groups_variables_prefix=self.get_groups_prefixes()) - # We normalize the arguments names - conf = self.normalize_configuration(conf=conf) - - # We cast every value in conf to the correct type - conf = self.cast_arguments_values(arguments=conf) - - groups_names = [] - for _, group in self.subgroup_parsers.items(): - - group_name = group.get_group_name() - group_conf = self._extract_group_environment_variables(group=group) - if len(group_conf) > 0: - conf[group_name] = group_conf - groups_names.append(group_name) - - for group_name in groups_names: - for subgroup_name, subgroup_arguments in conf[group_name].items(): - # We normalize the names in each group. The argument type for each subgroup - # is required - if 'type' not in subgroup_arguments: - raise MissingArgumentException(argument_name=group_name + '>' + subgroup_name + '>' + 'type') - subgroup_parser = self.subgroup_parsers[group_name].get_subgroup_parser(name=subgroup_arguments['type']) - subgroup_arguments = subgroup_parser.normalize_configuration(conf=subgroup_arguments) - - # We cast every value in subgroup_arguments to the correct type - subgroup_arguments = subgroup_parser.cast_arguments_values(arguments=subgroup_arguments) - conf[group_name][subgroup_name] = subgroup_arguments - - return conf - - def get_groups_prefixes(self) -> list: - """ - Return a list with the prefixes of different groups - """ - prefixes = [] - for _, subgroup in self.subgroup_parsers.items(): - prefixes.append(subgroup.get_prefix()) - - return prefixes - - def _extract_group_name_from_prefix(self, prefix: str) -> str: - """ - Extract the group name from the given prefix. It assumes the words in the prefix - are separated by self.default_separator_env_vars_names and the group name is composed by a single word - - """ - return prefix.split(self.default_separator_env_vars_names)[1].lower() # [, , ''] - - def _extract_simple_environment_variables_with_prefix(self, simple_variables_prefix: str, - groups_variables_prefix: list) -> dict: - """ - Extract from environment variables the ones starting with prefix and that do not belong to groups. - The returned dictionary contains the variables names as keys without prefix and in lower case - :param str simple_variables_prefix: Prefix to extract the simple environment variables - :param list groups_variables_prefix: List of group prefix for identifying simple variables - """ - simple_variables_with_prefix = {} - for var_name in os.environ: - is_group_variable = False - for group_variable_prefix in groups_variables_prefix: - if var_name.startswith(group_variable_prefix): - is_group_variable = True - break - if not is_group_variable and var_name.startswith(simple_variables_prefix): - var_name_without_prefix = var_name[len(simple_variables_prefix) - len(var_name):]. \ - lower().replace(self.default_separator_env_vars_names, self.default_separator_args_names) - simple_variables_with_prefix[var_name_without_prefix] = os.environ[var_name] - return simple_variables_with_prefix - - def _extract_group_environment_variables(self, group: SubgroupParserGroup) -> dict: - """ - Extract from environment variables the ones starting with group_prefix. - The returned dictionary contains the variables names as keys without prefix and in lower case - :param group: The subgroup related to the environment variables to be extracted - """ - group_variables_with_prefix = {} - group_prefix = group.get_prefix() - subgroups_arguments_names = group.get_longest_arguments_names() - subgroups_arguments_names.append('type') - subgroups_arguments_names = to_lower_case_and_replace_separators(subgroups_arguments_names, - self.default_separator_env_vars_names, - self.default_separator_args_names) - for environ_var_name in os.environ: - if environ_var_name.startswith(group_prefix): - # We remove the prefix and put the name in lower case - suffix_environ_var_name = environ_var_name[len(group_prefix) - len(environ_var_name):]. \ - lower().replace(self.default_separator_env_vars_names, self.default_separator_args_names) - - # We look for a related argument name in the subgroup - group_variable_name_lower_case = get_longest_related_suffix(suffix_environ_var_name, - subgroups_arguments_names) - - if group_variable_name_lower_case and \ - suffix_environ_var_name[ - suffix_environ_var_name.rfind(group_variable_name_lower_case) - 1] == \ - self.default_separator_args_names: - - # The subgroup's name is at the beginning of the suffix - subgroup_name = suffix_environ_var_name[0: - suffix_environ_var_name.rfind( - group_variable_name_lower_case) - 1] - - if subgroup_name not in group_variables_with_prefix: - group_variables_with_prefix[subgroup_name] = {} - group_variables_with_prefix[subgroup_name][group_variable_name_lower_case] = os.environ[ - environ_var_name] - - return group_variables_with_prefix - - -def cast_argument_value(arg_name: str, val: Any, argument: ConfigurationArgument): - """ - Cast the given value to argument.type - :param str arg_name: The argument name - :param Any val: Current value given to the argument - :param ConfigurationAgument argument: The argument definition - """ - try: - if argument.type is bool and val is not None and isinstance(val, str): - return string_to_bool(val) - - if argument.type is list and val is not None and isinstance(val, str): - return string_to_list(val) - - return argument.type(val) - except ValueError as exn: - raise BadTypeException(arg_name, argument.type) from exn diff --git a/src/powerapi/cli/config_validator.py b/src/powerapi/cli/config_validator.py deleted file mode 100644 index dfdd0869..00000000 --- a/src/powerapi/cli/config_validator.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import logging -import os - -from powerapi.exception import MissingArgumentException, NotAllowedArgumentValueException, FileDoesNotExistException, \ - UnexistingActorException - - -class ConfigValidator: - """ - Validate powerapi config and initialize missing default values - """ - - @staticmethod - def validate(config: dict): - """ - Validate powerapi config and initialize missing default values - """ - if 'verbose' not in config: - config['verbose'] = logging.NOTSET - if 'stream' not in config: - config['stream'] = False - if 'output' not in config: - logging.error("no output configuration found") - raise MissingArgumentException(argument_name='output') - - if 'input' not in config: - logging.error("no input configuration found") - raise MissingArgumentException(argument_name='input') - - for input_id in config['input']: - input_config = config['input'][input_id] - if input_config['type'] == 'csv' \ - and ( - 'files' not in input_config or input_config['files'] is None or len(input_config['files']) == 0): - logging.error("no files parameter found for csv input") - raise MissingArgumentException(argument_name='files') - - if input_config['type'] == 'csv' and config['stream']: - logging.error("stream mode cannot be used for csv input") - raise NotAllowedArgumentValueException("Stream mode cannot be used for csv input") - - if 'pre-processor' in config: - for pre_processor_id in config['pre-processor']: - pre_processor_config = config['pre-processor'][pre_processor_id] - - if 'puller' not in pre_processor_config: - logging.error("No puller name found for pre-processor: %s ", pre_processor_id) - raise MissingArgumentException(argument_name='puller') - - puller_id = pre_processor_config['puller'] - - if puller_id not in config['input']: - logging.error("Puller actor '%s' does not exist", puller_id) - raise UnexistingActorException(actor=puller_id) - - elif 'post-processor' in config: - for post_processor_id in config['post-processor']: - post_processor_config = config['post-processor'][post_processor_id] - - if 'pusher' not in post_processor_config: - logging.error("No pusher name found for post-processor: %s", post_processor_id) - raise MissingArgumentException(argument_name='pusher') - - pusher_id = post_processor_config['pusher'] - - if pusher_id not in config['output']: - logging.error("Pusher actor '%s' does not exist", pusher_id) - raise UnexistingActorException(actor=pusher_id) - - ConfigValidator._validate_input(config) - - @staticmethod - def _validate_input(config: dict): - """ - Check that csv input type has files that exist - """ - for input_config in config['input'].values(): - if input_config['type'] == 'csv': - for file_name in input_config['files']: - if not os.access(file_name, os.R_OK): - raise FileDoesNotExistException(file_name=file_name) - - @staticmethod - def _validate_binding(config: dict): - """ - Check that defined bindings use existing actors defined by the configuration - """ - for _, binding_infos in config['binding'].items(): - - if 'from' not in binding_infos: - logging.error("no from parameter found for binding") - raise MissingArgumentException(argument_name='from') - - if 'to' not in binding_infos: - logging.error("no to parameter found for binding") - raise MissingArgumentException(argument_name='to') - - # from_info[0] is the subgroup and from_info[1] the actor name - from_infos = binding_infos['from'].split('.') - - if from_infos[0] not in config or from_infos[1] not in config[from_infos[0]]: - logging.error("from actor does not exist") - raise UnexistingActorException(actor=binding_infos['from']) - - # to_info[0] is the subgroup and to_info[1] the actor name - to_infos = binding_infos['to'].split('.') - - if to_infos[0] not in config or to_infos[1] not in config[to_infos[0]]: - logging.error("to actor does not exist") - raise UnexistingActorException(actor=binding_infos['to']) diff --git a/src/powerapi/cli/generator.py b/src/powerapi/cli/generator.py deleted file mode 100644 index b9d91b8e..00000000 --- a/src/powerapi/cli/generator.py +++ /dev/null @@ -1,468 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import logging -from collections.abc import Callable - -from powerapi.actor import Actor, ActorProxy -from powerapi.database.driver import ReadableDatabaseFactory, WritableDatabaseFactory -from powerapi.exception import PowerAPIException, ModelNameAlreadyUsed, DatabaseNameDoesNotExist, ModelNameDoesNotExist, \ - DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed -from powerapi.filter import ReportFilter -from powerapi.processor.processor_actor import ProcessorActor -from powerapi.puller import PullerActor -from powerapi.pusher import PusherActor -from powerapi.report import HWPCReport, PowerReport, Report, FormulaReport -from powerapi.utils.metadata import build_metadata_mapping - -COMPONENT_TYPE_KEY = 'type' -COMPONENT_MODEL_KEY = 'model' -COMPONENT_DB_NAME_KEY = 'db' -COMPONENT_DB_COLLECTION_KEY = 'collection' -COMPONENT_DB_MANAGER_KEY = 'db_manager' -COMPONENT_DB_MAX_BUFFER_SIZE_KEY = 'max_buffer_size' -COMPONENT_URI_KEY = 'uri' - -ACTOR_NAME_KEY = 'actor_name' -REGEXP_KEY = 'regexp' - -K8S_API_MODE_KEY = 'api-mode' -K8S_API_KEY_KEY = 'api-key' -K8S_API_HOST_KEY = 'api-host' - -LISTENER_ACTOR_KEY = 'listener_actor' - -GENERAL_CONF_STREAM_MODE_KEY = 'stream' -GENERAL_CONF_VERBOSE_KEY = 'verbose' - - -class Generator: - """ - Generate an actor class and actor start message from config dict. - The config dict has the following structure: - { - "arg1_key": value, - "arg2_key": value - ... - "component_group_name1":{ - "arg1_cpn1_key": value, - "arg2_cpn1_key": value, - ... - - } - component_group_name2:{ - ... - } - ... - } - """ - - def __init__(self, component_group_name): - self.component_group_name = component_group_name - - def generate(self, main_config: dict) -> dict[str, Actor]: - """ - Generate an actor class and actor start message from config dict - """ - if self.component_group_name not in main_config: - raise PowerAPIException(f'Configuration error : Component {self.component_group_name} group is unknown') - - actors = {} - for component_name, component_config in main_config[self.component_group_name].items(): - try: - actors[component_name] = self._gen_actor(component_config, main_config, component_name) - except KeyError as exn: - raise PowerAPIException(f'Configuration error: Missing "{exn.args[0]}" argument for {component_name} component') from exn - except ValueError as exn: - raise PowerAPIException(f'Configuration error: Invalid parameter for {component_name} component: {exn.args[0]}') from exn - - return actors - - def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> Actor: - raise NotImplementedError() - - -class BaseGenerator(Generator): - """ - Generate an Actor and Start message from config - """ - - def __init__(self, component_group_name: str): - Generator.__init__(self, component_group_name) - self.report_classes: dict[str, type[Report]] = { - 'HWPCReport': HWPCReport, - 'PowerReport': PowerReport, - 'FormulaReport': FormulaReport, - } - - def _gen_actor(self, component_config: dict, main_config: dict, component_name: str): - model = self._get_report_class(component_config[COMPONENT_MODEL_KEY], component_config) - component_config[COMPONENT_MODEL_KEY] = model - - actor = self._actor_factory(component_name, main_config, component_config) - return actor - - def _get_report_class(self, model_name: str, component_config: dict): - if model_name not in self.report_classes: - raise PowerAPIException(f'Configuration error: model type {model_name} unknown') - - return self.report_classes[component_config[COMPONENT_MODEL_KEY]] - - def _actor_factory(self, actor_name: str, main_config: dict, component_config: dict): - raise NotImplementedError - - -class DBActorGenerator(BaseGenerator): - """ - ActorGenerator that initialise the start message with a database from config - """ - - def __init__(self, component_group_name: str): - super().__init__(component_group_name) - - self.db_factory: dict[str, Callable[[dict], ReadableDatabaseFactory | WritableDatabaseFactory]] = {} - - def remove_report_class(self, model_name: str): - """ - remove a Model from generator - """ - if model_name not in self.report_classes: - raise ModelNameDoesNotExist(model_name) - - del self.report_classes[model_name] - - def remove_db_factory(self, database_name: str): - """ - remove a database from generator - """ - if database_name not in self.db_factory: - raise DatabaseNameDoesNotExist(database_name) - - del self.db_factory[database_name] - - def add_report_class(self, model_name: str, report_class: type[Report]): - """ - add a report class to generator - """ - if model_name in self.report_classes: - raise ModelNameAlreadyUsed(model_name) - - self.report_classes[model_name] = report_class - - def add_db_factory(self, db_name: str, db_factory_function: Callable[[dict], ReadableDatabaseFactory | WritableDatabaseFactory]): - """ - add a database to generator - """ - if db_name in self.db_factory: - raise DatabaseNameAlreadyUsed(db_name) - - self.db_factory[db_name] = db_factory_function - - def _generate_db(self, db_name: str, component_config: dict): - try: - return self.db_factory[db_name](component_config) - except KeyError as exn: - raise PowerAPIException('Configuration error: Invalid database type: %s', db_name) from exn - except ImportError as exn: - raise PowerAPIException('Dependencies for %s database are not installed', db_name) from exn - - def _gen_actor(self, component_config: dict, main_config: dict, component_name: str): - model = self._get_report_class(component_config[COMPONENT_MODEL_KEY], component_config) - component_config[COMPONENT_MODEL_KEY] = model - database_manager = self._generate_db(component_config[COMPONENT_TYPE_KEY], component_config) - component_config[COMPONENT_DB_MANAGER_KEY] = database_manager - - actor = self._actor_factory(component_name, main_config, component_config) - return actor - - -class PullerGenerator(DBActorGenerator): - """ - Generate Puller Actor class and Puller start message from config - """ - - @staticmethod - def _csv_input_database_factory(conf: dict) -> ReadableDatabaseFactory: - """ - CSV Input database factory method. - """ - from powerapi.database.csv.driver import CSVInputFactory - return CSVInputFactory(conf['model'], conf['files']) - - @staticmethod - def _json_input_database_factory(conf: dict) -> ReadableDatabaseFactory: - """ - JSON Input database factory method. - """ - from powerapi.database.json.driver import JsonInputFactory - return JsonInputFactory(conf['model'], conf['filepath'], conf['compression']) - - @staticmethod - def _socket_database_factory(conf: dict) -> ReadableDatabaseFactory: - """ - Socket Input database factory method. - """ - from powerapi.database.socket.driver import SocketInputFactory - return SocketInputFactory(conf['model'], conf['host'], conf['port']) - - @staticmethod - def _mongodb_database_factory(conf: dict) -> ReadableDatabaseFactory: - """ - MongoDB Input database factory method. - """ - from powerapi.database.mongodb.driver import MongodbInputFactory - return MongodbInputFactory(conf['model'], conf['uri'], conf['db'], conf['collection']) - - def __init__(self, report_filter: ReportFilter): - """ - :param report_filter: Report filter to apply for incoming reports - """ - super().__init__('input') - - self.report_filter = report_filter - - self.add_db_factory('csv', self._csv_input_database_factory) - self.add_db_factory('json', self._json_input_database_factory) - self.add_db_factory('socket', self._socket_database_factory) - self.add_db_factory('mongodb', self._mongodb_database_factory) - - def _actor_factory(self, actor_name: str, main_config, component_config: dict) -> PullerActor: - """ - Actor factory method. - :param actor_name: Name of the actor - :param main_config: Global configuration - :param component_config: Actor configuration - :return: Configured Puller actor - """ - database = component_config[COMPONENT_DB_MANAGER_KEY] - stream_mode = main_config[GENERAL_CONF_STREAM_MODE_KEY] - logging_level = logging.DEBUG if main_config[GENERAL_CONF_VERBOSE_KEY] else logging.WARNING - return PullerActor(actor_name, database, self.report_filter, stream_mode, level_logger=logging_level) - - -class PusherGenerator(DBActorGenerator): - """ - Generate Pusher actor and Pusher start message from config - """ - - @staticmethod - def _csv_output_database_factory(conf: dict) -> WritableDatabaseFactory: - """ - CSV Output database factory method. - """ - from powerapi.database.csv.driver import CSVOutputFactory - return CSVOutputFactory(conf['model'], conf['directory']) - - @staticmethod - def _json_output_database_factory(conf: dict) -> WritableDatabaseFactory: - """ - JSON Output database factory method. - """ - from powerapi.database.json.driver import JsonOutputFactory - return JsonOutputFactory(conf['model'], conf['filepath'], conf['compression']) - - @staticmethod - def _mongodb_database_factory(conf: dict) -> WritableDatabaseFactory: - """ - MongoDB Output database factory method. - """ - from powerapi.database.mongodb.driver import MongodbOutputFactory - return MongodbOutputFactory(conf['model'], conf['uri'], conf['db'], conf['collection']) - - @staticmethod - def _influxdb2_database_factory(conf: dict) -> WritableDatabaseFactory: - """ - InfluxDB2 database factory method. - """ - from powerapi.database.influxdb2.driver import InfluxDB2OutputFactory - return InfluxDB2OutputFactory(conf['model'], conf['uri'], conf['org'], conf['bucket'], conf['token']) - - @staticmethod - def _prometheus_database_factory(conf: dict) -> WritableDatabaseFactory: - """ - Prometheus database factory method. - """ - from powerapi.database.prometheus.driver import PrometheusOutputFactory - return PrometheusOutputFactory(conf['model'], conf['addr'], conf['port'], conf.get('tags', [])) - - @staticmethod - def _clickhouse_database_factory(conf: dict) -> WritableDatabaseFactory: - """ - ClickHouse output database factory method. - """ - from powerapi.database.clickhouse.driver import ClickHouseOutputFactory - return ClickHouseOutputFactory(conf['model'], conf['host'], conf['port'], conf['username'], conf['password'], conf['database']) - - def __init__(self): - super().__init__('output') - - self.add_db_factory('csv', self._csv_output_database_factory) - self.add_db_factory('json', self._json_output_database_factory) - self.add_db_factory('mongodb', self._mongodb_database_factory) - self.add_db_factory('influxdb2', self._influxdb2_database_factory) - self.add_db_factory('prometheus', self._prometheus_database_factory) - self.add_db_factory('clickhouse', self._clickhouse_database_factory) - - def _actor_factory(self, actor_name: str, main_config: dict, component_config: dict) -> PusherActor: - """ - Actor factory method. - :param actor_name: Name of the actor - :param main_config: Global configuration - :param component_config: Actor configuration - :return: Configured Pusher actor - """ - database = component_config[COMPONENT_DB_MANAGER_KEY] - level_logger = logging.DEBUG if main_config[GENERAL_CONF_VERBOSE_KEY] else logging.WARNING - return PusherActor(actor_name, database, logger_level=level_logger) - - def generate_report_mapping(self, main_config: dict, actors: dict[str, Actor]) -> dict[type[Report], list[ActorProxy]]: - """ - Generate the report type to pusher actor mapping. - :param main_config: Main configuration - :param actors: Dictionary of actors (result of the `generate` method) - :return: Dictionary mapping the report type to actors that should process it - """ - if self.component_group_name not in main_config: - raise PowerAPIException(f'Configuration error: Component "{self.component_group_name}" is not defined') - - report_type_to_actor = {} - for component_name, component_config in main_config[self.component_group_name].items(): - try: - actor_proxy = actors[component_name].get_proxy() - report_type_to_actor.setdefault(component_config[COMPONENT_MODEL_KEY], []).append(actor_proxy) - except KeyError as exn: - raise PowerAPIException(f'Actor "{component_name}" is not defined') from exn - - return report_type_to_actor - - -class ProcessorGenerator(Generator): - """ - Generator that initializes the processor actor(s) from the configuration. - """ - - def __init__(self, component_group_name: str): - """ - :param component_group_name: Name of the component group - """ - super().__init__(component_group_name) - - self.processor_factory: dict[str, Callable[[dict], ProcessorActor]] = {} - - def remove_processor_factory(self, processor_type: str) -> None: - """ - Remove the given processor actor factory from the generator. - :param processor_type: Processor type name - """ - if processor_type not in self.processor_factory: - raise ProcessorTypeDoesNotExist(processor_type) - - del self.processor_factory[processor_type] - - def add_processor_factory(self, processor_type: str, processor_factory_function: Callable) -> None: - """ - Add the given processor actor factory to the generator. - :param processor_type: Processor type name - :param processor_factory_function: Factory method used to generate the processor actors - """ - if processor_type in self.processor_factory: - raise ProcessorTypeAlreadyUsed(processor_type) - - self.processor_factory[processor_type] = processor_factory_function - - def _generate_processor(self, processor_name: str, component_config: dict) -> ProcessorActor: - try: - return self.processor_factory[processor_name](component_config) - except KeyError as exn: - raise PowerAPIException('Configuration error: Invalid processor type: %s', processor_name) from exn - except ImportError as exn: - raise PowerAPIException('Dependencies for %s processor are not installed', processor_name) from exn - - def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> ProcessorActor: - """ - Helper method to generate a processor actor from the given configuration. - :param component_config: Configuration of the processor component - :param main_config: Global configuration - :param component_name: Name of the processor actor to generate - :return: Processor actor - """ - processor_actor_type = component_config[COMPONENT_TYPE_KEY] - component_config[ACTOR_NAME_KEY] = component_name - component_config[GENERAL_CONF_VERBOSE_KEY] = main_config[GENERAL_CONF_VERBOSE_KEY] - return self._generate_processor(processor_actor_type, component_config) - - -class PreProcessorGenerator(ProcessorGenerator): - """ - Generator that initializes the pre-processor actor(s) from the configuration. - """ - - def __init__(self): - super().__init__('pre-processor') - - self.add_processor_factory('k8s', self._k8s_pre_processor_factory) - self.add_processor_factory('openstack', self._openstack_pre_processor_factory) - - @staticmethod - def _k8s_pre_processor_factory(processor_config: dict) -> ProcessorActor: - """ - Kubernetes pre-processor actor factory. - :param processor_config: Pre-Processor configuration - :return: Configured Kubernetes pre-processor actor - """ - from powerapi.processor.pre.k8s.actor import KubernetesPreProcessorActor - from powerapi.processor.pre.k8s.monitor_agent import KubernetesMonitorConfig - - api_mode = processor_config[K8S_API_MODE_KEY] - api_host = processor_config.get(K8S_API_HOST_KEY, None) - api_key = processor_config.get(K8S_API_KEY_KEY, None) - label_mapping = build_metadata_mapping(processor_config.get('labels', []), prefix='k8s_pod_label_') - monitor_config = KubernetesMonitorConfig(api_mode, api_host, api_key, label_mapping) - - name = processor_config[ACTOR_NAME_KEY] - level_logger = logging.DEBUG if processor_config[GENERAL_CONF_VERBOSE_KEY] else logging.INFO - return KubernetesPreProcessorActor(name, monitor_config, level_logger) - - @staticmethod - def _openstack_pre_processor_factory(processor_config: dict) -> ProcessorActor: - """ - OpenStack pre-processor actor factory. - :param processor_config: Pre-Processor configuration - :return: Configured OpenStack pre-processor actor - """ - from powerapi.processor.pre.openstack.actor import OpenStackPreProcessorActor - from powerapi.processor.pre.openstack.monitor_agent import OpenStackMonitorConfig - - api_polling_interval = processor_config['polling-interval'] - metadata_mapping = build_metadata_mapping(processor_config.get('metadata', []), prefix='openstack_metadata_') - monitor_config = OpenStackMonitorConfig(api_polling_interval, metadata_mapping) - - name = processor_config[ACTOR_NAME_KEY] - level_logger = logging.DEBUG if processor_config[GENERAL_CONF_VERBOSE_KEY] else logging.INFO - return OpenStackPreProcessorActor(name, monitor_config, level_logger) diff --git a/src/powerapi/cli/parsing_manager.py b/src/powerapi/cli/parsing_manager.py deleted file mode 100644 index c2a70acd..00000000 --- a/src/powerapi/cli/parsing_manager.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import json -import logging -import sys -from collections.abc import Callable -from typing import Any - -from powerapi.cli.config_parser import RootConfigParser, SubgroupConfigParser, store_val -from powerapi.exception import MissingArgumentException, BadTypeException, AlreadyAddedSubparserException, \ - UnknownArgException, MissingValueException, BadContextException, RepeatedArgumentException, \ - AlreadyAddedSubgroupException -from ._utils import merge_dictionaries - - -class BaseConfigParsingManagerInterface: - """ - Abstract class for dealing with parsing of configurations. - """ - - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False) -> None: - """ - Add an argument to the parser. - """ - raise NotImplementedError - - def validate(self, conf: dict) -> dict: - """ - Validate the parsed configuration. - """ - raise NotImplementedError - - -class SubgroupConfigParsingManager(BaseConfigParsingManagerInterface): - """ - Sub Parser for MainConfigParser - """ - - def __init__(self, name: str): - self.subparser = {} - self.name = name - self.cli_parser = SubgroupConfigParser(name) - - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False) -> None: - """ - Add an argument to the parser. - """ - self.cli_parser.add_argument(*names, is_flag=is_flag, action=action, default_value=default_value, - help_text=help_text, argument_type=bool if is_flag else argument_type, - is_mandatory=is_mandatory) - - def validate(self, conf: dict) -> dict: - """ - Check the parsed configuration. - """ - - # check types - for args, value in conf.items(): - for _, waited_value in self.cli_parser.get_arguments().items(): - if args in waited_value.names: - # check type - if not isinstance(value, waited_value.type) and not waited_value.is_flag: - raise BadTypeException(args, waited_value.type) - - # Check that all the mandatory arguments are present - conf = self.cli_parser.validate(conf=conf) - - return conf - - -class RootConfigParsingManager(BaseConfigParsingManagerInterface): - """ - Parser abstraction for the configuration - """ - - def __init__(self): - self.subparser = {} - self.cli_parser = RootConfigParser() - - def add_argument_prefix(self, argument_prefix: str): - """ - Add a simple argument prefix to the cli_parser - :param argument_prefix: a new argument prefix to be added - """ - self.cli_parser.add_argument_prefix(argument_prefix=argument_prefix) - - def add_subgroup(self, name: str, help_text: str = '', prefix: str = ''): - """ - Add a group to the cli_parser - :param name: the group's name - :param help_text: a help text for the subgroup - :param prefix: a prefix related to the subgroup - """ - try: - self.cli_parser.add_subgroup(subgroup_type=name, help_text=help_text, prefix=prefix) - except AlreadyAddedSubgroupException as exn: - logging.error("Configuration error: %s", exn.msg) - sys.exit(-1) - - def add_subgroup_parser(self, subgroup_name: str, subgroup_parser: SubgroupConfigParsingManager): - - """ - Add a Subgroup Parser to call when is encountered - When name is encountered, the subgroup parser such as subgroup_parser.name match conf[name].type - """ - if subgroup_name in self.subparser: - if subgroup_parser.name in list(self.subparser[subgroup_name]): - raise AlreadyAddedSubparserException(subgroup_name) - else: - self.subparser[subgroup_name] = {} - - self.subparser[subgroup_name][subgroup_parser.name] = subgroup_parser - - self.cli_parser.add_subgroup_parser(subgroup_type=subgroup_name, subgroup_parser=subgroup_parser.cli_parser) - - def _parse_cli(self, cli_line: list) -> dict: - return self.cli_parser.parse(cli_line) - - def _parse_config_from_json_file(self, file_name: str, current_conf: dict) -> dict: - return merge_dictionaries(current_conf, self.cli_parser.parse_config_dict(file_name)) - - def _parse_config_from_environment_variables(self, current_conf: dict) -> dict: - return merge_dictionaries(current_conf, self.cli_parser.parse_config_environment_variables()) - - def add_argument(self, *names, is_flag: bool = False, action: Callable = store_val, default_value: Any = None, - help_text: str = '', argument_type: type = str, is_mandatory: bool = False) -> None: - """ - Add an argument to the parser. - """ - self.cli_parser.add_argument(*names, is_flag=is_flag, action=action, default_value=default_value, - help_text=help_text, argument_type=bool if is_flag else argument_type, - is_mandatory=is_mandatory) - - def validate(self, conf: dict) -> dict: - """ - Check the parsed configuration - """ - - # check types - for current_argument_name, current_argument_value in conf.items(): - is_an_arg = False - if current_argument_name in self.subparser: - for _, dic_value in current_argument_value.items(): - self.subparser[current_argument_name][dic_value["type"]].validate(dic_value) - is_an_arg = True - - if not is_an_arg: - for _, argument_definition in self.cli_parser.get_arguments().items(): - if current_argument_name in argument_definition.names: - is_an_arg = True - # check type - if not isinstance(current_argument_value, - argument_definition.type) and not argument_definition.is_flag: - raise BadTypeException(current_argument_name, argument_definition.type) - - if not is_an_arg: - raise UnknownArgException(current_argument_name) - - # Check that all the mandatory arguments are present - conf = self.cli_parser.validate(conf) - - return conf - - def parse(self, args: list | None = None) -> dict: - """ - Parse the configurations defined via le CLI, Environment Variables and configuration file. - The priority of defined values is the following: - 1. CLI - 2. Environment Variables - 3. Configuration File - - Call the method to produce a configuration dictionary - check the configuration - """ - - if not args: - args = sys.argv - - current_position = 0 - filename = None - for current_arg in args: - if current_arg == '--config-file': - if current_position + 1 == len(args): - logging.error("CLI Error: Config filepath needed with argument --config-file") - sys.exit(-1) - - filename = args[current_position + 1] - args.pop(current_position + 1) - args.pop(current_position) - break - current_position += 1 - - try: - - if len(args) > 1: - conf = self._parse_cli(args[1:]) - else: - conf = {} - conf = self._parse_config_from_environment_variables(current_conf=conf) - if filename: - conf = self._parse_config_from_json_file(file_name=filename, current_conf=conf) - - # We validate the conf - conf = self.validate(conf) - - except MissingValueException as exn: - logging.error('CLI error: Argument "%s" expect a value', exn.argument_name) - sys.exit(-1) - - except BadTypeException as exn: - logging.error('Configuration error: %s', exn.msg) - sys.exit(-1) - - except UnknownArgException as exn: - logging.error('Configuration error: Argument "%s" is unknown', exn.argument_name) - sys.exit(-1) - - except BadContextException as exn: - logging.error('CLI error: %s', exn.msg) - sys.exit(-1) - - except FileNotFoundError: - logging.error("Configuration Error: Configuration file not found") - sys.exit(-1) - - except json.JSONDecodeError as exn: - logging.error('JSON error: "%s" at line %d column %d', exn.msg, exn.lineno, exn.colno) - sys.exit(-1) - - except MissingArgumentException as exn: - logging.error("Configuration Error: %s", exn.msg) - sys.exit(-1) - - except RepeatedArgumentException as exn: - logging.error("Configuration Error: %s", exn.msg) - sys.exit(-1) - - return conf diff --git a/src/powerapi/cli/__init__.py b/src/powerapi/config/__init__.py similarity index 97% rename from src/powerapi/cli/__init__.py rename to src/powerapi/config/__init__.py index 34334ce1..0ff4604d 100644 --- a/src/powerapi/cli/__init__.py +++ b/src/powerapi/config/__init__.py @@ -26,5 +26,3 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from .config_validator import ConfigValidator diff --git a/src/powerapi/config/_utils.py b/src/powerapi/config/_utils.py new file mode 100644 index 00000000..5206829f --- /dev/null +++ b/src/powerapi/config/_utils.py @@ -0,0 +1,79 @@ +# Copyright (c) 2023, Inria +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from copy import deepcopy +from typing import Any + + +def string_to_bool(value: str) -> bool: + """ + Convert a textual boolean value. + :param value: Textual boolean value. + :return: Converted boolean. + :raises ValueError: If the value is not a recognized boolean. + """ + normalized_value = value.strip().casefold() + if normalized_value in ('yes', 'y', 'true', 't', '1'): + return True + if normalized_value in ('no', 'n', 'false', 'f', '0'): + return False + + raise ValueError(f'Invalid boolean value: {value}') + + +def string_to_list(value: str) -> list[str]: + """ + Transforms a comma separated list to a list of strings. + :param value: The string to be converted + :return: List of strings + """ + if value == '': + return [] + + return [v.strip() for v in value.split(',')] + + +def merge_dictionaries(*configurations: dict[str, Any]) -> dict[str, Any]: + """ + Recursively merge configurations from lowest to highest precedence. + + Later configurations override earlier ones. Inputs are not modified. + :param configurations: Configuration dictionaries ordered from lowest to highest precedence. + :return: A new recursively merged configuration dictionary. + """ + merged = {} + + for configuration in configurations: + for key, value in configuration.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = merge_dictionaries(merged[key], value) + else: + merged[key] = deepcopy(value) + + return merged diff --git a/src/powerapi/cli/binding_manager.py b/src/powerapi/config/binding_manager.py similarity index 100% rename from src/powerapi/cli/binding_manager.py rename to src/powerapi/config/binding_manager.py diff --git a/src/powerapi/config/cli_parser.py b/src/powerapi/config/cli_parser.py new file mode 100644 index 00000000..c6de36c9 --- /dev/null +++ b/src/powerapi/config/cli_parser.py @@ -0,0 +1,451 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +from dataclasses import dataclass + +from powerapi.config.config_parser import ( + ArgumentDefinition, + ComponentGroupSchema, + ComponentSchema, + ConfigurationSchema, + ConfigurationSectionSchema, +) +from powerapi.exception import PowerAPIExceptionWithMessage + +_CONFIG_ASSIGNMENTS_DEST = '_powerapi_config_assignments' +_CONFIG_FILE_DEST = '_powerapi_config_file' + + +class CLIParseException(PowerAPIExceptionWithMessage): + """ + Exception raised when command-line arguments cannot be parsed. + """ + + +@dataclass(frozen=True) +class ConfigAssignment: + """ + Raw value associated with a hierarchical configuration path. + """ + path: tuple[str, ...] + value: str + + +@dataclass(frozen=True) +class CLIParseResult: + """ + Configuration values and configuration file selected on the CLI. + """ + configuration: dict + config_file: str | None + + +@dataclass(frozen=True) +class _HelpTheme: + """ + Argparse colors assigned to configuration help elements. + """ + heading_color: str = '' + selector_color: str = '' + path_color: str = '' + supported_type_color: str = '' + component_type_color: str = '' + details_color: str = '' + reset: str = '' + + def _colorize(self, text: str, color: str) -> str: + """ + Apply one theme color to text. + :param text: Text to colorize. + :param color: ANSI color sequence, or an empty string when colors are disabled. + :return: Colorized or unchanged text. + """ + return f'{color}{text}{self.reset}' + + def heading(self, text: str) -> str: + """ + Colorize a configuration help heading. + :param text: Heading to colorize. + :return: Colorized or unchanged heading. + """ + return self._colorize(text, self.heading_color) + + def selector(self, text: str) -> str: + """ + Colorize a configuration block selector. + :param text: Selector to colorize. + :return: Colorized or unchanged selector. + """ + return self._colorize(text, self.selector_color) + + def path(self, text: str) -> str: + """ + Colorize a nested configuration path. + :param text: Path to colorize. + :return: Colorized or unchanged path. + """ + return self._colorize(text, self.path_color) + + def supported_type(self, text: str) -> str: + """ + Colorize a supported component type. + :param text: Component type to colorize. + :return: Colorized or unchanged component type. + """ + return self._colorize(text, self.supported_type_color) + + def component_type(self, text: str) -> str: + """ + Colorize a selected component type. + :param text: Component type to colorize. + :return: Colorized or unchanged component type. + """ + return self._colorize(text, self.component_type_color) + + def details(self, text: str) -> str: + """ + Colorize configuration argument details. + :param text: Details to colorize. + :return: Colorized or unchanged details. + """ + return self._colorize(text, self.details_color) + + +def _parse_assignment(expression: str) -> ConfigAssignment: + """ + Parse a dotted ``PATH=VALUE`` configuration assignment. + :param expression: Assignment expression to parse. + :return: Parsed configuration path and raw value. + :raises CLIParseException: If the expression does not contain a valid supported path. + """ + path_expression, separator, value = expression.partition('=') + if separator == '': + raise CLIParseException(f'Invalid configuration assignment "{expression}": expected PATH=VALUE') + + path = tuple(path_expression.split('.')) + if any(segment == '' for segment in path): + raise CLIParseException(f'Invalid configuration assignment "{expression}": path contains an empty segment') + if len(path) not in (1, 3): + raise CLIParseException(f'Invalid configuration assignment "{expression}": expected PROPERTY or GROUP.COMPONENT.PROPERTY') + + return ConfigAssignment(path=path, value=value) + + +def _apply_assignment(configuration: dict, assignment: ConfigAssignment) -> None: + """ + Apply an assignment to a nested configuration. + :param configuration: Configuration to update. + :param assignment: Parsed assignment to apply. + :raises CLIParseException: If the assignment conflicts with an existing path. + """ + target = configuration + dotted_path = '.'.join(assignment.path) + + for segment in assignment.path[:-1]: + target = target.setdefault(segment, {}) + if not isinstance(target, dict): + raise CLIParseException(f'Conflicting configuration path: "{dotted_path}"') + + property_name = assignment.path[-1] + if isinstance(target.get(property_name), dict): + raise CLIParseException(f'Conflicting configuration path: "{dotted_path}"') + + target[property_name] = assignment.value + + +def _build_configuration(expressions: list[str]) -> dict: + """ + Build a nested configuration dictionary from dotted assignments. + :param expressions: Assignment expressions ordered as they appeared on the command line. + :return: Nested configuration containing the assigned raw values. + :raises CLIParseException: If an expression is invalid or conflicts with another path. + """ + configuration = {} + + for expression in expressions: + _apply_assignment(configuration, _parse_assignment(expression)) + + return configuration + + +def _get_argument_definitions(schema: ConfigurationSectionSchema) -> list[ArgumentDefinition]: + """ + Return each argument definition once in registration order. + :param schema: Configuration section containing the arguments. + :return: Canonical argument definitions. + """ + return list(schema.arguments.values()) + + +def _get_argparse_theme() -> _HelpTheme: + """ + Return argparse's active color theme when its expected layout is available. + :return: Active argparse theme, or a plain-text fallback. + """ + formatter = argparse.RawDescriptionHelpFormatter('') + try: + theme = getattr(formatter, '_theme') # noqa: B009 + return _HelpTheme( + heading_color=theme.heading, + selector_color=theme.long_option, + path_color=theme.summary_long_option, + supported_type_color=theme.summary_action, + component_type_color=theme.action, + details_color=theme.summary_label, + reset=theme.reset, + ) + except AttributeError: + return _HelpTheme() + + +def _format_argument_help(argument: ArgumentDefinition, theme: _HelpTheme) -> str: + """ + Format the description and constraints of a configuration argument. + :param argument: Argument definition to describe. + :param theme: Active argparse color theme. + :return: Human-readable argument help. + """ + details = [argument.argument_type.__name__] + if argument.is_mandatory: + details.append('required') + elif argument.default_value is not None: + details.append(f"default: {argument.default_value!r}") + + details_text = theme.details(f'({", ".join(details)})') + return f'{argument.help_text} {details_text}' if argument.help_text else details_text + + +def _format_assignment_help(theme: _HelpTheme) -> str: + """ + Format the supported explicit-assignment syntax. + :param theme: Active argparse color theme. + :return: Assignment syntax and examples. + """ + return '\n'.join([ + theme.heading('configuration assignments:'), + f' {theme.path('PROPERTY=VALUE')}', + f' {theme.path('GROUP.NAME.PROPERTY=VALUE')}', + ' Example: -C stream=false -C input.sensor.type=socket -C input.sensor.port=9080', + ]) + + +def _format_root_arguments_help(schema: ConfigurationSchema, theme: _HelpTheme) -> str: + """ + Format root configuration paths. + :param schema: Configuration schema containing the root arguments. + :param theme: Active argparse color theme. + :return: Root configuration help, or an empty string when no root arguments are registered. + """ + arguments = _get_argument_definitions(schema) + if not arguments: + return '' + + lines = [theme.heading('root properties:')] + for argument in arguments: + lines.append(f' {theme.path(argument.name)}') + lines.append(f' {_format_argument_help(argument, theme)}') + + return '\n'.join(lines) + + +def _format_component_help(group_name: str, component: ComponentSchema, theme: _HelpTheme) -> str: + """ + Format configuration paths for one component type. + :param group_name: Name of the component group. + :param component: Component schema to describe. + :param theme: Active argparse color theme. + :return: Component type assignment and its property paths. + """ + selector = theme.selector(f'{group_name}.NAME.type') + component_type = theme.component_type(component.name) + lines = [f' {selector}={component_type}'] + for argument in _get_argument_definitions(component): + path = theme.path(f'{group_name}.NAME.{argument.name}') + lines.append(f' {path}') + lines.append(f' {_format_argument_help(argument, theme)}') + + return '\n'.join(lines) + + +def _format_section_help(group_name: str, section_name: str, section: ConfigurationSectionSchema, theme: _HelpTheme) -> str: + """ + Format configuration paths for one fixed section. + :param group_name: Name of the configuration group. + :param section_name: Name of the fixed section. + :param section: Configuration section schema to describe. + :param theme: Active argparse color theme. + :return: Fixed section property paths. + """ + selector = theme.selector(f'{group_name}.{section_name}') + lines = [f' {selector}:'] + for argument in _get_argument_definitions(section): + path = theme.path(f'{group_name}.{section_name}.{argument.name}') + lines.append(f' {path}') + lines.append(f' {_format_argument_help(argument, theme)}') + + return '\n'.join(lines) + + +def _format_group_help(group: ComponentGroupSchema, theme: _HelpTheme) -> str: + """ + Format configuration paths for a configuration group. + :param group: Configuration group to describe. + :param theme: Active argparse color theme. + :return: Group configuration help, or an empty string when the group has no registered schemas. + """ + if not group.components and not group.sections: + return '' + + lines = [theme.heading(f'{group.group_name} configuration:')] + + if group.help_text: + lines.append(f' {group.help_text}') + + if group.components: + selector = theme.selector(f'{group.group_name}.NAME.type') + supported_types = ', '.join(map(theme.supported_type, group.components)) + lines.append(f' {selector}') + lines.append(f' Supported types: {supported_types}') + lines.append('') + lines.append('\n\n'.join( + _format_component_help(group.group_name, component, theme) + for component in group.components.values() + )) + + lines.extend( + _format_section_help(group.group_name, section_name, section, theme) + for section_name, section in group.sections.items() + ) + + return '\n'.join(lines) + + +def _format_configuration_help(schema: ConfigurationSchema) -> str: + """ + Format assignment paths registered in a configuration schema. + :param schema: Configuration schema to describe. + :return: Text appended to the standard argparse help. + """ + theme = _get_argparse_theme() + sections = [ + _format_assignment_help(theme), + _format_root_arguments_help(schema, theme), + *(_format_group_help(group, theme) for group in schema.groups.values()), + ] + return '\n\n'.join(section for section in sections if section) + + +class CLIArgumentParser: + """ + Parse ordinary CLI options and repeatable configuration overrides. + """ + + def __init__(self, schema: ConfigurationSchema) -> None: + """ + Initialize a schema-driven command-line parser. + :param schema: Configuration schema used to register root options and generate help. + """ + self._schema = schema + + @staticmethod + def _add_schema_arguments(argument: ArgumentDefinition, parser: argparse.ArgumentParser) -> None: + """ + Register one schema argument as a command-line option. + :param argument: Schema argument to register. + :param parser: Argument parser receiving the option. + :raises CLIParseException: If the option conflicts with an existing command-line option. + """ + kwargs = { + 'default': argparse.SUPPRESS, + 'dest': argument.name, + 'help': argument.help_text, + } + if argument.is_flag: + kwargs['action'] = 'store_true' + + try: + parser.add_argument(f'--{argument.name}', **kwargs) + except argparse.ArgumentError as error: + raise CLIParseException(f'Failed to add argument: {error}') from error + + def _build_parser(self) -> argparse.ArgumentParser: + """ + Build an argument parser from the current configuration schema. + :return: Configured argument parser. + :raises CLIParseException: If schema arguments define conflicting command-line options. + """ + parser = argparse.ArgumentParser( + allow_abbrev=False, + exit_on_error=False, + epilog=_format_configuration_help(self._schema), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + '--config-file', + dest=_CONFIG_FILE_DEST, + metavar='FILE', + help='Load configuration from a JSON file', + ) + + parser.add_argument( + '-C', '--set-config', + action='append', + default=None, + dest=_CONFIG_ASSIGNMENTS_DEST, + metavar='PATH=VALUE', + help='Set a configuration value using a dotted path; may be repeated', + ) + + for argument in _get_argument_definitions(self._schema): + self._add_schema_arguments(argument, parser) + + return parser + + def parse(self, args: list[str]) -> CLIParseResult: + """ + Parse command-line arguments without validating component schemas. + :param args: Command-line arguments without the executable name. + :return: Parsed root configuration, dotted assignments, and optional configuration file path. + :raises CLIParseException: If argparse rejects the arguments or a dotted assignment is invalid. + """ + parser = self._build_parser() + try: + namespace = parser.parse_args(args) + except argparse.ArgumentError as error: + raise CLIParseException(f'Failed to parse CLI: {error}') from error + + parsed_arguments = dict(vars(namespace)) + expressions = parsed_arguments.pop(_CONFIG_ASSIGNMENTS_DEST) or [] + config_file = parsed_arguments.pop(_CONFIG_FILE_DEST) + + configuration = _build_configuration(expressions) + configuration.update(parsed_arguments) + + return CLIParseResult(configuration=configuration, config_file=config_file) diff --git a/src/powerapi/config/common_cli_parsing_manager.py b/src/powerapi/config/common_cli_parsing_manager.py new file mode 100644 index 00000000..a24c630a --- /dev/null +++ b/src/powerapi/config/common_cli_parsing_manager.py @@ -0,0 +1,466 @@ +# Copyright (c) 2021, Inria +# Copyright (c) 2021, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from powerapi.config.config_parser import ComponentSchema +from powerapi.config.parsing_manager import ConfigurationParsingManager + + +def generate_env_prefix(*components: str, root_prefix: str = 'POWERAPI') -> str: + """ + Generate the environment variable prefix from the given components. + :param components: Additional prefix components. + :param root_prefix: Root namespace for the prefix. + :return: The normalized environment variable prefix. + """ + return '_'.join( + normalized_part.upper() for part in (root_prefix, *components) if (normalized_part := part.strip()) + ) + '_' + + +class PullerSchema(ComponentSchema): + """ + Component schema with arguments shared by every puller input. + """ + + def __init__(self, name: str) -> None: + """ + Initialize a puller schema with the common report model argument. + """ + super().__init__(name) + + self.add_argument( + 'model', + help_text='Report type produced by this input source', + default_value='HWPCReport' + ) + + +class PusherSchema(ComponentSchema): + """ + Component schema with arguments shared by every pusher output. + """ + + def __init__(self, name: str) -> None: + """ + Initialize a pusher schema with the common report model argument. + """ + super().__init__(name) + + self.add_argument( + 'model', + help_text='Report type consumed by this output destination', + default_value='PowerReport' + ) + + +class PreProcessorSchema(ComponentSchema): + """ + Component schema with arguments shared by every pre-processor. + """ + + def __init__(self, name: str) -> None: + """ + Initialize a pre-processor schema with the puller binding argument. + """ + super().__init__(name) + + self.add_argument( + 'puller', + help_text='Name of the puller actor this pre-processor receives reports from', + is_mandatory=True, + ) + + +class CommonCLIParsingManager(ConfigurationParsingManager): + """ + Configuration manager that registers PowerAPI's built-in CLI component options. + """ + + def __init__(self) -> None: + """ + Initialize the configuration manager and register all built-in component schemas. + """ + super().__init__() + + self._register_environment_prefixes() + self._register_groups() + self._register_root_arguments() + self._register_input_schemas() + self._register_output_schemas() + self._register_pre_processor_schemas() + + def _register_environment_prefixes(self) -> None: + """ + Register environment variable prefixes accepted by the configuration manager. + """ + self.add_argument_prefix(generate_env_prefix()) + + def _register_groups(self) -> None: + """ + Register top-level component groups accepted by the CLI. + """ + self.add_group( + name='input', + prefix=generate_env_prefix('INPUT'), + help_text='Configure an input source with -C input.NAME.PROPERTY=VALUE' + ) + self.add_group( + name='output', + prefix=generate_env_prefix('OUTPUT'), + help_text='Configure an output destination with -C output.NAME.PROPERTY=VALUE' + ) + self.add_group( + name='pre-processor', + prefix=generate_env_prefix('PRE_PROCESSOR'), + help_text='Configure a pre-processor with -C pre-processor.NAME.PROPERTY=VALUE' + ) + self.add_group( + name='post-processor', + prefix=generate_env_prefix('POST_PROCESSOR'), + help_text='Configure a post-processor with -C post-processor.NAME.PROPERTY=VALUE' + ) + + def _register_root_arguments(self) -> None: + """ + Register root-level options that apply to the whole PowerAPI process. + """ + self.add_argument( + 'verbose', + is_flag=True, + default_value=False, + help_text='Enable verbose logging', + ) + self.add_argument( + 'stream', + is_flag=True, + default_value=False, + help_text='Enable stream processing mode', + ) + + def _register_input_schemas(self): + """ + Register all built-in input source schemas. + """ + self._register_mongodb_input_schema() + self._register_socket_input_schema() + self._register_csv_input_schema() + self._register_json_input_schema() + + def _register_mongodb_input_schema(self): + """ + Register the MongoDB input schema. + """ + schema_mongo_input = PullerSchema('mongodb') + + schema_mongo_input.add_argument( + 'uri', + help_text='MongoDB connection URI', + is_mandatory=True + ) + schema_mongo_input.add_argument( + 'db', + help_text='MongoDB database name', + is_mandatory=True + ) + schema_mongo_input.add_argument( + 'collection', + help_text='MongoDB collection name', + is_mandatory=True + ) + + self.add_component('input', schema_mongo_input) + + def _register_socket_input_schema(self): + """ + Register the Socket input schema. + """ + schema_socket_input = PullerSchema('socket') + + schema_socket_input.add_argument( + 'host', + help_text='Host address the socket listens on', + default_value='localhost' + ) + schema_socket_input.add_argument( + 'port', + help_text="Port number the socket listens on", + argument_type=int, + default_value=9080, + ) + + self.add_component('input', schema_socket_input) + + def _register_csv_input_schema(self): + """ + Register the CSV input schema. + """ + schema_csv_input = PullerSchema('csv') + + schema_csv_input.add_argument( + 'files', + help_text='Comma-separated list of CSV input files', + argument_type=list, + is_mandatory=True + ) + + self.add_component('input', schema_csv_input) + + def _register_json_input_schema(self): + """ + Register the JSON input schema. + """ + schema_json_input = PullerSchema('json') + + schema_json_input.add_argument( + 'filepath', + help_text='Path to the JSON input file', + is_mandatory=True + ) + schema_json_input.add_argument( + 'compression', + help_text='Input compression format: auto, gzip, lzma, or none', + default_value='auto' + ) + + self.add_component('input', schema_json_input) + + def _register_output_schemas(self): + """ + Register all built-in output destination schemas. + """ + self._register_mongodb_output_schema() + self._register_prometheus_output_schema() + self._register_csv_output_schema() + self._register_json_output_schema() + self._register_influxdb2_output_schema() + self._register_clickhouse_output_schema() + + def _register_mongodb_output_schema(self): + """ + Register the MongoDB output schema. + """ + schema_mongo_output = PusherSchema('mongodb') + + schema_mongo_output.add_argument( + 'uri', + help_text='MongoDB connection URI', + is_mandatory=True + ) + schema_mongo_output.add_argument( + 'db', + help_text='MongoDB database name', + is_mandatory=True + ) + schema_mongo_output.add_argument( + 'collection', + help_text='MongoDB collection name', + is_mandatory=True + ) + + self.add_component('output', schema_mongo_output) + + def _register_prometheus_output_schema(self): + """ + Register the Prometheus output schema. + """ + schema_prometheus_output = PusherSchema('prometheus') + + schema_prometheus_output.add_argument( + 'addr', + help_text='Host address the Prometheus HTTP server listens on', + default_value='localhost' + ) + schema_prometheus_output.add_argument( + 'port', + help_text='Port number the Prometheus HTTP server listens on', + argument_type=int, + default_value=8000 + ) + schema_prometheus_output.add_argument( + 'tags', + help_text='Comma-separated list of report metadata fields exposed as metric labels', + argument_type=list + ) + + self.add_component('output', schema_prometheus_output) + + def _register_csv_output_schema(self): + """ + Register the CSV output schema. + """ + schema_csv_output = PusherSchema('csv') + + schema_csv_output.add_argument( + 'directory', + help_text='Directory where CSV output files are written', + is_mandatory=True + ) + + self.add_component('output', schema_csv_output) + + def _register_json_output_schema(self): + """ + Register the JSON output schema. + """ + schema_json_output = PusherSchema('json') + + schema_json_output.add_argument( + 'filepath', + help_text='Path to the JSON output file', + is_mandatory=True + ) + schema_json_output.add_argument( + 'compression', + help_text='Output compression format: auto, gzip, lzma, or none', + default_value='auto' + ) + + self.add_component('output', schema_json_output) + + def _register_influxdb2_output_schema(self): + """ + Register the InfluxDB 2 output schema. + """ + schema_influx2_output = PusherSchema('influxdb2') + + schema_influx2_output.add_argument( + 'uri', + help_text='InfluxDB server URI', + is_mandatory=True + ) + schema_influx2_output.add_argument( + 'token', + help_text='InfluxDB API token', + is_mandatory=True + ) + schema_influx2_output.add_argument( + 'org', + help_text='InfluxDB organization name', + is_mandatory=True + ) + schema_influx2_output.add_argument( + 'bucket', + help_text='InfluxDB bucket name', + is_mandatory=True + ) + + self.add_component('output', schema_influx2_output) + + def _register_clickhouse_output_schema(self): + """ + Register the ClickHouse output schema. + """ + schema_clickhouse_output = PusherSchema('clickhouse') + + schema_clickhouse_output.add_argument( + 'host', + help_text='ClickHouse server host', + is_mandatory=True, + ) + schema_clickhouse_output.add_argument( + 'port', + help_text='ClickHouse server port', + argument_type=int, + default_value=8123, + ) + schema_clickhouse_output.add_argument( + 'username', + help_text='ClickHouse username', + default_value='default', + ) + schema_clickhouse_output.add_argument( + 'password', + help_text='ClickHouse password', + default_value='', + ) + schema_clickhouse_output.add_argument( + 'database', + help_text='ClickHouse database name', + default_value='default', + ) + + self.add_component('output', schema_clickhouse_output) + + def _register_pre_processor_schemas(self): + """ + Register all built-in pre-processor schemas. + """ + self._register_k8s_pre_processor_schema() + self._register_openstack_pre_processor_schema() + + def _register_k8s_pre_processor_schema(self): + """ + Register the Kubernetes pre-processor schema. + """ + schema_k8s_pre_processor = PreProcessorSchema('kubernetes') + + schema_k8s_pre_processor.add_argument( + 'api-mode', + help_text='Kubernetes API access mode: local, manual, or cluster', + default_value='cluster' + ) + + schema_k8s_pre_processor.add_argument( + 'api-key', + help_text='Kubernetes bearer token for manual API mode', + ) + + schema_k8s_pre_processor.add_argument( + 'api-host', + help_text='Kubernetes API host for manual API mode', + ) + + schema_k8s_pre_processor.add_argument( + 'labels', + help_text='Comma-separated list of Kubernetes pod labels added to reports as metadata', + argument_type=list + ) + + self.add_component('pre-processor', schema_k8s_pre_processor) + + def _register_openstack_pre_processor_schema(self): + """ + Register the OpenStack pre-processor schema. + """ + schema_openstack_pre_processor = PreProcessorSchema('openstack') + + schema_openstack_pre_processor.add_argument( + 'polling-interval', + help_text='OpenStack API polling interval in seconds', + argument_type=float, + default_value=10.0 + ) + + schema_openstack_pre_processor.add_argument( + 'metadata', + help_text='Comma-separated list of OpenStack server metadata fields added to reports', + argument_type=list + ) + + self.add_component('pre-processor', schema_openstack_pre_processor) diff --git a/src/powerapi/config/config_loader.py b/src/powerapi/config/config_loader.py new file mode 100644 index 00000000..b9613e46 --- /dev/null +++ b/src/powerapi/config/config_loader.py @@ -0,0 +1,170 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +import os +from collections.abc import Iterable + +from powerapi.config.config_parser import ( + ComponentGroupSchema, + ConfigurationSchema, +) +from powerapi.exception import ConfigurationError + +_EnvironmentItems = Iterable[tuple[str, str]] + + +class JSONConfigLoader: + """ + Load partial configuration values from a JSON file. + """ + + def load(self, file_name: str | None) -> dict: + """ + Load a JSON configuration. + :param file_name: Path of the JSON file, or None when no configuration file was selected. + :return: The loaded configuration, or an empty dictionary when no file was selected. + :raises FileNotFoundError: If the selected file does not exist. + :raises ConfigurationError: If the input file is invalid. + """ + if file_name is None: + return {} + + try: + with open(file_name, encoding='utf-8') as config_file: + configuration = json.load(config_file) + except json.JSONDecodeError as error: + raise ConfigurationError(f'Invalid JSON in configuration file "{file_name}": {error}') from error + + if not isinstance(configuration, dict): + raise ConfigurationError('Expected a JSON object') + + return configuration + + +class EnvironmentConfigLoader: + """ + Load configuration values using the established PowerAPI environment format. + """ + + def __init__(self, schema: ConfigurationSchema) -> None: + """ + Initialize an environment configuration loader. + :param schema: Schema describing the accepted root, section, and component properties. + """ + self._schema = schema + + def load(self) -> dict: + """ + Load raw configuration values from the environment. + :return: The configuration extracted from the current process environment. + """ + environment = tuple(os.environ.items()) + configuration = self._load_root_configuration(environment) + + for group_name, group in self._schema.groups.items(): + if not group.prefix: + continue + + group_configuration = self._extract_group_values(group, environment) + if group_configuration: + configuration[group_name] = group_configuration + + return configuration + + def _load_root_configuration(self, environment: _EnvironmentItems) -> dict: + """ + Load root configuration values from an environment snapshot. + :param environment: Environment variable names and values. + :return: Raw root configuration values. + """ + configuration = {} + group_prefixes = [group.prefix for group in self._schema.groups.values() if group.prefix] + + for prefix in self._schema.arguments_prefix: + root_values = self._extract_root_values(prefix, group_prefixes, environment) + configuration.update(root_values) + + return configuration + + def _extract_root_values(self, prefix: str, group_prefixes: list[str], environment: _EnvironmentItems) -> dict: + """ + Extract root properties belonging to one environment prefix. + :param prefix: Prefix identifying root configuration variables. + :param group_prefixes: Prefixes reserved for component groups. + :param environment: Environment variable names and values. + :return: Raw root configuration values. + """ + values = {} + for variable_name, value in environment: + if not variable_name.startswith(prefix): + continue + + if any(variable_name.startswith(group_prefix) for group_prefix in group_prefixes): + continue + + property_name = self._normalize_name(variable_name[len(prefix):]) + values[property_name] = value + + return values + + def _extract_group_values(self, group: ComponentGroupSchema, environment: _EnvironmentItems) -> dict: + """ + Extract entry properties belonging to one environment group. + :param group: Schema of the configuration group. + :param environment: Environment variable names and values. + :return: Raw configurations indexed by component or section name. + """ + values = {} + normalized_names = sorted((self._normalize_name(name) for name in [*group.get_argument_names(), 'type']), key=len, reverse=True) + + for variable_name, value in environment: + if not variable_name.startswith(group.prefix): + continue + + suffix = self._normalize_name(variable_name[len(group.prefix):]) + for property_name in normalized_names: + marker = f'{self._schema.default_separator_args_names}{property_name}' + if not suffix.endswith(marker): + continue + + component_name = suffix[:-len(marker)] + if component_name: + values.setdefault(component_name, {})[property_name] = value + + break + + return values + + def _normalize_name(self, name: str) -> str: + """ + Convert an environment name fragment to its configuration spelling. + :param name: Environment name fragment. + :return: Lowercase configuration name using the configured argument separator. + """ + return name.lower().replace(self._schema.default_separator_env_vars_names, self._schema.default_separator_args_names) diff --git a/src/powerapi/config/config_parser.py b/src/powerapi/config/config_parser.py new file mode 100644 index 00000000..62286d76 --- /dev/null +++ b/src/powerapi/config/config_parser.py @@ -0,0 +1,385 @@ +# Copyright (c) 2021, INRIA +# Copyright (c) 2021, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from copy import deepcopy +from dataclasses import dataclass +from typing import Any + +from powerapi.exception import ConfigurationError + +from ._utils import string_to_bool, string_to_list + + +@dataclass(frozen=True) +class ArgumentDefinition: + """ + Definition of a configuration property. + """ + name: str + is_flag: bool = False + default_value: Any = None + help_text: str = '' + argument_type: type[Any] = str + is_mandatory: bool = False + + +class ConfigurationSectionSchema: + """ + Schema for a flat set of configuration properties. + """ + + def __init__(self) -> None: + self.arguments: dict[str, ArgumentDefinition] = {} + + def add_argument( + self, + name: str, + *, + is_flag: bool = False, + default_value: Any = None, + help_text: str = '', + argument_type: type = str, + is_mandatory: bool = False, + ) -> None: + """ + Register a configuration property. + :param name: Configuration property name. + :param is_flag: Whether the property represents a boolean flag. + :param default_value: Value used when the property is omitted. + :param help_text: User-facing description of the property. + :param argument_type: Type used to cast non-flag values. + :param is_mandatory: Whether the property must be defined. + :raises ValueError: If the property name is already registered. + """ + if name in self.arguments: + raise ValueError(f'Configuration property "{name}" is already registered') + + self.arguments[name] = ArgumentDefinition( + name=name, + is_flag=is_flag, + default_value=default_value, + help_text=help_text, + argument_type=bool if is_flag else argument_type, + is_mandatory=is_mandatory, + ) + + def validate(self, conf: dict, path: str = '') -> dict: + """ + Cast properties, require mandatory values, and apply defaults. + :param conf: Partial configuration to validate. + :param path: Dotted path prepended to configuration errors. + :return: Validated configuration with defaults applied. + :raises ConfigurationError: If the configuration is not a dictionary or contains an unknown, missing, or incorrectly typed property. + """ + if not isinstance(conf, dict): + raise ConfigurationError('Expected dict', path or None) + + validated = {} + for name, value in conf.items(): + if name not in self.arguments: + raise ConfigurationError('Unknown property', _join_path(path, name)) + + validated[name] = cast_argument_value(_join_path(path, name), value, self.arguments[name]) + + definitions = self.arguments.values() + for argument in definitions: + name = argument.name + if name not in validated: + if argument.is_mandatory: + raise ConfigurationError('Missing required value', _join_path(path, name)) + if argument.default_value is not None: + validated[name] = deepcopy(argument.default_value) + + return validated + + +class ComponentSchema(ConfigurationSectionSchema): + """ + Schema for one component type in a configuration group. + """ + + def __init__(self, name: str) -> None: + """ + Initialize a component schema. + :param name: Component type handled by this schema. + """ + super().__init__() + self.name = name + + +class ComponentGroupSchema: + """ + Schemas for the dynamic components and fixed sections in a configuration group. + """ + + def __init__(self, group_name: str, help_text: str = '', prefix: str = '') -> None: + """ + Initialize a configuration group schema. + :param group_name: Configuration name of the group. + :param help_text: User-facing description of the group. + :param prefix: Environment-variable prefix assigned to the group. + """ + self.group_name = group_name + self.help_text = help_text + self.prefix = prefix + self.components: dict[str, ComponentSchema] = {} + self.sections: dict[str, ConfigurationSectionSchema] = {} + + def get_argument_names(self) -> list[str]: + """ + Return all canonical property names accepted by the group. + :return: Canonical component and section property names without duplicates. + """ + names = [] + for component in self.components.values(): + names.extend(component.arguments) + for section in self.sections.values(): + names.extend(section.arguments) + + return list(dict.fromkeys(names)) + + def validate(self, conf: dict, path: str) -> dict: + """ + Validate the fixed sections and dynamic components in a group. + :param conf: Group configuration to validate. + :param path: Dotted path of the group. + :return: Validated group configuration with section defaults applied. + :raises ConfigurationError: If the group or one of its entries is invalid. + """ + if not isinstance(conf, dict): + raise ConfigurationError('Expected dict', path) + + validated = {} + for entry_name, entry_values in conf.items(): + entry_path = _join_path(path, entry_name) + validated[entry_name] = self._validate_entry(entry_name, entry_values, entry_path) + + for section_name, section in self.sections.items(): + if section_name in validated: + continue + + section_values = section.validate({}, _join_path(path, section_name)) + if section_values: + validated[section_name] = section_values + + return validated + + def _validate_entry(self, name: str, values: dict, path: str) -> dict: + """ + Validate one fixed section or dynamic component entry. + :param name: Name of the section or component instance. + :param values: Entry configuration to validate. + :param path: Dotted path of the entry. + :return: Validated section or component configuration. + :raises ConfigurationError: If the entry structure, type, or properties are invalid. + """ + if not isinstance(values, dict): + raise ConfigurationError('Expected dict', path) + + if name in self.sections: + return self.sections[name].validate(values, path) + + if 'type' not in values: + raise ConfigurationError('Missing required value', f'{path}.type') + + component_type = values['type'] + try: + schema = self.components[component_type] + except (KeyError, TypeError) as error: + raise ConfigurationError(f'Unknown component type "{component_type}"', f'{path}.type') from error + + component_values = {name: value for name, value in values.items() if name != 'type'} + return { + 'type': component_type, + **schema.validate(component_values, path), + } + + +class ConfigurationSchema(ConfigurationSectionSchema): + """ + Schema and validation rules for a complete PowerAPI configuration. + """ + + def __init__(self, separator_env_vars_names: str = '_', separator_args_names: str = '-') -> None: + """ + Initialize a complete configuration schema. + :param separator_env_vars_names: Separator used in environment-variable names. + :param separator_args_names: Separator used in canonical configuration property names. + """ + super().__init__() + self.groups: dict[str, ComponentGroupSchema] = {} + self.arguments_prefix: list[str] = [] + self.default_separator_env_vars_names = separator_env_vars_names + self.default_separator_args_names = separator_args_names + + def add_argument( + self, + name: str, + *, + is_flag: bool = False, + default_value: Any = None, + help_text: str = '', + argument_type: type = str, + is_mandatory: bool = False, + ) -> None: + """ + Register a root configuration property. + :param name: Configuration property name. + :param is_flag: Whether the property represents a boolean flag. + :param default_value: Value used when the property is omitted. + :param help_text: User-facing description of the property. + :param argument_type: Type used to cast non-flag values. + :param is_mandatory: Whether the property must be defined. + :raises ValueError: If the name is already registered as a property or group. + """ + if name in self.groups: + raise ValueError(f'Configuration name "{name}" is already registered as a group') + + super().add_argument( + name, + is_flag=is_flag, + default_value=default_value, + help_text=help_text, + argument_type=argument_type, + is_mandatory=is_mandatory, + ) + + def add_group(self, name: str, help_text: str = '', prefix: str = '') -> None: + """ + Register a top-level configuration group. + :param name: Configuration name of the group. + :param help_text: User-facing description of the group. + :param prefix: Environment-variable prefix assigned to the group. + :raises ValueError: If the name is already registered as a property or group. + """ + if name in self.groups: + raise ValueError(f'Configuration group "{name}" is already registered') + if name in self.arguments: + raise ValueError(f'Configuration name "{name}" is already registered as a property') + + self.groups[name] = ComponentGroupSchema(name, help_text, prefix) + + def add_component(self, group_name: str, component: ComponentSchema) -> None: + """ + Register a component schema in an existing group. + :param group_name: Group receiving the component schema. + :param component: Component schema to register. + :raises ValueError: If the group is unknown or the component type is already registered in it. + """ + if group_name not in self.groups: + raise ValueError(f'Configuration group "{group_name}" is not registered') + + group = self.groups[group_name] + if component.name in group.components: + raise ValueError(f'Component type "{component.name}" is already registered in group "{group_name}"') + + group.components[component.name] = component + + def add_section(self, group_name: str, section_name: str, section: ConfigurationSectionSchema) -> None: + """ + Register a fixed configuration section in an existing group. + :param group_name: Group receiving the configuration section. + :param section_name: Name identifying and reserving the section in the group. + :param section: Configuration section schema to register. + :raises ValueError: If the group is unknown or the section name is already registered in it. + """ + if group_name not in self.groups: + raise ValueError(f'Configuration group "{group_name}" is not registered') + + group = self.groups[group_name] + if section_name in group.sections: + raise ValueError(f'Configuration section "{section_name}" is already registered in group "{group_name}"') + + group.sections[section_name] = section + + def add_argument_prefix(self, argument_prefix: str) -> None: + """ + Register a non-overlapping root environment-variable prefix. + :param argument_prefix: Environment-variable prefix to register. + :raises ValueError: If the prefix overlaps an existing prefix. + """ + for existing_prefix in self.arguments_prefix: + if argument_prefix.startswith(existing_prefix) or existing_prefix.startswith(argument_prefix): + raise ValueError(f'Environment prefix "{argument_prefix}" conflicts with "{existing_prefix}"') + + self.arguments_prefix.append(argument_prefix) + + def validate(self, conf: dict, path: str = '') -> dict: + """ + Validate the complete nested configuration against the registered schema. + :param conf: Merged configuration to validate. + :param path: Dotted path prepended to configuration errors. + :return: Canonical validated configuration with defaults applied. + :raises ConfigurationError: If any root, section, or component configuration value is invalid. + """ + if not isinstance(conf, dict): + raise ConfigurationError('Expected dict', path or None) + + root_values = {name: value for name, value in conf.items() if name not in self.groups} + validated = super().validate(root_values, path) + + for group_name, group in self.groups.items(): + group_path = _join_path(path, group_name) + validated_group = group.validate(conf.get(group_name, {}), group_path) + + if validated_group or group_name in conf: + validated[group_name] = validated_group + + return validated + + +def cast_argument_value(path: str, value: Any, argument: ArgumentDefinition) -> Any: + """ + Cast one value according to its property definition. + :param path: Dotted path of the property being cast. + :param value: Raw configuration value. + :param argument: Definition describing the expected type. + :return: Value cast to the declared type. + :raises ConfigurationError: If the value cannot be cast to the declared type. + """ + try: + if isinstance(value, argument.argument_type): + return value + if argument.argument_type is bool and isinstance(value, str): + return string_to_bool(value) + if argument.argument_type is list and isinstance(value, str): + return string_to_list(value) + return argument.argument_type(value) + except (TypeError, ValueError) as error: + raise ConfigurationError(f'Expected {argument.argument_type.__name__}', path) from error + + +def _join_path(path: str, property_name: str) -> str: + """ + Append a property name to a configuration path. + :param path: Existing dotted configuration path, or an empty string for the root. + :param property_name: Property segment to append. + :return: Combined dotted path. + """ + return f'{path}.{property_name}' if path else property_name diff --git a/src/powerapi/config/generator.py b/src/powerapi/config/generator.py new file mode 100644 index 00000000..cf7a9781 --- /dev/null +++ b/src/powerapi/config/generator.py @@ -0,0 +1,506 @@ +# Copyright (c) 2021, INRIA +# Copyright (c) 2021, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +from collections.abc import Callable + +from powerapi.actor import Actor, ActorProxy +from powerapi.database.driver import ReadableDatabaseFactory, WritableDatabaseFactory +from powerapi.exception import ConfigurationError, PowerAPIException +from powerapi.filter import ReportFilter +from powerapi.processor.processor_actor import ProcessorActor +from powerapi.puller import PullerActor +from powerapi.pusher import PusherActor +from powerapi.report import FormulaReport, HWPCReport, PowerReport, Report +from powerapi.utils.metadata import build_metadata_mapping + +COMPONENT_TYPE_KEY = 'type' +COMPONENT_MODEL_KEY = 'model' + +ACTOR_NAME_KEY = 'actor_name' + +GENERAL_CONF_STREAM_MODE_KEY = 'stream' +GENERAL_CONF_VERBOSE_KEY = 'verbose' + +_NON_STREAMING_INPUT_TYPES = frozenset(('csv', 'json')) + + +class Generator[ActorT: Actor]: + """ + Generate actors for one configured component group. + """ + + def __init__(self, component_group_name: str): + """ + Initialize a generator for a component group. + :param component_group_name: Name of the component group to generate. + """ + self.component_group_name = component_group_name + + def generate(self, main_config: dict) -> dict[str, ActorT]: + """ + Generate every actor configured in the component group. + :param main_config: Canonical PowerAPI configuration. + :return: Generated actors indexed by component name. + :raises PowerAPIException: If the component group is missing or a component configuration is invalid. + """ + if self.component_group_name not in main_config: + raise PowerAPIException(f'Configuration error: Component "{self.component_group_name}" is not defined') + + actors = {} + for component_name, component_config in main_config[self.component_group_name].items(): + actors[component_name] = self._gen_actor(component_config, main_config, component_name) + + return actors + + def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> ActorT: + """ + Generate one actor from its component configuration. + :param component_config: Canonical component configuration. + :param main_config: Canonical PowerAPI configuration. + :param component_name: Name of the component to generate. + :return: Generated actor. + """ + raise NotImplementedError() + + +class DBActorGenerator[ActorT: Actor, DBFactoryT: ReadableDatabaseFactory | WritableDatabaseFactory](Generator[ActorT]): + """ + Resolve database factories before generating database-backed actors. + """ + + def __init__(self, component_group_name: str): + """ + Initialize a database-backed actor generator. + :param component_group_name: Name of the component group to generate. + """ + super().__init__(component_group_name) + self.report_classes: dict[str, type[Report]] = { + 'HWPCReport': HWPCReport, + 'PowerReport': PowerReport, + 'FormulaReport': FormulaReport, + } + self.database_factories: dict[str, Callable[[dict], DBFactoryT]] = {} + + def _get_report_class(self, model_name: str) -> type[Report]: + """ + Resolve a configured report model name. + :param model_name: Registered report model name. + :return: Report class registered for the configured model. + :raises PowerAPIException: If the report model is unknown. + """ + try: + return self.report_classes[model_name] + except KeyError as error: + raise PowerAPIException(f'Configuration error: Unknown report model "{model_name}"') from error + + def add_report_class(self, model_name: str, report_class: type[Report]): + """ + Register a report class. + :param model_name: Name identifying the report model. + :param report_class: Report class associated with the model name. + :raises ValueError: If the report model is already registered. + """ + if model_name in self.report_classes: + raise ValueError(f'Report model "{model_name}" is already registered') + + self.report_classes[model_name] = report_class + + def add_db_factory(self, db_name: str, db_factory_function: Callable[[dict], DBFactoryT]): + """ + Register a database factory. + :param db_name: Database type handled by the factory. + :param db_factory_function: Function creating a database factory from component configuration. + :raises ValueError: If the database type is already registered. + """ + if db_name in self.database_factories: + raise ValueError(f'Database type "{db_name}" is already registered') + + self.database_factories[db_name] = db_factory_function + + def _create_database_factory(self, db_name: str, component_config: dict) -> DBFactoryT: + """ + Create a database factory for a component. + :param db_name: Registered database type. + :param component_config: Canonical component configuration. + :return: Configured readable or writable database factory. + :raises PowerAPIException: If the database type is unknown or its optional dependencies are unavailable. + """ + try: + factory = self.database_factories[db_name] + except KeyError as error: + raise PowerAPIException(f'Configuration error: Invalid database type: {db_name}') from error + + try: + return factory(component_config) + except ImportError as error: + raise PowerAPIException(f'Dependencies for {db_name} database are not installed') from error + + def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> ActorT: + """ + Resolve the report model and database factory before generating an actor. + :param component_config: Canonical component configuration. + :param main_config: Canonical PowerAPI configuration. + :param component_name: Name of the component to generate. + :return: Generated database-backed actor. + :raises PowerAPIException: If the report model or database type is unknown or a dependency is unavailable. + """ + factory_config = dict(component_config) + factory_config[COMPONENT_MODEL_KEY] = self._get_report_class(component_config[COMPONENT_MODEL_KEY]) + database_factory = self._create_database_factory( + component_config[COMPONENT_TYPE_KEY], + factory_config, + ) + + return self._actor_factory(component_name, main_config, database_factory) + + def _actor_factory(self, actor_name: str, main_config: dict, database_factory: DBFactoryT) -> ActorT: + """ + Create a database-backed actor from a resolved component configuration. + :param actor_name: Name assigned to the actor. + :param main_config: Canonical PowerAPI configuration. + :param database_factory: Configured database factory. + :return: Generated actor. + """ + raise NotImplementedError + + +class PullerGenerator(DBActorGenerator[PullerActor, ReadableDatabaseFactory]): + """ + Generate puller actors from input component configurations. + """ + + @staticmethod + def _csv_input_database_factory(conf: dict) -> ReadableDatabaseFactory: + """ + Create a CSV input database factory. + :param conf: Canonical CSV input configuration. + :return: Configured CSV input factory. + """ + from powerapi.database.csv.driver import CSVInputFactory + return CSVInputFactory(conf['model'], conf['files']) + + @staticmethod + def _json_input_database_factory(conf: dict) -> ReadableDatabaseFactory: + """ + Create a JSON input database factory. + :param conf: Canonical JSON input configuration. + :return: Configured JSON input factory. + """ + from powerapi.database.json.driver import JsonInputFactory + return JsonInputFactory(conf['model'], conf['filepath'], conf['compression']) + + @staticmethod + def _socket_database_factory(conf: dict) -> ReadableDatabaseFactory: + """ + Create a socket input database factory. + :param conf: Canonical socket input configuration. + :return: Configured socket input factory. + """ + from powerapi.database.socket.driver import SocketInputFactory + return SocketInputFactory(conf['model'], conf['host'], conf['port']) + + @staticmethod + def _mongodb_database_factory(conf: dict) -> ReadableDatabaseFactory: + """ + Create a MongoDB input database factory. + :param conf: Canonical MongoDB input configuration. + :return: Configured MongoDB input factory. + """ + from powerapi.database.mongodb.driver import MongodbInputFactory + return MongodbInputFactory(conf['model'], conf['uri'], conf['db'], conf['collection']) + + def __init__(self, report_filter: ReportFilter): + """ + Initialize a puller generator with the built-in input types. + :param report_filter: Report filter applied to incoming reports. + """ + super().__init__('input') + + self.report_filter = report_filter + + self.add_db_factory('csv', self._csv_input_database_factory) + self.add_db_factory('json', self._json_input_database_factory) + self.add_db_factory('socket', self._socket_database_factory) + self.add_db_factory('mongodb', self._mongodb_database_factory) + + def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> PullerActor: + """ + Generate a puller after checking that its input supports the configured execution mode. + :param component_config: Canonical input component configuration. + :param main_config: Canonical PowerAPI configuration. + :param component_name: Name of the input component. + :return: Configured puller actor. + :raises ConfigurationError: If stream mode is enabled for a non-streaming input type. + """ + input_type = component_config[COMPONENT_TYPE_KEY] + if main_config[GENERAL_CONF_STREAM_MODE_KEY] and input_type in _NON_STREAMING_INPUT_TYPES: + raise ConfigurationError(f'Stream mode cannot be used with a {input_type} input', GENERAL_CONF_STREAM_MODE_KEY) + + return super()._gen_actor(component_config, main_config, component_name) + + def _actor_factory(self, actor_name: str, main_config: dict, database_factory: ReadableDatabaseFactory) -> PullerActor: + """ + Create a puller actor. + :param actor_name: Name assigned to the actor. + :param main_config: Canonical PowerAPI configuration. + :param database_factory: Configured readable database factory. + :return: Configured puller actor. + """ + stream_mode = main_config[GENERAL_CONF_STREAM_MODE_KEY] + logging_level = logging.DEBUG if main_config[GENERAL_CONF_VERBOSE_KEY] else logging.WARNING + return PullerActor(actor_name, database_factory, self.report_filter, stream_mode, level_logger=logging_level) + + +class PusherGenerator(DBActorGenerator[PusherActor, WritableDatabaseFactory]): + """ + Generate pusher actors from output component configurations. + """ + + @staticmethod + def _csv_output_database_factory(conf: dict) -> WritableDatabaseFactory: + """ + Create a CSV output database factory. + :param conf: Canonical CSV output configuration. + :return: Configured CSV output factory. + """ + from powerapi.database.csv.driver import CSVOutputFactory + return CSVOutputFactory(conf['model'], conf['directory']) + + @staticmethod + def _json_output_database_factory(conf: dict) -> WritableDatabaseFactory: + """ + Create a JSON output database factory. + :param conf: Canonical JSON output configuration. + :return: Configured JSON output factory. + """ + from powerapi.database.json.driver import JsonOutputFactory + return JsonOutputFactory(conf['model'], conf['filepath'], conf['compression']) + + @staticmethod + def _mongodb_database_factory(conf: dict) -> WritableDatabaseFactory: + """ + Create a MongoDB output database factory. + :param conf: Canonical MongoDB output configuration. + :return: Configured MongoDB output factory. + """ + from powerapi.database.mongodb.driver import MongodbOutputFactory + return MongodbOutputFactory(conf['model'], conf['uri'], conf['db'], conf['collection']) + + @staticmethod + def _influxdb2_database_factory(conf: dict) -> WritableDatabaseFactory: + """ + Create an InfluxDB 2 output database factory. + :param conf: Canonical InfluxDB 2 output configuration. + :return: Configured InfluxDB 2 output factory. + """ + from powerapi.database.influxdb2.driver import InfluxDB2OutputFactory + return InfluxDB2OutputFactory(conf['model'], conf['uri'], conf['org'], conf['bucket'], conf['token']) + + @staticmethod + def _prometheus_database_factory(conf: dict) -> WritableDatabaseFactory: + """ + Create a Prometheus output database factory. + :param conf: Canonical Prometheus output configuration. + :return: Configured Prometheus output factory. + """ + from powerapi.database.prometheus.driver import PrometheusOutputFactory + return PrometheusOutputFactory(conf['model'], conf['addr'], conf['port'], conf.get('tags', [])) + + @staticmethod + def _clickhouse_database_factory(conf: dict) -> WritableDatabaseFactory: + """ + Create a ClickHouse output database factory. + :param conf: Canonical ClickHouse output configuration. + :return: Configured ClickHouse output factory. + """ + from powerapi.database.clickhouse.driver import ClickHouseOutputFactory + return ClickHouseOutputFactory(conf['model'], conf['host'], conf['port'], conf['username'], conf['password'], conf['database']) + + def __init__(self): + """ + Initialize a pusher generator with the built-in output types. + """ + super().__init__('output') + + self.add_db_factory('csv', self._csv_output_database_factory) + self.add_db_factory('json', self._json_output_database_factory) + self.add_db_factory('mongodb', self._mongodb_database_factory) + self.add_db_factory('influxdb2', self._influxdb2_database_factory) + self.add_db_factory('prometheus', self._prometheus_database_factory) + self.add_db_factory('clickhouse', self._clickhouse_database_factory) + + def _actor_factory(self, actor_name: str, main_config: dict, database_factory: WritableDatabaseFactory) -> PusherActor: + """ + Create a pusher actor. + :param actor_name: Name assigned to the actor. + :param main_config: Canonical PowerAPI configuration. + :param database_factory: Configured writable database factory. + :return: Configured pusher actor. + """ + level_logger = logging.DEBUG if main_config[GENERAL_CONF_VERBOSE_KEY] else logging.WARNING + return PusherActor(actor_name, database_factory, logger_level=level_logger) + + def generate_report_mapping(self, main_config: dict, actors: dict[str, PusherActor]) -> dict[type[Report], list[ActorProxy]]: + """ + Map report types to generated pusher proxies. + :param main_config: Canonical PowerAPI configuration. + :param actors: Generated pusher actors indexed by component name. + :return: Pusher proxies indexed by report type. + :raises PowerAPIException: If the output group or a configured actor is missing. + """ + if self.component_group_name not in main_config: + raise PowerAPIException(f'Configuration error: Component "{self.component_group_name}" is not defined') + + report_type_to_actor = {} + for component_name, component_config in main_config[self.component_group_name].items(): + try: + actor_proxy = actors[component_name].get_proxy() + except KeyError as error: + raise PowerAPIException(f'Actor "{component_name}" is not defined') from error + + report_type = self._get_report_class(component_config[COMPONENT_MODEL_KEY]) + report_type_to_actor.setdefault(report_type, []).append(actor_proxy) + + return report_type_to_actor + + +class ProcessorGenerator(Generator[ProcessorActor]): + """ + Generator that initializes the processor actor(s) from the configuration. + """ + + def __init__(self, component_group_name: str): + """ + Initialize a processor generator. + :param component_group_name: Name of the component group to generate. + """ + super().__init__(component_group_name) + + self.processor_factories: dict[str, Callable[[dict], ProcessorActor]] = {} + + def add_processor_factory(self, processor_type: str, processor_factory_function: Callable[[dict], ProcessorActor]) -> None: + """ + Register a processor actor factory. + :param processor_type: Processor type handled by the factory. + :param processor_factory_function: Function creating a processor actor from component configuration. + :raises ValueError: If the processor type is already registered. + """ + if processor_type in self.processor_factories: + raise ValueError(f'Processor type "{processor_type}" is already registered') + + self.processor_factories[processor_type] = processor_factory_function + + def _create_processor(self, processor_name: str, component_config: dict) -> ProcessorActor: + """ + Create a processor actor for a component. + :param processor_name: Registered processor type. + :param component_config: Resolved processor component configuration. + :return: Configured processor actor. + :raises PowerAPIException: If the processor type is unknown or its optional dependencies are unavailable. + """ + try: + factory = self.processor_factories[processor_name] + except KeyError as error: + raise PowerAPIException(f'Configuration error: Invalid processor type: {processor_name}') from error + + try: + return factory(component_config) + except ImportError as error: + raise PowerAPIException(f'Dependencies for {processor_name} processor are not installed') from error + + def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> ProcessorActor: + """ + Add shared processor settings and generate one processor actor. + :param component_config: Canonical component configuration. + :param main_config: Canonical PowerAPI configuration. + :param component_name: Name of the processor actor to generate. + :return: Configured processor actor. + :raises PowerAPIException: If the processor type is unknown or its optional dependencies are unavailable. + """ + runtime_config = dict(component_config) + processor_actor_type = component_config[COMPONENT_TYPE_KEY] + runtime_config[ACTOR_NAME_KEY] = component_name + runtime_config[GENERAL_CONF_VERBOSE_KEY] = main_config[GENERAL_CONF_VERBOSE_KEY] + return self._create_processor(processor_actor_type, runtime_config) + + +class PreProcessorGenerator(ProcessorGenerator): + """ + Generator that initializes the pre-processor actor(s) from the configuration. + """ + + def __init__(self): + """ + Initialize a pre-processor generator with the built-in processor types. + """ + super().__init__('pre-processor') + + self.add_processor_factory('kubernetes', self._k8s_pre_processor_factory) + self.add_processor_factory('openstack', self._openstack_pre_processor_factory) + + @staticmethod + def _k8s_pre_processor_factory(processor_config: dict) -> ProcessorActor: + """ + Create a Kubernetes pre-processor actor. + :param processor_config: Resolved Kubernetes pre-processor configuration. + :return: Configured Kubernetes pre-processor actor. + """ + from powerapi.processor.pre.k8s.actor import KubernetesPreProcessorActor + from powerapi.processor.pre.k8s.monitor_agent import KubernetesMonitorConfig + + api_mode = processor_config['api-mode'] + api_host = processor_config.get('api-host') + api_key = processor_config.get('api-key') + label_mapping = build_metadata_mapping(processor_config.get('labels', []), prefix='k8s_pod_label_') + monitor_config = KubernetesMonitorConfig(api_mode, api_host, api_key, label_mapping) + + name = processor_config[ACTOR_NAME_KEY] + level_logger = logging.DEBUG if processor_config[GENERAL_CONF_VERBOSE_KEY] else logging.INFO + return KubernetesPreProcessorActor(name, monitor_config, level_logger) + + @staticmethod + def _openstack_pre_processor_factory(processor_config: dict) -> ProcessorActor: + """ + Create an OpenStack pre-processor actor. + :param processor_config: Resolved OpenStack pre-processor configuration. + :return: Configured OpenStack pre-processor actor. + """ + from powerapi.processor.pre.openstack.actor import OpenStackPreProcessorActor + from powerapi.processor.pre.openstack.monitor_agent import ( + OpenStackMonitorConfig, + ) + + api_polling_interval = processor_config['polling-interval'] + metadata_mapping = build_metadata_mapping(processor_config.get('metadata', []), prefix='openstack_metadata_') + monitor_config = OpenStackMonitorConfig(api_polling_interval, metadata_mapping) + + name = processor_config[ACTOR_NAME_KEY] + level_logger = logging.DEBUG if processor_config[GENERAL_CONF_VERBOSE_KEY] else logging.INFO + return OpenStackPreProcessorActor(name, monitor_config, level_logger) diff --git a/src/powerapi/config/parsing_manager.py b/src/powerapi/config/parsing_manager.py new file mode 100644 index 00000000..a4a378a6 --- /dev/null +++ b/src/powerapi/config/parsing_manager.py @@ -0,0 +1,161 @@ +# Copyright (c) 2021, INRIA +# Copyright (c) 2021, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys +from typing import Any + +from powerapi.config.cli_parser import CLIArgumentParser +from powerapi.config.config_loader import EnvironmentConfigLoader, JSONConfigLoader +from powerapi.config.config_parser import ( + ComponentSchema, + ConfigurationSchema, + ConfigurationSectionSchema, +) + +from ._utils import merge_dictionaries + + +class ConfigurationParsingManager: + """ + Register the schema and orchestrate all configuration sources. + """ + + def __init__(self) -> None: + self.schema = ConfigurationSchema() + self.argument_parser = CLIArgumentParser(self.schema) + self.json_loader = JSONConfigLoader() + self.environment_loader = EnvironmentConfigLoader(self.schema) + + def add_argument_prefix(self, argument_prefix: str) -> None: + """ + Register an environment-variable prefix for root properties. + :param argument_prefix: Environment-variable prefix to register. + :raises ValueError: If the prefix overlaps an existing prefix. + """ + self.schema.add_argument_prefix(argument_prefix=argument_prefix) + + def add_group(self, name: str, help_text: str = '', prefix: str = '') -> None: + """ + Register a configuration group and its environment prefix. + :param name: Configuration name of the group. + :param help_text: User-facing description of the group. + :param prefix: Environment-variable prefix assigned to the group. + :raises ValueError: If the group name is already registered. + """ + self.schema.add_group(name, help_text=help_text, prefix=prefix) + + def add_component(self, group_name: str, component: ComponentSchema) -> None: + """ + Register a component schema in a group. + :param group_name: Group receiving the component schema. + :param component: Component schema to register. + :raises ValueError: If the group is unknown or the component type is already registered. + """ + self.schema.add_component(group_name, component) + + def add_section(self, group_name: str, section_name: str, section: ConfigurationSectionSchema) -> None: + """ + Register a fixed configuration section in a group. + :param group_name: Group receiving the configuration section. + :param section_name: Name identifying and reserving the section in the group. + :param section: Configuration section schema to register. + :raises ValueError: If the group is unknown or the section name is already registered. + """ + self.schema.add_section(group_name, section_name, section) + + def add_argument( + self, + name: str, + *, + is_flag: bool = False, + default_value: Any = None, + help_text: str = '', + argument_type: type = str, + is_mandatory: bool = False, + ) -> None: + """ + Register a root property in the configuration schema. + :param name: Configuration property name. + :param is_flag: Whether the option is a boolean flag. + :param default_value: Value used when the property is omitted. + :param help_text: Description displayed in command-line help. + :param argument_type: Type used to cast non-flag values. + :param is_mandatory: Whether the property must be defined. + :raises ValueError: If the property name is already registered. + """ + self.schema.add_argument( + name, + is_flag=is_flag, + default_value=default_value, + help_text=help_text, + argument_type=argument_type, + is_mandatory=is_mandatory, + ) + + def validate(self, conf: dict) -> dict: + """ + Validate and canonicalize a merged configuration. + :param conf: Merged configuration to validate. + :return: Canonical validated configuration with defaults applied. + :raises ConfigurationError: If the configuration is invalid. + """ + return self.schema.validate(conf) + + def _parse_configuration_sources(self, cli_line: list[str]) -> dict: + """ + Load and merge every configuration source. + :param cli_line: Command-line arguments without the executable name. + :return: Merged configuration with CLI, environment, then file precedence. + :raises CLIParseException: If command-line arguments are invalid. + :raises FileNotFoundError: If the selected JSON configuration file does not exist. + :raises ConfigurationError: If the selected file does not contain a valid JSON configuration object. + """ + parsed_cli = self.argument_parser.parse(cli_line) + parsed_config_file = self.json_loader.load(parsed_cli.config_file) + parsed_environment = self.environment_loader.load() + + return merge_dictionaries(parsed_config_file, parsed_environment, parsed_cli.configuration) + + def parse(self, args: list[str] | None = None) -> dict: + """ + Load, merge, and validate configuration values. + + Precedence is CLI, then environment variables, then the JSON file. + Parsing and validation errors propagate to the application boundary. + :param args: Command-line arguments including an optional executable name, or None to use ``sys.argv``. + :return: Merged, canonical, and validated PowerAPI configuration. + :raises CLIParseException: If command-line arguments are invalid. + :raises FileNotFoundError: If the selected JSON configuration file does not exist. + :raises ConfigurationError: If the selected file contains invalid JSON or the merged configuration is invalid. + """ + if args is None: + args = sys.argv + + cli_line = args[1:] if args and not args[0].startswith('-') else args + return self.validate(self._parse_configuration_sources(cli_line)) diff --git a/src/powerapi/exception.py b/src/powerapi/exception.py index dbd301a0..cac1e627 100644 --- a/src/powerapi/exception.py +++ b/src/powerapi/exception.py @@ -34,7 +34,7 @@ class PowerAPIException(Exception): """ def __init__(self, *args: object): - Exception.__init__(self, args) + Exception.__init__(self, *args) class PowerAPIExceptionWithMessage(PowerAPIException): @@ -43,10 +43,27 @@ class PowerAPIExceptionWithMessage(PowerAPIException): """ def __init__(self, msg): - PowerAPIException.__init__(self) + PowerAPIException.__init__(self, msg) self.msg = msg +class ConfigurationError(PowerAPIExceptionWithMessage): + """ + Exception raised when configuration loading or validation fails. + """ + + def __init__(self, reason: str, path: str | None = None): + """ + Initialize a configuration error. + :param reason: User-facing explanation of the invalid configuration. + :param path: Dotted path of the invalid value, or None for an error affecting the full configuration. + """ + self.reason = reason + self.path = path + prefix = f'Invalid configuration at "{path}": ' if path else 'Invalid configuration: ' + super().__init__(prefix + reason) + + class BadInputData(PowerAPIException): """ Exception raised when the data read in input are not @@ -64,244 +81,6 @@ def __init__(self, argument_name: str): self.argument_name = argument_name -class NoNameSpecifiedForSubgroupException(ParserException): - """ - Exception raised when attempting to parse substring thant describe a component which not contains the component name - """ - - -class SubgroupAlreadyExistException(ParserException): - """ - Exception raised when attempting to parse a substring to create a component with a name that already exist - """ - - -class SubgroupDoesNotExistException(ParserException): - """ - Exception raised when attempting to add arguments to a subgroup that does not exist - """ - - -class SubgroupParserWithoutNameArgumentException(PowerAPIException): - """ - Exception raised when a subparser without argument name is added to a parser - """ - - -class TooManyArgumentNamesException(ParserException): - """ - Exception raised when attemtping to add an argument with too much names - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - - -class AlreadyAddedArgumentException(ParserException): - """ - Exception raised when attempting to add an argument to a parser that already - have this argument - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Parser already contain an argument ' + argument_name - - -class AlreadyAddedSubparserException(ParserException): - """ - Exception raised when attempting to add a parser that already exists """ - - def __init__(self, parser_name: str): - ParserException.__init__(self, parser_name) - self.msg = 'Parser already contain SubParser with name ' + parser_name - - -class AlreadyAddedSubgroupException(ParserException): - """ - Exception raised when attempting to add a subgroup that already exists """ - - def __init__(self, subgroup_name: str): - ParserException.__init__(self, subgroup_name) - self.msg = 'Parser already contain Subgroup with name ' + subgroup_name - - -class MissingArgumentException(ParserException): - """ - Exception raised when a mandatory argument is missing - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Argument with name(s) ' + argument_name + ' is missing' - - -class RepeatedArgumentException(ParserException): - """ - Exception raised when an argument is repeated several times in a configuration - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Argument with name(s) ' + argument_name + ' is repeated' - - -class MissingValueException(ParserException): - """ - Exception raised when an argument that require a value is caught without - its value - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Argument ' + argument_name + ' require a value' - - -class UnknownArgException(ParserException): - """ - Exception raised when the parser catch an argument that it can't handle - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - self.msg = 'Unknown argument ' + argument_name - - -class BadTypeException(ParserException): - """ - Exception raised when an argument is parsed with a value of an incorrect type - """ - - def __init__(self, argument_name: str, arg_type: type): - ParserException.__init__(self, argument_name) - self.msg = argument_name + " expect " + arg_type.__name__ - - -class BadContextException(ParserException): - """ - Exception raised when the parser catch an argument that it can't handle in - the current context - """ - - def __init__(self, argument_name: str, context_list: list): - ParserException.__init__(self, argument_name) - self.context_list = context_list - self.msg = 'argument ' + argument_name + 'not used in the correct context\nUse it with the following arguments :' - for main_arg_name, context_name in context_list: - self.msg += '\n --' + main_arg_name + ' ' + context_name - - -class NotAllowedArgumentValueException(PowerAPIException): - """ - This exception happens when the configuration define an argument value that is incompatible with other arguments' - values - """ - - -class FileDoesNotExistException(PowerAPIExceptionWithMessage): - """ - This exception happens when the configuration define a input file that does not exist or is not accessible - """ - - def __init__(self, file_name: str): - PowerAPIExceptionWithMessage.__init__(self, "The File " + file_name + " does not exist or is not accessible") - self.file_name = file_name - - -class SameLengthArgumentNamesException(ParserException): - """ - Exception raised when attempting to add an argument with names that have the same length - - """ - - def __init__(self, argument_name: str): - ParserException.__init__(self, argument_name) - - -class ModelNameAlreadyUsed(PowerAPIException): - """ - Exception raised when attempting to add to a DBActorGenerator a model factory with a name already bound to another - model factory in the DBActorGenerator - """ - - def __init__(self, model_name: str): - PowerAPIException.__init__(self) - self.model_name = model_name - - -class DatabaseNameDoesNotExist(PowerAPIException): - """ - Exception raised when attempting to remove to a DBActorGenerator a database factory with a name that is not bound to - another database factory in the DBActorGenerator - """ - - def __init__(self, database_name: str): - PowerAPIException.__init__(self) - self.database_name = database_name - - -class DatabaseNameAlreadyUsed(PowerAPIException): - """ - Exception raised when attempting to add to a DBActorGenerator a database factory with a name already bound to - another database factory in the DBActorGenerator - """ - - def __init__(self, database_name: str): - PowerAPIException.__init__(self) - self.database_name = database_name - - -class ModelNameDoesNotExist(PowerAPIException): - """ - Exception raised when attempting to remove to a DBActorGenerator a model factory with a name that is not bound to - another model factory in the DBActorGenerator - """ - - def __init__(self, model_name: str): - PowerAPIException.__init__(self) - self.model_name = model_name - - -class InvalidPrefixException(PowerAPIException): - """ - Exception raised when attempting to add a new prefix that is a prefix of an existing one or - vice-versa - """ - - def __init__(self, existing_prefix: str, new_prefix: str): - PowerAPIException.__init__(self) - self.new_prefix = new_prefix - self.existing_prefix = existing_prefix - self.msg = "The new prefix " + self.new_prefix + " has a conflict with the existing prefix " \ - + self.existing_prefix - - -class ProcessorTypeDoesNotExist(PowerAPIException): - """ - Exception raised when attempting to remove to a ProcessorActorGenerator a processor factory with a type that is not - bound to a processor factory - """ - - def __init__(self, processor_type: str): - PowerAPIException.__init__(self) - self.processor_type = processor_type - - -class ProcessorTypeAlreadyUsed(PowerAPIException): - """ - Exception raised when attempting to add to a ProcessorActorGenerator a processor factory with a type already bound - to another processor factory - """ - - def __init__(self, processor_type: str): - PowerAPIException.__init__(self) - self.processor_type = processor_type - - class UnsupportedActorTypeException(ParserException): """ Exception raised when the binding manager do not support an actor type diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py deleted file mode 100644 index 92bd7be7..00000000 --- a/tests/unit/cli/conftest.py +++ /dev/null @@ -1,462 +0,0 @@ -# Copyright (c) 2023, Inria -# Copyright (c) 2023, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import sys -from copy import deepcopy - -import pytest - -import tests.utils.cli as test_files_module -from powerapi.cli.config_parser import SubgroupConfigParser, BaseConfigParser, store_true, RootConfigParser -from powerapi.cli.parsing_manager import RootConfigParsingManager, SubgroupConfigParsingManager -from tests.utils.cli.base_config_parser import load_configuration_from_json_file, generate_cli_configuration_from_json_file - - -@pytest.fixture(name="invalid_csv_io_stream_config") -def csv_input_output_stream_config(): - """ - Invalid configuration with csv as input and output and stream mode enabled - """ - return load_configuration_from_json_file(file_name='csv_input_output_stream_mode_enabled_configuration.json') - - -@pytest.fixture -def several_inputs_outputs_stream_config(): - """ - Configuration with several inputs and outputs and stream mode enabled - """ - return load_configuration_from_json_file('several_inputs_outputs_stream_mode_enabled_configuration.json') - - -@pytest.fixture -def single_input_multiple_outputs_with_different_report_type(): - """ - Configuration with several inputs and outputs and stream mode enabled - """ - return load_configuration_from_json_file('single_input_multiple_outputs_with_different_report_type_configuration.json') - - -@pytest.fixture -def several_inputs_outputs_stream_socket_without_some_arguments_config(several_inputs_outputs_stream_config): - """ - Configuration with several inputs and outputs and stream mode enabled. Some arguments - of socket input are removed - """ - for _, current_input in several_inputs_outputs_stream_config["input"].items(): - if current_input['type'] == 'socket': - current_input.pop('port') - - return several_inputs_outputs_stream_config - - -@pytest.fixture -def csv_io_postmortem_config(invalid_csv_io_stream_config): - """ - Valid configuration with csv as input and output and stream mode disabled - """ - invalid_csv_io_stream_config["stream"] = False - return invalid_csv_io_stream_config - - -@pytest.fixture -def csv_io_postmortem_config_without_optional_arguments(csv_io_postmortem_config): - """ - Valid configuration with csv as input and output without optional arguments, i.e., - stream, verbose, and name model for inputs and outputs - """ - csv_io_postmortem_config.pop('stream') - csv_io_postmortem_config.pop('verbose') - - for current_input_name in csv_io_postmortem_config['input']: - csv_io_postmortem_config['input'][current_input_name].pop('model') - csv_io_postmortem_config['input'][current_input_name].pop('name') - - for current_ouput_name in csv_io_postmortem_config['output']: - csv_io_postmortem_config['output'][current_ouput_name].pop('model') - csv_io_postmortem_config['output'][current_ouput_name].pop('name') - - return csv_io_postmortem_config - - -@pytest.fixture -def config_without_input(csv_io_postmortem_config): - """ - Invalid configuration without inputs - """ - csv_io_postmortem_config.pop('input') - - return csv_io_postmortem_config - - -@pytest.fixture -def config_without_output(csv_io_postmortem_config): - """ - Invalid configuration without inputs - """ - csv_io_postmortem_config.pop('output') - - return csv_io_postmortem_config - - -@pytest.fixture -def subgroup_parser(): - """ - A subgroup parser with one argument "-a" - """ - parser = SubgroupConfigParser('test') - parser.add_argument('a', is_flag=True) - return parser - - -@pytest.fixture -def create_empty_files_from_config(invalid_csv_io_stream_config: dict): - """ - Create on the module path the files that are indicated on csv input. - When they are no longer required, those files are erased - """ - for _, input_config in invalid_csv_io_stream_config['input'].items(): - if input_config['type'] == 'csv': - for file_str in input_config['files']: - if not os.path.isfile(file_str): - with open(file_str, 'w') as file: - file.close() - - yield - - for _, input_config in invalid_csv_io_stream_config['input'].items(): - if input_config['type'] == 'csv': - for file_str in input_config['files']: - if os.path.isfile(file_str): - os.remove(file_str) - - -@pytest.fixture -def base_config_parser(): - """ - Return a BaseConfigParser with mandatory and optional arguments - """ - - parser = BaseConfigParser() - - parser.add_argument('arg1', 'argument1', 'argumento1', default_value=3, argument_type=int, is_mandatory=False) - - parser.add_argument('argumento2', 'arg2', argument_type=str, is_mandatory=True) - - parser.add_argument('arg3', 'argument3', argument_type=bool, is_mandatory=False) - - parser.add_argument('dded', 'arg4', argument_type=float, is_mandatory=True) - - parser.add_argument('arg5', '5', default_value='default value', argument_type=str, help_text='help 5') - - return parser - - -@pytest.fixture -def root_config_parser_with_mandatory_and_optional_arguments(): - """ - Return a RootConfigParser with mandatory and optional arguments - """ - - parser = RootConfigParser() - - parser.add_argument('a', argument_type=bool, is_flag=True, action=store_true) - - parser.add_argument('argument1', 'arg1', default_value=3, argument_type=int, is_mandatory=False) - - parser.add_argument('argumento2', '2', argument_type=str, is_mandatory=True) - - parser.add_argument('arg3', 'argument3', argument_type=bool, is_mandatory=False) - - parser.add_argument('d', 'arg4', argument_type=float, is_mandatory=True) - - parser.add_argument('arg5', '5', default_value='default value', argument_type=str, - help_text='help 5') - - return parser - - -@pytest.fixture -def root_config_parser_with_subgroups(root_config_parser_with_mandatory_and_optional_arguments): - """ - Return a RootConfigParser with subgroups - """ - - root_config_parser_with_mandatory_and_optional_arguments.add_argument_prefix(argument_prefix='TEST_') - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='g1', prefix='TEST_G1_') - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='g2', prefix='TEST_G2_') - - subgroup_parser_g1 = SubgroupConfigParser(name='type1') - subgroup_parser_g1.add_argument('1', 'a1', argument_type=str, is_mandatory=True) - subgroup_parser_g1.add_argument('2', 'a2', argument_type=bool, default_value=True) - subgroup_parser_g1.add_argument('3', 'a3', argument_type=str, default_value=69) - subgroup_parser_g1.add_argument('n', 'name', argument_type=str) - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup_parser(subgroup_type='g1', - subgroup_parser=subgroup_parser_g1) - - subgroup_parser_g2 = SubgroupConfigParser(name='type2') - subgroup_parser_g2.add_argument('1', 'a1', argument_type=float, is_mandatory=False) - subgroup_parser_g2.add_argument('2', 'a2', argument_type=str) - subgroup_parser_g2.add_argument('3', 'a3', argument_type=str) - subgroup_parser_g2.add_argument('4', 'a4', argument_type=str) - subgroup_parser_g2.add_argument('n', 'name', argument_type=str) - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup_parser(subgroup_type='g2', - subgroup_parser=subgroup_parser_g2) - - return root_config_parser_with_mandatory_and_optional_arguments - - -@pytest.fixture -def base_config_parser_no_mandatory_arguments(): - """ - Return a BaseConfigParser without mandatory arguments - """ - parser = BaseConfigParser() - - parser.add_argument('arg1', default_value=4, argument_type=int) - - parser.add_argument('arg2', argument_type=str) - - parser.add_argument('arg3', argument_type=bool) - - parser.add_argument('arg4', argument_type=int) - - parser.add_argument('arg5', argument_type=int) - - return parser - - -@pytest.fixture -def base_config_parser_str_representation(): - """ - Return expected representation for a BaseConfigParser used in unit tests - """ - return ' --arg1, --argument1, --argumento1 : \n' + \ - ' --argumento2, --arg2 : \n' + \ - ' --arg3, --argument3 : \n' + \ - ' --dded, --arg4 : \n' + \ - ' --arg5, -5 : help 5\n' - - -@pytest.fixture -def root_config_parsing_manager(): - """ - Return a RootConfigParsingManager with a flag argument 'a' - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a', argument_type=bool, is_flag=True, action=store_true) - parser_manager.add_subgroup(name='sub') - - return parser_manager - - -@pytest.fixture -def root_config_parsing_manager_with_mandatory_and_optional_arguments(): - """ - Return a RootConfigParsingManager with several arguments, some of them are mandatory - """ - parser_manager = RootConfigParsingManager() - - parser_manager.add_argument_prefix(argument_prefix='TEST_') - - parser_manager.add_subgroup(name='input', prefix='TEST_INPUT_') - - parser_manager.add_subgroup(name='output', prefix='TEST_OUTPUT_') - - parser_manager.add_argument('a', argument_type=bool, is_flag=True, action=store_true) - - parser_manager.add_argument('1', 'argument1', default_value=3, argument_type=int, is_mandatory=False) - - parser_manager.add_argument('argumento2', '2', argument_type=str, is_mandatory=True) - - parser_manager.add_argument('arg3', 'argument3', argument_type=bool, is_mandatory=False) - - parser_manager.add_argument('d', 'arg4', argument_type=float, is_mandatory=True) - - parser_manager.add_argument('arg5', '5', default_value='default value', argument_type=str, - help_text='help 5') - - i1_type_subgroup_parser_manager = SubgroupConfigParsingManager(name="i1_type") - i1_type_subgroup_parser_manager.add_argument('model', 'm', argument_type=str, is_mandatory=True) - i1_type_subgroup_parser_manager.add_argument('db', 'd', argument_type=str, is_mandatory=False) - i1_type_subgroup_parser_manager.add_argument('port', 'p', argument_type=int, is_mandatory=False) - i1_type_subgroup_parser_manager.add_argument('name', 'n', argument_type=str, is_mandatory=False, - default_value='my_i1_instance') - - parser_manager.add_subgroup_parser(subgroup_name="input", subgroup_parser=i1_type_subgroup_parser_manager) - - o1_type_subgroup_parser_manager = SubgroupConfigParsingManager(name="o1_type") - o1_type_subgroup_parser_manager.add_argument('model', 'm', argument_type=str, is_mandatory=True) - o1_type_subgroup_parser_manager.add_argument('db', 'd', argument_type=str, is_mandatory=False) - o1_type_subgroup_parser_manager.add_argument('name', 'n', argument_type=str, is_mandatory=False, - default_value='my_o1_instance') - o1_type_subgroup_parser_manager.add_argument('collection', 'c', argument_type=str) - - parser_manager.add_subgroup_parser(subgroup_name="output", subgroup_parser=o1_type_subgroup_parser_manager) - - o2_type_subgroup_parser_manager = SubgroupConfigParsingManager(name="o2_type") - o2_type_subgroup_parser_manager.add_argument('model', 'm', argument_type=str, is_mandatory=True) - o2_type_subgroup_parser_manager.add_argument('db', 'd', argument_type=str, is_mandatory=False) - o2_type_subgroup_parser_manager.add_argument('name', 'n', argument_type=str, is_mandatory=False, - default_value='my_o2_instance') - o2_type_subgroup_parser_manager.add_argument('collection', 'c', argument_type=str) - - parser_manager.add_subgroup_parser(subgroup_name="output", subgroup_parser=o2_type_subgroup_parser_manager) - - return parser_manager - - -@pytest.fixture -def test_files_path(): - """ - Return the path of directory containing tests files - """ - return test_files_module.__path__[0] - - -@pytest.fixture -def cli_configuration(config_file: str, monkeypatch): - """ - Load in sys.argv a configuration with arguments extracted from a json file - """ - monkeypatch.setattr(sys, 'argv', generate_cli_configuration_from_json_file(file_name=config_file)) - - -@pytest.fixture -def empty_cli_configuration(monkeypatch): - """ - Clean the CLI arguments - """ - monkeypatch.setattr(sys, 'argv', []) - - -@pytest.fixture -def output_input_configuration(): - """ - Return a dictionary containing bindings with a processor - """ - return load_configuration_from_json_file(file_name='output_input_configuration.json') - - -@pytest.fixture(params=['k8s_pre_processor_complete_configuration.json']) -def pre_processor_complete_configuration(request): - """ - Return a dictionary containing a configuration with pre-processor - """ - return load_configuration_from_json_file(file_name=request.param) - - -@pytest.fixture -def pre_processor_config_without_puller(pre_processor_complete_configuration): - """ - Return a configuration with processors but without bindings - """ - - pre_processor_complete_configuration['pre-processor']['my_processor'].pop('puller') - - return pre_processor_complete_configuration - - -@pytest.fixture -def empty_pre_processor_config(pre_processor_complete_configuration): - """ - Return a configuration with bindings but without processors - """ - - pre_processor_complete_configuration.pop('pre-processor') - - return pre_processor_complete_configuration - - -@pytest.fixture(params=['k8s_pre_processor_with_non_existing_puller_configuration.json']) -def pre_processor_with_unexisting_puller_configuration(request): - """ - Return a dictionary containing a pre-processor with a puller that doesn't exist - """ - return load_configuration_from_json_file(request.param) - - -def get_config_with_longest_argument_names(config: dict, arguments: dict): - """ - Return a copy of the provided configuration with the longest name for each argument - :param dict config: Configuration to be modified - :param dict arguments: Arguments definition - """ - config_longest_names = {} - for argument_name in config.keys(): - current_argument = arguments[argument_name] - longest_argument_name = get_longest_name(current_argument.names) - config_longest_names[longest_argument_name] = config[argument_name] - - return config_longest_names - - -def get_longest_name(names: list): - """ - Return the longest name in the provide list - :param list names: List of names - """ - longest_name = "" - - for name in names: - if len(name) > len(longest_name): - longest_name = name - - return longest_name - - -def get_config_with_default_values(config: dict, arguments: dict): - """ - Get a configuration that contains all optional arguments with their default values - :param dict config: Configuration to be modified - :param dict arguments: Arguments definition - """ - - processed_arguments = [] - - config_all_values = deepcopy(config) - - for current_argument_name, current_argument in arguments.items(): - if current_argument not in processed_arguments: - argument_value_already_defined = False - - for name in current_argument.names: - if name in config: - argument_value_already_defined = True - - break - - if not argument_value_already_defined and current_argument.default_value is not None: - config_all_values[current_argument_name] = current_argument.default_value - - processed_arguments.append(current_argument) - - return config_all_values diff --git a/tests/unit/cli/test_config_parser.py b/tests/unit/cli/test_config_parser.py deleted file mode 100644 index 52e16a84..00000000 --- a/tests/unit/cli/test_config_parser.py +++ /dev/null @@ -1,978 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest - -from powerapi.cli.config_parser import ConfigurationArgument, BaseConfigParser, RootConfigParser, SubgroupConfigParser -from powerapi.cli.config_parser import store_true, store_val -from powerapi.exception import AlreadyAddedArgumentException, BadTypeException, UnknownArgException, \ - BadContextException, MissingValueException, SubgroupAlreadyExistException, \ - SubgroupParserWithoutNameArgumentException, \ - NoNameSpecifiedForSubgroupException, TooManyArgumentNamesException, MissingArgumentException, \ - SameLengthArgumentNamesException, AlreadyAddedSubgroupException -from tests.unit.cli.conftest import get_config_with_longest_argument_names, get_config_with_default_values - -from tests.utils.cli.base_config_parser import load_configuration_from_json_file, \ - generate_configuration_tuples_from_json_file, define_environment_variables_configuration_from_json_file, \ - remove_environment_variables_configuration - - -############### -# PARSER TEST # -############### -def test_add_argument_that_already_exists(): - """ - Test if an AlreadyAddedArgumentException is raised when an argument that already exists is added to a - BaseConfigParser - """ - - parser = BaseConfigParser() - parser.add_argument('a') - parser.add_argument('bb', 'bbb', 'b', 'bbbbbb') - - with pytest.raises(AlreadyAddedArgumentException): - parser.add_argument('a') - - with pytest.raises(AlreadyAddedArgumentException): - parser.add_argument('bbb') - - with pytest.raises(AlreadyAddedArgumentException): - parser.add_argument('bbbbbb') - - with pytest.raises(AlreadyAddedArgumentException): - parser.add_argument('ccc', 'a') - - assert len(parser.arguments) == 5 - - -def test_get_arguments_returns_all_stored_arguments(): - """ - Test if all the arguments are correctly stored by BasePaserConfig - """ - longest_name_arg_a = 'aaa' - name_arg_a_1 = 'a' - name_arg_a_2 = 'ab' - expected_argument_a = ConfigurationArgument(names=[name_arg_a_1, longest_name_arg_a, name_arg_a_2], - argument_type=bool, default_value=False, - help_text='This a parameter', is_mandatory=False, is_flag=True) - - longest_name_arg_xx = 'XXXX' - name_arg_xx_1 = 'xx' - name_arg_xx_2 = 'xax' - expected_argument_xx = ConfigurationArgument(names=[name_arg_xx_1, longest_name_arg_xx, name_arg_xx_2], - argument_type=str, default_value='Hi', - help_text='This is another parameter', is_mandatory=True, - is_flag=False, action=store_val) - - parser = BaseConfigParser() - parser.add_argument(name_arg_a_1, longest_name_arg_a, name_arg_a_2, is_mandatory=expected_argument_a.is_mandatory, - is_flag=expected_argument_a.is_flag, - argument_type=expected_argument_a.type, help_text=expected_argument_a.help_text, - default_value=expected_argument_a.default_value) - - parser.add_argument(name_arg_xx_1, longest_name_arg_xx, name_arg_xx_2, - is_mandatory=expected_argument_xx.is_mandatory, is_flag=expected_argument_xx.is_flag, - argument_type=expected_argument_xx.type, help_text=expected_argument_xx.help_text, - default_value=expected_argument_xx.default_value) - - arguments = parser.get_arguments() - - assert len(arguments) == 6 - - assert longest_name_arg_a in arguments - - assert longest_name_arg_xx in arguments - - assert expected_argument_a == arguments.get(longest_name_arg_a) - - assert expected_argument_xx == arguments.get(longest_name_arg_xx) - - -def test_get_mandatory_arguments_return_all_mandatory_argument(base_config_parser): - """ - Test that all the mandatory arguments are identified by the paser - """ - expected_mandatory_args_names = ['arg2', 'arg4'] - - mandatory_args = base_config_parser._get_mandatory_arguments() - - assert len(mandatory_args) == len(expected_mandatory_args_names) - - for expected_mandatory_args_name in expected_mandatory_args_names: - is_present = False - for mandatory_arg in mandatory_args: - if expected_mandatory_args_name in mandatory_arg.names: - is_present = True - break - assert is_present - - -def test_get_mandatory_arguments_return_empty_list_with_no_mandatory_args(base_config_parser_no_mandatory_arguments): - """ - Test that mandatory arguments list is empty if parser does not have mandatory arguments - """ - mandatory_args = base_config_parser_no_mandatory_arguments._get_mandatory_arguments() - - assert not mandatory_args - - -def test_validate_check_mandatory_arguments_on_configuration(base_config_parser): - """ - Test if mandatory arguments are verified by the parser - """ - conf = load_configuration_from_json_file('basic_configuration.json') - config_longest_names = get_config_with_default_values(config=conf, arguments=base_config_parser.arguments) - config_longest_names = get_config_with_longest_argument_names(config=config_longest_names, - arguments=base_config_parser.arguments) - conf_without_mandatory_arguments = \ - load_configuration_from_json_file('basic_configuration_without_mandatory_arguments.json') - - try: - validated_config = base_config_parser.validate(conf) - assert validated_config == config_longest_names - except MissingArgumentException as e: - pytest.fail(f'Missing arguments: {e}') - - with pytest.raises(MissingArgumentException): - _ = base_config_parser.validate(conf_without_mandatory_arguments) - - -def test_validate_accept_configuration_when_no_mandatory_arguments_exist(base_config_parser_no_mandatory_arguments): - """ - Test if a configuration passes the validation if there is no mandatory argument - """ - conf = load_configuration_from_json_file('basic_configuration_without_mandatory_arguments.json') - - try: - validated_config = base_config_parser_no_mandatory_arguments.validate(conf) - assert validated_config == conf - except MissingArgumentException as e: - pytest.fail(f'Missing argument: {e}') - - -def test_validate_adds_default_values_for_no_arguments_defined_in_configuration_that_have_one(base_config_parser): - """ - Test if parser add default values for arguments that are not in configuration and that have one - """ - conf = load_configuration_from_json_file('basic_configuration_without_arguments_with_default_values.json') - - expected_conf_default_values = get_config_with_default_values(config=conf, arguments=base_config_parser.arguments) - expected_conf_default_values = get_config_with_longest_argument_names(config=expected_conf_default_values, - arguments=base_config_parser.arguments) - - validated_config = base_config_parser.validate(conf) - assert validated_config == expected_conf_default_values - - -def test_get_arguments_str_return_str_with_all_information(base_config_parser, base_config_parser_str_representation): - """ - Test that the parser is able to return a string with all the information related to it in a correct format - """ - - arguments_str = base_config_parser._get_arguments_str(' ') - - assert arguments_str == base_config_parser_str_representation - - -def test_parser_return_correct_values_for_each_argument(base_config_parser): - """ - Test that the _parser method return correct values for different arguments in configuration - """ - - args = generate_configuration_tuples_from_json_file('basic_configuration.json') - acc = {} - - expected_acc = {'argumento1': 5, - "argumento2": "this a mandatory argument", - "argument3": False, - "dded": 10.5} - - args, acc = base_config_parser._parse(args, acc) - - assert not args - assert acc == expected_acc - - -def test_parser_raise_an_exception_with_an_unknown_argument(base_config_parser): - """ - Test that the _parser method return correct values for different arguments in configuration - """ - - args = generate_configuration_tuples_from_json_file('basic_configuration.json') - args.append(('unknown_arg', 'This is a new argument')) - acc = {} - - with pytest.raises(NotImplementedError): - _, _ = base_config_parser._parse(args, acc) - - -##################### -# ROOT PARSER TESTS # -##################### -# Test add_argument optargs # - - -def test_add_short_argument(): - """ - Test if an argument when a short name is added to the short_arg string - """ - parser = RootConfigParser(help_arg=False) - assert parser.short_arg == '' - parser.add_argument('a') - assert parser.short_arg == 'a:' - - assert len(parser.arguments) == 1 - - -def test_add_flag_argument_with_short_name(): - """ - Test if a flag argument with a short name was added to the short_arg string - """ - parser = RootConfigParser(help_arg=False) - assert parser.short_arg == '' - parser.add_argument('a', is_flag=True) - assert parser.short_arg == 'a' - - assert len(parser.arguments) == 1 - - -def test_add_several_arguments_with_short_names(): - """ - Test if the arguments were added to the short_arg string - """ - parser = RootConfigParser(help_arg=False) - assert parser.short_arg == '' - parser.add_argument('a', is_flag=True) - assert parser.short_arg == 'a' - parser.add_argument('b', is_flag=True) - assert parser.short_arg == 'ab' - parser.add_argument('c') - assert parser.short_arg == 'abc:' - parser.add_argument('d') - assert parser.short_arg == 'abc:d:' - parser.add_argument('e') - parser.add_argument('f') - parser.add_argument('g') - assert parser.short_arg == 'abc:d:e:f:g:' - - assert len(parser.arguments) == 7 - - -def test_add_argument_with_two_short_names_raise_an_exception(): - """ - Test if adding an argument with two short names raises a SameLengthArgumentNamesException - """ - parser = RootConfigParser(help_arg=False) - - with pytest.raises(SameLengthArgumentNamesException): - parser.add_argument('a', 'b') - - assert parser.short_arg == '' - assert not parser.arguments - - -def test_add_argument_with_long_name(): - """ - Test if an argument with a long name is added to the long_arg list - """ - parser = RootConfigParser(help_arg=False) - assert not parser.long_arg - parser.add_argument('aaa') - assert parser.long_arg == ['aaa='] - - assert len(parser.arguments) == 1 - - -def test_add_flag_argument_with_long(): - """ - Test if a flag argument with a long name is added to the long_arg list - """ - parser = RootConfigParser(help_arg=False) - assert not parser.long_arg - parser.add_argument('aaa', is_flag=True) - assert parser.long_arg == ['aaa'] - - assert len(parser.arguments) == 1 - - -def test_add_argument_with_more_than_two_names_raise_an_exception(): - """ - Test if adding an argument with more than two names raises a TooManyArgumentNamesException - """ - parser = RootConfigParser(help_arg=False) - assert not parser.long_arg - with pytest.raises(TooManyArgumentNamesException): - parser.add_argument('aaa', 'bbb', 'ccc', is_flag=True) - - with pytest.raises(TooManyArgumentNamesException): - parser.add_argument('aaa', 'b', 'c') - - assert not parser.long_arg - assert not parser.arguments - - -# full parsing test # - -def check_parsing_result(parser, input_str, outputs): - """ - Check that input_str is correctly parsed by parser - """ - result = parser.parse(input_str.split()) - - assert len(result) == len(outputs) - assert result == outputs - - -def test_parsing_of_config_of_empty_parser(): - """ - Test the parsing of strings with an empty base parser and retrieve the following results: - - - "": {} - - "-z": UnknownArgException(z) - - "-a": UnknownArgException(a) - - "-a --sub toto -b": UnknownArgException(a) - - "-b": UnknownArgException(b) - - Parser description: - - - root parser arguments: None - """ - parser = RootConfigParser(help_arg=False) - - check_parsing_result(parser, '', {}) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-z', None) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-a', None) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-a --sub toto -b', None) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-b', None) - - -def test_parsing_of_config_of_a_parser(): - """ - Test the parsing of strings with a not empty root parser and retrieve the following results: - - - "": {} - - "-z": UnknownArgException(z) - - "-a": {a: True} - - "-a --sub toto -b": UnknownArgException(sub) - - "-b": UnknownArgException(b) - - Parser description: - - - root parser arguments: -a - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', is_flag=True, action=store_true) - - check_parsing_result(parser, '', {}) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-z', None) - - check_parsing_result(parser, '-a', {'a': True}) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-a --sub toto -b', None) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-b', None) - - -def test_parsing_of_config_of_a_parser_with_a_subgroup_parser(): - """ - Test the parsing of strings with a not empty rooy parser and retrieve the following results: - - - "" : {} - - "-z": UnknownArgException(z) - - "-a": {a: True} - - "-a --sub toto -b": NoNameSpecifiedForSubgroupException - - "-a --sub toto -b --name titi" : {a:True, sub: { titi: { 'type': 'toto', b: True}}} - - "-b": BadContextException(b, [toto]) - - Parser description: - - - root parser arguments: -a - - subparser toto bound to the argument sub with sub arguments: -b and --name - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', is_flag=True, action=store_true) - parser.add_subgroup(subgroup_type='sub') - - subparser = SubgroupConfigParser('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - parser.add_subgroup_parser(subgroup_type='sub', subgroup_parser=subparser) - - check_parsing_result(parser, '', {}) - - with pytest.raises(UnknownArgException): - check_parsing_result(parser, '-z', None) - - check_parsing_result(parser, '-a', {'a': True}) - - with pytest.raises(NoNameSpecifiedForSubgroupException): - check_parsing_result(parser, '-a --sub toto -b', {}) - - check_parsing_result(parser, '-a --sub toto -b --name titi', - {'a': True, 'sub': {'titi': {'type': 'toto', 'b': True}}}) - - with pytest.raises(BadContextException): - check_parsing_result(parser, '-b', None) - - -def test_parsing_of_config_with_several_subgroups_with_the_same_name_in_a_parser_with_a_subgroup_parser(): - """ - Test the parsing of several subgroups with the same name. The result is: - {sub:{'titi' : {'type': 'toto'}, 'tutu': {'type': 'toto', 'b':True}, 'tata': {'type': 'toto'}}} - - The subgroups are created with the following cli: - --sub toto --name titi --sub toto -b --name tutu --sub toto --name tata - - Parser description: - - - root parser arguments: None - - subparser toto bound to the argument sub with sub arguments: -b and -n --name - - """ - parser = RootConfigParser(help_arg=False) - parser.add_subgroup(subgroup_type='sub') - - subparser = SubgroupConfigParser('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - parser.add_subgroup_parser(subgroup_type='sub', subgroup_parser=subparser) - - check_parsing_result(parser, '--sub toto --name titi --sub toto -b --name tutu --sub toto --name tata', - {'sub': {'titi': {'type': 'toto'}, 'tutu': {'type': 'toto', 'b': True}, - 'tata': {'type': 'toto'}}}) - - -def test_parsing_of_several_subgroups_with_different_name_in_a_parser_with_several_subgroup_parsers(): - """ - Test the parsing of several subgroups with different name. The result is: - {sub:{'titi' : {'type': 'toto'}, 'tete': {'type': 'tutu'}}} - - The subgroups are created with the following cli: - --sub toto --name titi --sub tutu --name tete --sub tata --name tate - - Parser description: - - - root parser arguments: None - - subparser toto bound to the argument sub with sub argument: -n --name - - subparser tutu bound to the argument sub with sub argument: -n --name - - subparser tata bound to the argument sub with sub argument: -n --name - """ - parser = RootConfigParser(help_arg=False) - - parser.add_subgroup(subgroup_type='sub') - - subparser = SubgroupConfigParser('toto') - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) - - subparser = SubgroupConfigParser('tutu') - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) - - subparser = SubgroupConfigParser('tata') - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) - - check_parsing_result(parser, '--sub toto --name titi --sub tutu --name tete --sub tata --name tate', - {'sub': {'titi': {'type': 'toto'}, 'tete': {'type': 'tutu'}, 'tate': {'type': 'tata'}}}) - - -def test_parsing_of_config_with_repeated_subgroups_names_raise_an_exception(): - """ - Test if an SubgroupAlreadyExistException is raised with two subgroups of the same name. - The subgroups are created with the following cli: - --sub toto --name titi --sub toto --name titi - - Parser description: - - - root parser arguments: None - - subparser toto bound to the argument sub with sub arguments: -b (flag) and -n --name - """ - parser = RootConfigParser(help_arg=False) - parser.add_subgroup(subgroup_type='sub') - - subparser = SubgroupConfigParser('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) - - with pytest.raises(SubgroupAlreadyExistException): - check_parsing_result(parser, '--sub toto --name titi --sub toto --name titi', None) - - -def test_parsing_of_argument_with_val(): - """ - Test the parsing of strings with a root parser and retrieve the following results : - - - "-c" : MissingValue(c) - - "-c 1" : {c : 1} - - Parser description : - - - root parser arguments : -c (not flag) - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('c') - - with pytest.raises(MissingValueException): - check_parsing_result(parser, '-c', None) - - check_parsing_result(parser, '-c 1', {'c': '1'}) - - -# multi name tests # -def test_parsing_of_argument_with_long_short_names_and_val(): - """ - Test if the parsing of an argument with long and short names and a value works correctly. - The value is only bound to the long name in the parsing result. - - Parser description: - - - root parser arguments: -c --coco - - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('c', 'coco') - - check_parsing_result(parser, '-c 1', {'coco': '1'}) - - -def test_add_argument_with_two_long_names_with_same_length_raise_an_exception(): - """ - Test if the parser raise an exception SameLengthArgumentNamesException when a - long argument with the same length is added. No subgroup parse must be added - - Parser description: - - - root parser arguments: None - - """ - parser = RootConfigParser(help_arg=False) - with pytest.raises(SameLengthArgumentNamesException): - parser.add_argument('coco', 'dodo') - - assert len(parser.arguments) == 0 - - -# Type tests # -def test_parsing_arguments_with_val_has_correct_default_type(): - """ - Test if parsing arguments created with default type have default type in the parsing result. - - Parser description: - - - root parser arguments: -a, -b --bb, -c --cc - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a') - parser.add_argument('b', 'bb') - parser.add_argument('c', 'cc') - result = parser.parse('-a 1 --bb string -c string_again'.split()) - assert len(result) == 3 - assert 'a' in result - assert isinstance(result['a'], str) - assert result['a'] == '1' - - assert 'bb' in result - assert isinstance(result['bb'], str) - assert result['bb'] == 'string' - - assert 'cc' in result - assert isinstance(result['cc'], str) - assert result['cc'] == 'string_again' - - -def test_parsing_arguments_with_val_has_correct_type(): - """ - Test if parsing arguments created with no default type have correct type in the parsing result. - - Parser description: - - - root parser arguments: -a, -b --bb, -c --cc - - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', argument_type=int) - result = parser.parse('-a 1'.split()) - assert len(result) == 1 - assert 'a' in result - assert isinstance(result['a'], int) - - -def test_parsing_argument_with_wrong_type_raise_an_exception(): - """ - Test if parsing arguments with a given value that can be parsed to the defined type raises an exception - - Parser description: - - - root parser arguments: -a --xx - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', 'xx', argument_type=int) - - with pytest.raises(BadTypeException): - parser.parse('-a a'.split()) - - with pytest.raises(BadTypeException): - parser.parse('--xx toto'.split()) - - -# parse with ComponentSubparser tests # -def test_add_subgroup_parser_that_already_exist_raise_an_exception(): - """ - Test if adding a subgroup parser that already exists raises an AlreadyAddedArgumentException. - The subgroup parsers of root parser must not be affected - - Parser description: - - - root parser arguments: None - - subparser titi bound to the argument toto with sub arguments: -n - """ - parser = RootConfigParser(help_arg=False) - parser.add_subgroup(subgroup_type='toto') - subparser = SubgroupConfigParser('titi') - subparser.add_argument('n', 'name') - - repeated_subparser = SubgroupConfigParser('titi') - repeated_subparser.add_argument('n', 'name') - repeated_subparser.add_argument('arg2', 'a') - - parser.add_subgroup_parser('toto', subparser) - - with pytest.raises(AlreadyAddedArgumentException): - parser.add_subgroup_parser('toto', repeated_subparser) - - assert len(parser.subgroup_parsers) == 1 - assert len(parser.subgroup_parsers['toto'].subparsers['titi'].arguments) == 2 - - -def test_add_subgroup_parser_with_argument_name_work(): - """ - Test if adding a subgroup parser with a name argument works - Parser description: - - - root parser arguments: None - - subparser titi bound to the argument sub with sub arguments: -a --aaa, -n --name - """ - parser = RootConfigParser(help_arg=False) - parser.add_subgroup(subgroup_type='sub') - subparser = SubgroupConfigParser('titi') - subparser.add_argument('a', 'aaa', is_flag=True, action=store_true, default_value=False) - subparser.add_argument('n', 'name') - parser.add_subgroup_parser('sub', subparser) - - assert len(parser.subgroup_parsers) == 1 - assert len(parser.subgroup_parsers['sub'].subparsers['titi'].arguments) == 4 - assert 'a' in parser.subgroup_parsers['sub'].subparsers['titi'].arguments - assert 'aaa' in parser.subgroup_parsers['sub'].subparsers['titi'].arguments - assert 'n' in parser.subgroup_parsers['sub'].subparsers['titi'].arguments - assert 'name' in parser.subgroup_parsers['sub'].subparsers['titi'].arguments - - -def test_add_subgroup_parser_without_argument_name_raise_an_exception(): - """ - Test if adding a subgroup parser with no argument 'name' raises a - SubgroupParserWithoutNameArgumentException - Parser description: - - - root parser arguments: None - """ - parser = RootConfigParser(help_arg=False) - subparser = SubgroupConfigParser('titi') - - with pytest.raises(SubgroupParserWithoutNameArgumentException): - parser.add_subgroup_parser('toto', subparser) - - assert len(parser.subgroup_parsers) == 0 - - -def test_parsing_empty_string_return_an_empty_dict(): - """ - Test that the result of parsing an empty string is an empty dict - - Parser description: - - - root parser arguments: -a, -b - - """ - parser = RootConfigParser(help_arg=False) - parser.add_argument('a', default_value=1) - parser.add_argument('b', default_value=False, argument_type=bool) - result = parser.parse(''.split()) - assert len(result) == 0 - - -def test_parsing_dict_return_configuration_with_arguments_long_name( - root_config_parser_with_mandatory_and_optional_arguments, - test_files_path): - """ - Test that the result of parsing a dictionary configuration with long and short names for arguments - returns a dictionary configuration only with long names for arguments - - Parser description: - - - root parser arguments: -a, -1 --argument1, -2 --argumento2, --arg3 --argument3, -d --arg4, --arg5 -5 - - """ - config_file = 'root_manager_basic_configuration_with_long_and_short_names.json' - expected_result = load_configuration_from_json_file(config_file) - expected_result['argumento2'] = expected_result.pop('2') - expected_result['arg5'] = expected_result.pop('5') - - result = root_config_parser_with_mandatory_and_optional_arguments.parse_config_dict( - file_name=test_files_path + '/' + config_file) - assert result == expected_result - - -############################ -# SUBGROUP_PARSER TEST # -############################ -def test_subgroup_parser_empty(subgroup_parser): - """ - Test subgroup parser on an empty token list. - Must return an empty dictionary as parse result and an empty token - list - """ - assert subgroup_parser.parse([]) == ([], {}) - - -def test_subgroup_parser_parsing_an_argument(subgroup_parser): - """ - Test component_subparser, parse a token list which contain only subparser - argument [('a', '')]. - - must return return a dictionary {'a':''} as parse result - and an empty token list - - """ - assert subgroup_parser.parse([('a', '')]) == ([], {'a': None}) - - -def test_subgroup_parsing_parser_several_arguments(subgroup_parser): - """ - Test subgroup parser parses a token list which contains subparser - argument and arguments from other parser[('a', ''), ('b', '')]. - - must return a dictionary {'a':''} as parse result - and a token list that contains the unparsed arguments : [('b', '')]. - - """ - assert subgroup_parser.parse([('a', ''), ('b', '')]) == ([('b', '')], - {'a': None}) - - -def test_subgroup_parser_parsing_empty_argument_list_return_an_empty_dict(): - """ - Test that the result of parsing an empty string is an empty dict - """ - subparser = SubgroupConfigParser('toto') - subparser.add_argument('a', default_value=1) - subparser.add_argument('x', default_value=False) - _acc, result = subparser.parse([]) - assert len(result) == 0 - - -def test_normalize_configuration_dict_select_long_names_for_every_argument_in_config(base_config_parser): - """ - Test that every argument in a configuration has at the end its long name after the normalization - """ - conf = load_configuration_from_json_file(file_name='basic_configuration_with_long_and_short_names.json') - expected_conf = load_configuration_from_json_file(file_name='basic_configuration.json') - expected_conf['argumento1'] = expected_conf.pop('arg1') - expected_conf['arg5'] = conf['5'] - expected_conf['dded'] = expected_conf.pop('arg4') - - result = base_config_parser.normalize_configuration(conf=conf) - - assert result == expected_conf - - -def test_normalize_configuration_dict_select_long_names_for_every_argument_in_config_for_root_config_parser( - root_config_parser_with_mandatory_and_optional_arguments): - """ - Test that every argument in a configuration has at the end its long name after the normalization - """ - conf = load_configuration_from_json_file(file_name='basic_configuration_with_long_and_short_names.json') - expected_conf = load_configuration_from_json_file(file_name='basic_configuration.json') - expected_conf['argument1'] = expected_conf.pop('arg1') - expected_conf['arg5'] = conf['5'] - - result = root_config_parser_with_mandatory_and_optional_arguments.normalize_configuration(conf=conf) - - assert result == expected_conf - - -def test_normalize_config_dict_select_long_names_for_every_argument_in_config_with_subgroups_for_root_config_parser( - root_config_parser_with_subgroups): - """ - Test that every argument in a configuration has at the end its long name after the normalization - """ - conf_file = 'basic_configuration_with_subgroups.json' - conf = load_configuration_from_json_file(file_name=conf_file) - expected_conf = load_configuration_from_json_file(file_name=conf_file) - expected_conf['argument1'] = expected_conf.pop('arg1') - expected_conf['argument3'] = expected_conf.pop('arg3') - expected_conf['arg5'] = expected_conf.pop('5') - expected_conf['g2']['g2_sg1']['a4'] = expected_conf['g2']['g2_sg1'].pop('a4') - - result = root_config_parser_with_subgroups.normalize_configuration(conf=conf) - - assert result == expected_conf - - -def test_parse_config_environment_variables_return_correct_configuration(root_config_parser_with_subgroups): - """ - Test that the parsing of environment variables works correctly - """ - conf_file = 'basic_configuration_with_subgroups.json' - - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=conf_file, - simple_argument_prefix=root_config_parser_with_subgroups.arguments_prefix[0], - group_arguments_prefix=root_config_parser_with_subgroups.get_groups_prefixes()) - - expected_conf = load_configuration_from_json_file(file_name=conf_file) - - expected_conf['g1']['g1-sg1'] = {} - expected_conf['g1']['g1-sg2'] = {} - expected_conf['g2']['g2-sg1'] = {} - g1_sg1 = expected_conf['g1'].pop('g1_sg1') - g1_sg2 = expected_conf['g1'].pop('g1_sg2') - g2_sg1 = expected_conf['g2'].pop('g2_sg1') - - expected_conf['g1']['g1-sg1']['a1'] = str(g1_sg1['a1']) - expected_conf['g1']['g1-sg1']['a2'] = bool(g1_sg1['a2']) - expected_conf['g1']['g1-sg1']['a3'] = str(g1_sg1['a3']) - expected_conf['g1']['g1-sg1']['type'] = str(g1_sg1['type']) - - expected_conf['g1']['g1-sg2']['a1'] = str(g1_sg2['a1']) - expected_conf['g1']['g1-sg2']['a2'] = g1_sg2['a2'] - expected_conf['g1']['g1-sg2']['a3'] = g1_sg2['a3'] - expected_conf['g1']['g1-sg2']['type'] = g1_sg2['type'] - - expected_conf['argument1'] = expected_conf.pop('arg1') - expected_conf['argument3'] = expected_conf.pop('arg3') - expected_conf['arg5'] = expected_conf.pop('5') - - expected_conf['g2']['g2-sg1']['a1'] = float(g2_sg1['a1']) - expected_conf['g2']['g2-sg1']['a3'] = g2_sg1['a3'] - expected_conf['g2']['g2-sg1']['a4'] = g2_sg1['a4'] - expected_conf['g2']['g2-sg1']['type'] = g2_sg1['type'] - - result = root_config_parser_with_subgroups.parse_config_environment_variables() - - assert result == expected_conf - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parse_config_environment_variables_with_wrong_argument_raise_an_exception( - root_config_parser_with_subgroups): - """ - Test that the parsing of environment variables raises a BadTypeException with wrong types - """ - conf_file = 'basic_configuration_with_subgroups_wrong_argument_type_value.json' - - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=conf_file, - simple_argument_prefix=root_config_parser_with_subgroups.arguments_prefix[0], - group_arguments_prefix=root_config_parser_with_subgroups.get_groups_prefixes()) - - with pytest.raises(BadTypeException): - _ = root_config_parser_with_subgroups.parse_config_environment_variables() - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_add_subgroup(root_config_parser_with_mandatory_and_optional_arguments): - """ - Test that a subgroup is correctly added - """ - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 0 - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='sub') - - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 1 - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='sub1') - - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 2 - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='sub2') - - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 3 - - root_config_parser_with_mandatory_and_optional_arguments.add_subgroup(subgroup_type='sub3') - - assert len(root_config_parser_with_mandatory_and_optional_arguments.subgroup_parsers) == 4 - - -def test_add_repeated_subgroup_raise_an_exception(root_config_parser_with_subgroups): - """ - Test that adding a repeated subgroup raises an AlreadyAddedSubgroupException - """ - assert len(root_config_parser_with_subgroups.subgroup_parsers) == 2 - - with pytest.raises(AlreadyAddedSubgroupException): - root_config_parser_with_subgroups.add_subgroup(subgroup_type='g2') - - assert len(root_config_parser_with_subgroups.subgroup_parsers) == 2 - - -def test_get_subgroups_prefix(root_config_parser_with_subgroups): - """ - Test that all the subgroups prefixes are returned - """ - expected_prefixes = ['TEST_G1_', 'TEST_G2_'] - - result = root_config_parser_with_subgroups.get_groups_prefixes() - assert len(result) == len(expected_prefixes) - assert result == expected_prefixes - - -def test_get_longest_arguments_names(root_config_parser_with_subgroups): - """ - Test that all the arguments of the parser are returned - """ - expected_arguments_names = ['help', 'a', 'argument1', 'argumento2', 'argument3', 'arg4', 'arg5', 'g1', 'g2'] - - result = root_config_parser_with_subgroups.get_longest_arguments_names() - assert len(result) == len(expected_arguments_names) - assert result == expected_arguments_names diff --git a/tests/unit/cli/test_config_validator.py b/tests/unit/cli/test_config_validator.py deleted file mode 100644 index b169e0ac..00000000 --- a/tests/unit/cli/test_config_validator.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (c) 2023, INRIA -# Copyright (c) 2023, University of Lille -# All rights reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest - -from powerapi.cli import ConfigValidator -from powerapi.exception import NotAllowedArgumentValueException, MissingArgumentException, FileDoesNotExistException, \ - UnexistingActorException, PowerAPIException -from tests.utils.cli.base_config_parser import load_configuration_from_json_file - - -def test_config_in_stream_mode_with_csv_input_raise_an_exception(invalid_csv_io_stream_config): - """ - Test that an invalid configuration with stream mode enabled and csv as input is detected by the ConfigValidator - """ - with pytest.raises(NotAllowedArgumentValueException): - ConfigValidator.validate(invalid_csv_io_stream_config) - - -def test_config_in_postmortem_mode_with_csv_input_is_validated(create_empty_files_from_config, - csv_io_postmortem_config): - """ - Test that a valid configuration is detected by the ConfigValidator when stream mode is disabled. - The files list for the input has to be transformed into a list - """ - try: - expected_result = load_configuration_from_json_file('csv_input_output_stream_mode_enabled_configuration.json') - expected_result['stream'] = False - - ConfigValidator.validate(csv_io_postmortem_config) - assert csv_io_postmortem_config == expected_result - - except NotAllowedArgumentValueException as e: - pytest.fail(f'Invalid argument value: {e}') - - -def test_valid_config_postmortem_csv_input_without_optional_arguments_is_validated(create_empty_files_from_config, - csv_io_postmortem_config_without_optional_arguments): - """ - Test that a valid configuration is detected by the ConfigValidator when stream mode is disabled. - Default values has to be defined and the files list for the input has to be transformed into a list - """ - expected_result = csv_io_postmortem_config_without_optional_arguments.copy() - for current_input in expected_result['input']: - expected_result['input'][current_input]['name'] = 'default_puller' - expected_result['input'][current_input]['model'] = 'HWPCReport' - - expected_result['stream'] = False - expected_result['verbose'] = False - - ConfigValidator.validate(csv_io_postmortem_config_without_optional_arguments) - - assert csv_io_postmortem_config_without_optional_arguments == expected_result - - -def test_config_with_csv_input_with_files_that_do_not_exist_raise_an_exception(csv_io_postmortem_config): - """ - Test that validation of a configuration indicating files that do not exist in csv as input raises a - FileDoesNotExistException - """ - with pytest.raises(FileDoesNotExistException) as raised_exception: - ConfigValidator.validate(csv_io_postmortem_config) - - assert raised_exception.value.file_name == '/tmp/rapl.csv' - - -def test_config_without_inputs_raise_an_exception(config_without_input): - """ - Test that validation of an invalid configuration without inputs raises a MissingArgumentException - """ - with pytest.raises(MissingArgumentException) as raised_exception: - ConfigValidator.validate(config_without_input) - - assert raised_exception.value.argument_name == 'input' - - -def test_config_without_outputs_raise_an_exception(config_without_output): - """ - Test that validation of an invalid configuration without outputs raises a MissingArgumentException - """ - with pytest.raises(MissingArgumentException) as raised_exception: - ConfigValidator.validate(config_without_output) - - assert raised_exception.value.argument_name == 'output' - - -def test_config_with_pre_processor_but_without_puller_raise_an_exception(pre_processor_config_without_puller): - """ - Test that validation of a configuration with pre-processors but without a related puller raises a - MissingArgumentException - """ - with pytest.raises(MissingArgumentException) as raised_exception: - ConfigValidator.validate(pre_processor_config_without_puller) - - assert raised_exception.value.argument_name == 'puller' - - -def test_config_with_empty_pre_processor_pass_validation(empty_pre_processor_config): - """ - Test that validation of a configuration without pre-processors passes validation - """ - try: - ConfigValidator.validate(empty_pre_processor_config) - except MissingArgumentException as e: - pytest.fail(f'Missing argument: {e}') - - -def test_config_with_pre_processor_with_unexisting_puller_actor_raise_an_exception( - pre_processor_with_unexisting_puller_configuration): - """ - Test that validation of a configuration with unexisting actors raise an exception - """ - with pytest.raises(UnexistingActorException) as raised_exception: - ConfigValidator.validate(pre_processor_with_unexisting_puller_configuration) - - assert raised_exception.value.actor == \ - pre_processor_with_unexisting_puller_configuration['pre-processor']['my_processor']['puller'] - - -def test_validation_of_correct_configuration_with_pre_processors(pre_processor_complete_configuration): - """ - Test that a correct configuration with processors and bindings passes the validation - """ - try: - ConfigValidator.validate(pre_processor_complete_configuration) - except PowerAPIException as e: - pytest.fail(f'Configuration validation failed: {e}') - - -def test_validation_of_correct_configuration_without_pre_processors_and_bindings(output_input_configuration): - """ - Test that a correct configuration without pre-processors passes the validation - """ - try: - ConfigValidator.validate(output_input_configuration) - except PowerAPIException as e: - pytest.fail(f'Configuration validation failed: {e}') diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py deleted file mode 100644 index 37b64ffe..00000000 --- a/tests/unit/cli/test_generator.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright (c) 2023, Inria -# Copyright (c) 2023, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import pytest - -from powerapi.cli.generator import ModelNameDoesNotExist -from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, PreProcessorGenerator -from powerapi.database.csv.driver import CSVInputFactory, CSVOutputFactory -from powerapi.database.json.driver import JsonInputFactory, JsonOutputFactory -from powerapi.database.socket.driver import SocketInputFactory -from powerapi.exception import PowerAPIException -from powerapi.filter import BroadcastReportFilter -from powerapi.puller import PullerActor -from powerapi.pusher import PusherActor -from powerapi.report import PowerReport, FormulaReport - - -def test_generate_puller_from_empty_config_dict_raise_an_exception(): - """ - Test that PullerGenerator raises a PowerAPIException when there is no input argument - """ - conf = {} - generator = PullerGenerator(BroadcastReportFilter()) - - with pytest.raises(PowerAPIException): - generator.generate(conf) - - -def test_generate_several_pullers_from_config(several_inputs_outputs_stream_config): - """ - Test that several inputs are correctly used to generate the related actors - """ - generator = PullerGenerator(BroadcastReportFilter()) - pullers = generator.generate(several_inputs_outputs_stream_config) - - assert len(pullers) == len(several_inputs_outputs_stream_config['input']) - - for puller_name, current_puller_infos in several_inputs_outputs_stream_config['input'].items(): - assert puller_name in pullers - assert isinstance(pullers[puller_name], PullerActor) - - db_factory = pullers[puller_name].database_factory - - if current_puller_infos['type'] == 'csv': - assert isinstance(db_factory, CSVInputFactory) - assert db_factory.input_files == current_puller_infos['files'] - elif current_puller_infos['type'] == 'socket': - assert isinstance(db_factory, SocketInputFactory) - assert db_factory.host == current_puller_infos['host'] - assert db_factory.port == current_puller_infos['port'] - elif current_puller_infos['type'] == 'json': - assert isinstance(db_factory, JsonInputFactory) - assert db_factory.output_filepath == current_puller_infos['filepath'] - else: - pytest.fail(f'Unsupported puller type: {current_puller_infos["type"]}') - - -def test_generate_puller_raise_exception_when_missing_arguments_in_socket_input( - several_inputs_outputs_stream_socket_without_some_arguments_config): - """ - Test that PullerGenerator raise a PowerAPIException when some arguments are missing for socket input - """ - generator = PullerGenerator(BroadcastReportFilter()) - - with pytest.raises(PowerAPIException): - generator.generate(several_inputs_outputs_stream_socket_without_some_arguments_config) - - -def test_remove_model_factory_that_does_not_exist_on_a_DBActorGenerator_must_raise_ModelNameDoesNotExist(): - """ - Test that an exception is raised when a model factory that does not exist is erased - """ - generator = DBActorGenerator('input') - num_report_classes = len(generator.report_classes) - - with pytest.raises(ModelNameDoesNotExist): - generator.remove_report_class('model') - - assert len(generator.report_classes) == num_report_classes - - -def test_remove_hwpc_report_model_and_generate_puller_from_a_config_using_model(several_inputs_outputs_stream_config): - """ - PullerGenerator should raise an exception when the model of an input is not defined. - """ - generator = PullerGenerator(BroadcastReportFilter()) - generator.remove_report_class('HWPCReport') - - with pytest.raises(PowerAPIException): - _ = generator.generate(several_inputs_outputs_stream_config) - - -def test_remove_csv_database_factory_and_generate_puller_from_a_config_using_type(several_inputs_outputs_stream_config): - """ - PullerGenerator should raise an exception when the database of an input is not defined. - """ - generator = PullerGenerator(BroadcastReportFilter()) - generator.remove_db_factory('csv') - - with pytest.raises(PowerAPIException): - _ = generator.generate(several_inputs_outputs_stream_config) - - -def test_generate_pusher_from_empty_config_dict_raise_an_exception(): - """ - Test that PusherGenerator raise an exception when there is no output argument - """ - conf = {} - generator = PusherGenerator() - - with pytest.raises(PowerAPIException): - generator.generate(conf) - - -def test_generate_several_pushers_from_config(several_inputs_outputs_stream_config): - """ - Test that several outputs are correctly used to generate the related actors - - """ - generator = PusherGenerator() - pushers = generator.generate(several_inputs_outputs_stream_config) - - assert len(pushers) == len(several_inputs_outputs_stream_config['output']) - - for pusher_name, current_pusher_infos in several_inputs_outputs_stream_config['output'].items(): - assert pusher_name in pushers - assert isinstance(pushers[pusher_name], PusherActor) - - db_factory = pushers[pusher_name].database_factory - pusher_type = current_pusher_infos['type'] - - if pusher_type == 'csv': - assert isinstance(db_factory, CSVOutputFactory) - assert db_factory.output_directory == current_pusher_infos['directory'] - elif pusher_type == 'json': - assert isinstance(db_factory, JsonOutputFactory) - assert db_factory.output_filepath == current_pusher_infos['filepath'] - else: - pytest.fail(f'Unsupported pusher type: {pusher_type}') - - -def test_generate_pusher_report_type_to_actor_mapping(single_input_multiple_outputs_with_different_report_type): - """ - Test generating a report type to actor mapping from a configuration having multiple outputs for different report types. - """ - config = single_input_multiple_outputs_with_different_report_type - generator = PusherGenerator() - actors = generator.generate(config) - report_mapping = generator.generate_report_mapping(config, actors) - - assert set(report_mapping.keys()) == {PowerReport, FormulaReport} - assert [proxy.actor_name for proxy in report_mapping[PowerReport]] == ['powerrep1', 'powerrep2'] - assert [proxy.actor_type for proxy in report_mapping[PowerReport]] == [PusherActor, PusherActor] - assert [proxy.actor_name for proxy in report_mapping[FormulaReport]] == ['formularep'] - assert [proxy.actor_type for proxy in report_mapping[FormulaReport]] == [PusherActor] - - -def test_generate_pre_processor_from_empty_config_dict_raise_an_exception(): - """ - Test that PreProcessGenerator raise an exception when there is no processor argument - """ - conf = {} - generator = PreProcessorGenerator() - - with pytest.raises(PowerAPIException): - generator.generate(conf) diff --git a/tests/unit/cli/test_parsing_manager.py b/tests/unit/cli/test_parsing_manager.py deleted file mode 100644 index 8463b567..00000000 --- a/tests/unit/cli/test_parsing_manager.py +++ /dev/null @@ -1,1272 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys -import pytest - -from powerapi.cli.config_parser import store_true, store_val -from powerapi.cli.parsing_manager import RootConfigParsingManager, \ - SubgroupConfigParsingManager -from powerapi.exception import AlreadyAddedArgumentException, BadTypeException, UnknownArgException, \ - MissingValueException, SubgroupAlreadyExistException, SubgroupParserWithoutNameArgumentException, \ - NoNameSpecifiedForSubgroupException, AlreadyAddedSubparserException, \ - SameLengthArgumentNamesException, BadContextException -from tests.utils.cli.base_config_parser import load_configuration_from_json_file, \ - define_environment_variables_configuration_from_json_file, \ - remove_environment_variables_configuration - - -############### -# PARSER TEST # -############### -def test_add_argument_to_cli_parser_that_already_exist_raise_an_exception(): - """ - Tests if adding an argument that already exists to a parser raises an - AlreadyAddedArgumentException - """ - - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a') - - with pytest.raises(AlreadyAddedArgumentException): - parser_manager.add_argument('a') - - with pytest.raises(AlreadyAddedArgumentException): - parser_manager.add_argument('help') - - assert len(parser_manager.cli_parser.arguments) == 3 # help argument + a argument - - -##################### -# MAIN PARSER TESTS # -##################### -def test_add_flag_arguments_with_short_name_to_cli_parser(): - """ - Test that adding flag arguments with a short name to cli parser modified short_arg string - """ - parser_manager = RootConfigParsingManager() - assert parser_manager.cli_parser.short_arg == 'h' - parser_manager.add_argument('a', is_flag=True) - parser_manager.add_argument('x', is_flag=True) - assert parser_manager.cli_parser.short_arg == 'hax' - - -def test_add_flag_arguments_and_no_flag_arguments_with_short_name_to_cli_parser(): - """ - Test if adding arguments (flag and no flag) to the parser modified short_arg string. - long_arg list is not changed - """ - parser_manager = RootConfigParsingManager() - assert parser_manager.cli_parser.short_arg == 'h' - parser_manager.add_argument('a', is_flag=True) - assert parser_manager.cli_parser.short_arg == 'ha' - parser_manager.add_argument('b') - assert parser_manager.cli_parser.short_arg == 'hab:' - parser_manager.add_argument('c', is_flag=True) - assert parser_manager.cli_parser.short_arg == 'hab:c' - - assert len(parser_manager.cli_parser.long_arg) == 1 # Only help is arg argument - - assert parser_manager.cli_parser.long_arg == ["help"] # Only help is arg argument - - -def test_add_arguments_with_long_name_to_cli_parser(): - """ - Test if adding arguments with long name to cli parser modifies long_arg list. - short_arg string is not changed - """ - parser_manager = RootConfigParsingManager() - assert parser_manager.cli_parser.long_arg == ['help'] - parser_manager.add_argument('aaa') - assert parser_manager.cli_parser.long_arg == ['help', 'aaa='] - parser_manager.add_argument('xx') - assert parser_manager.cli_parser.long_arg == ['help', 'aaa=', 'xx='] - - assert parser_manager.cli_parser.short_arg == 'h' - - -def test_add_flag_arguments_with_long_name_to_cli_parser(): - """ - Test if adding a flag arguments with long to the parser modifies the long_arg list. - short_arg string is not changed - """ - parser_manager = RootConfigParsingManager() - assert parser_manager.cli_parser.long_arg == ['help'] - parser_manager.add_argument('aaa', is_flag=True) - assert parser_manager.cli_parser.long_arg == ['help', 'aaa'] - parser_manager.add_argument('tttt', is_flag=True) - assert parser_manager.cli_parser.long_arg == ['help', 'aaa', 'tttt'] - - assert parser_manager.cli_parser.short_arg == 'h' - - -# full parsing test # -def check_parse_cli_result(parser_manager: RootConfigParsingManager, input_str: str, outputs: dict): - """ - Check that input_str is correctly parsed by parser - """ - result = parser_manager._parse_cli(input_str.split()) - - assert len(result) == len(outputs) - assert result == outputs - - -def test_arguments_string_parsing_with_empty_parsing_manager(): - """ - Test to parse arguments provided as string with a parsing parser and retrieve the following results : - - - "" : {} - - "-z" : UnknownArgException(z) - - "-a" : UnknownArgException(a) - - "-a --sub toto -b" : UnknownArgException(a) - - "-b" : UnknownArgException(b) - - ConfigParsingManager description: - - - base parser arguments : None - """ - parser_manager = RootConfigParsingManager() - - check_parse_cli_result(parser_manager, '', {}) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(parser_manager, '-z', None) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(parser_manager, '-a', None) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(parser_manager, '-a --sub toto -b', None) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(parser_manager, '-b', None) - - -def test_arguments_dict_validation_with_empty_parsing_manager(): - """ - Test validation of arguments dictionary with a parsing manager and retrieve the following results: - - - "" : {} - - "-z value" : UnknownArgException(z) - - "-a 10 " : UnknownArgException(a) - - "-a 10 --sub toto -b" : UnknownArgException(a) - - "-b" : UnknownArgException(b) - - ConfigParsingManager description: - - - base parser arguments : None - """ - parser_manager = RootConfigParsingManager() - dic_z = { - "z": "value" - } - - dic_a = { - "a": 10 - } - - dic_a_sub = { - "a": 10, - "sub": { - "type": "toto", - "b": True - } - } - - dic_b = { - "b": True - } - - with pytest.raises(UnknownArgException): - parser_manager.validate(dic_z) - - with pytest.raises(UnknownArgException): - parser_manager.validate(dic_a) - - with pytest.raises(UnknownArgException): - parser_manager.validate(dic_a_sub) - - with pytest.raises(UnknownArgException): - parser_manager.validate(dic_b) - - -def test_arguments_dict_validation_with_parsing_manager(root_config_parsing_manager): - """ - Test validation of arguments dictionary with a parsing manager and retrieve the following results: - - - "" : {} - - "-z" : UnknownArgException(z) - - "-a" : {a: True} - - "-a --sub toto -b" : UnknownArgException(sub) - - "-b" : UnknownArgException(b) - - ConfigParsingManager description: - - - base parser arguments : -a - """ - - dic_z = { - "z": True - } - - dic_a = { - "a": True - } - - dic_a_sub = { - "a": True, - "sub1": { - "type": "toto", - "b": True - } - } - - dic_b = { - "b": True - } - - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_z) - - assert root_config_parsing_manager.validate(dic_a) == dic_a - - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_a_sub) - - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_b) - - -def test_arguments_string_parsing_with_subgroup_parser_in_subgroup_parsing_manager(root_config_parsing_manager): - """ - Test to parse arguments with a parsing manager containing a subgroup parser. It must retrieve the following - results : - - - "" : {} - - "-z" : UnknownArgException(z) - - "-a" : {a: True} - - "-a --sub toto -b" : NoNameSpecifiedForSubgroupException - - "-a --sub toto -b --name titi" : {a:True, sub: { titi: { 'type': 'toto', b: True}}} - - "-b" : BadContextException(b, [toto]) - - ConfigParsingManager description: - - - base parser arguments: -a - - subgroup parser toto bound to the argument sub with sub arguments: -b and --name - """ - - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - check_parse_cli_result(root_config_parsing_manager, "", {}) - - with pytest.raises(UnknownArgException): - check_parse_cli_result(root_config_parsing_manager, "-z", {}) - - check_parse_cli_result(root_config_parsing_manager, '-a', {'a': True}) - - with pytest.raises(NoNameSpecifiedForSubgroupException): - check_parse_cli_result(root_config_parsing_manager, '-a --sub toto -b', {}) - - check_parse_cli_result(root_config_parsing_manager, '-a --sub toto -b --name titi', - {'a': True, 'sub': {'titi': {'type': 'toto', 'b': True}}}) - - with pytest.raises(BadContextException): - check_parse_cli_result(root_config_parsing_manager, "-b", {}) - - -def test_arguments_dict_validation_with_subgroup_parser_in_subgroup_parsing_manager(root_config_parsing_manager): - """ - Test to validate arguments with a parsing manager containing a subgroup parser. It must retrieve the following - results: - - - "" : {} - - "-z" : UnknownArgException(z) - - "-a" : {a: True} - - "-a --sub toto -b --name titi" : {a:True, sub: { titi: { 'type': 'toto', b: True}}} - - "-b" : UnknownArgException(b) - - ConfigParsingManager description: - - - base parser arguments : -a - - subparser toto bound to the argument sub with sub arguments : -b and --name - """ - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('type', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - dic_a = {'a': True} - - dic_z = { - "z": True - } - - dic_b = { - 'b': "type" - } - - dic_a_sub = { - 'a': True, - 'sub': { - 'titi': - { - 'type': 'toto', - 'b': "type" - } - } - } - - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_z) - - with pytest.raises(UnknownArgException): - root_config_parsing_manager.validate(dic_b) - - assert root_config_parsing_manager.validate(dic_a) == dic_a - - assert root_config_parsing_manager.validate(dic_a_sub) == dic_a_sub - - assert root_config_parsing_manager.validate({}) == {} - - -def test_parsing_of_two_subgroups_of_the_same_type_with_subgroup_parsing_manager(root_config_parsing_manager): - """ - Test the parsing of two subgroups of the same type created with the following cli: - --sub toto --name titi --sub toto -b --name tutu - - The result must be: - {sub:{'titi' : {'type': 'toto'}, 'tutu': {'type': 'toto', 'b':True}}} - - """ - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - check_parse_cli_result(root_config_parsing_manager, '--sub toto --name titi --sub toto -b --name tutu', - {'sub': {'titi': {'type': 'toto'}, 'tutu': {'type': 'toto', 'b': True}}}) - - -def test_validation_of_two_subgroups_of_the_same_type_in_subgroup_parsing_manager(root_config_parsing_manager): - """ - Test the validation of two subgroups of the same type created with the following cli: - --sub toto --name titi --sub toto -b -n 'my_name' - - The result must be: - {sub:{'titi' : {'type': 'toto'}, 'tutu': {'type': 'toto', 'n':'my_name'}}} - - """ - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('type') - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - expected_dic = {'sub': {'titi': {'type': 'toto'}, 'tutu': {'type': 'toto', 'n': 'my_name'}}} - assert root_config_parsing_manager.validate(expected_dic) == expected_dic - - -def test_parsing_of_two_subgroups_of_different_type_in_subgroup_parsing_manager(root_config_parsing_manager): - """ - Test the validation of two subgroups of different type created with the following cli: - Create two component with different type with the following cli : - --sub toto --name titi --sub tutu --name tete - - The result must be: - {sub:{'titi' : {'type': 'toto'}, 'tete': {'type': 'tutu'}}} - - """ - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - subparser = SubgroupConfigParsingManager('tutu') - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - check_parse_cli_result(root_config_parsing_manager, '--sub toto --name titi --sub tutu --name tete', - {'sub': {'titi': {'type': 'toto'}, 'tete': {'type': 'tutu'}}}) - - -def test_parsing_of_repeated_subgroups_in_subgroup_parsing_manager_raise_an_exception(root_config_parsing_manager): - """ - Test the parsing of two subgroups with same type and name created with the following cli: - --sub toto --name titi --sub toto --name titi - - SubgroupAlreadyExistException must be raised - """ - - subparser = SubgroupConfigParsingManager('toto') - subparser.add_argument('b', is_flag=True, action=store_true) - subparser.add_argument('n', 'name') - root_config_parsing_manager.add_subgroup_parser('sub', subparser) - - with pytest.raises(SubgroupAlreadyExistException): - check_parse_cli_result(root_config_parsing_manager, '--sub toto --name titi --sub toto --name titi', None) - - -def test_arguments_string_parsing_with_and_without_val_in_root_parsing_manager(root_config_parsing_manager): - """ - Test to parse arguments with and without value. The expected results are: - - - "-c" : MissingValue(c) - - "-d" : MissingValue(d) - - "-c 1" : {c : 1} - - "-d 10" : {d : 10} - - ConfigParsingManager description: - - - base parser arguments: -a (flag), -c (no flag), -d (int) - """ - - root_config_parsing_manager.add_argument('c') - - root_config_parsing_manager.add_argument('d', argument_type=int) - - with pytest.raises(MissingValueException): - check_parse_cli_result(root_config_parsing_manager, '-c', None) - - with pytest.raises(MissingValueException): - check_parse_cli_result(root_config_parsing_manager, '-d', None) - - check_parse_cli_result(root_config_parsing_manager, '-c 1', {'c': '1'}) - - check_parse_cli_result(root_config_parsing_manager, '-d 10', {'d': 10}) - - -def test_validation_of_arguments_dict_parsing_with_val_in_root_parsing_manager(root_config_parsing_manager): - """ - Test the validation of arguments with value. The expected results are: - - - "-c 1" : {c : 1} - - "-d 89" : {d : 89} - - ConfigParsingManager description: - - - base parser arguments: -a (flag), -c (not flag), -d (int) - """ - root_config_parsing_manager.add_argument('c') - - root_config_parsing_manager.add_argument('d', argument_type=int) - - dic_c = {'c': '1'} - dic_d = {'d': 89} - - assert root_config_parsing_manager.validate(dic_c) == dic_c - - assert root_config_parsing_manager.validate(dic_d) == dic_d - - -def test_arguments_string_parsing_type_checking_in_root_parsing_manager(root_config_parsing_manager): - """ - Test that the type of argument is correctly checked by the parsing manager when a string is used as input - """ - root_config_parsing_manager.add_argument('c', argument_type=int) - - with pytest.raises(BadTypeException): - check_parse_cli_result(root_config_parsing_manager, '-c string', {'c': 'string'}) - - check_parse_cli_result(root_config_parsing_manager, '-c 1', {'c': 1}) - - -def test_validation_of_arguments_dict_type_checking_in_root_parsing_manager(root_config_parsing_manager): - """ - Test that the argument type is correctly validated by the parser when a dict is used as input - """ - root_config_parsing_manager.add_argument('c', argument_type=int) - - str_dic = {'c': 'string'} - int_dic = {'c': 42} - - with pytest.raises(BadTypeException): - root_config_parsing_manager.validate(str_dic) - - assert root_config_parsing_manager.validate(int_dic) == int_dic - - -# multi name tests # -def test_arguments_string_parsing_with_long_and_short_names_in_root_parsing_manager(root_config_parsing_manager): - """ - Test that arguments parsing only relates parsing result to long name in arguments with long and short names - """ - root_config_parsing_manager.add_argument('c', 'coco') - root_config_parsing_manager.add_argument('d', 'xx', argument_type=int) - - check_parse_cli_result(root_config_parsing_manager, '-c 1', {'coco': '1'}) - - check_parse_cli_result(root_config_parsing_manager, '-d 555', {'xx': 555}) - - -def test_add_arguments_with_two_short_names_raise_an_exception_in_root_parsing_manager(root_config_parsing_manager): - """ - Test if adding arguments to a parser with two short names raise a SameLengthArgumentNamesException - The arguments are not added - """ - with pytest.raises(SameLengthArgumentNamesException): - root_config_parsing_manager.add_argument('c', 'd') - - with pytest.raises(SameLengthArgumentNamesException): - root_config_parsing_manager.add_argument('t', 's') - - assert len(root_config_parsing_manager.cli_parser.arguments) == 4 # --help, -h and sub - - assert root_config_parsing_manager.cli_parser.long_arg == ['help', 'sub='] - assert root_config_parsing_manager.cli_parser.short_arg == 'ha' - - -def test_add_arguments_with_two_long_names_raise_an_exception_in_root_parsing_manager(root_config_parsing_manager): - """ - Test if adding arguments to a parser with long names raise a SameLengthArgumentNamesException. - The arguments are not added - """ - with pytest.raises(SameLengthArgumentNamesException): - root_config_parsing_manager.add_argument('coco', 'dodo') - - with pytest.raises(SameLengthArgumentNamesException): - root_config_parsing_manager.add_argument('ddddd', 'plplp') - - assert len(root_config_parsing_manager.cli_parser.arguments) == 4 # -a, --help, -h and sub - - assert root_config_parsing_manager.cli_parser.long_arg == ['help', 'sub='] - assert root_config_parsing_manager.cli_parser.short_arg == 'ha' - - -# Type tests # -def test_add_argument_with_default_type_in_root_parsing_manager(): - """ - Test if adding arguments without type has string (default type) as type - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a') - parser_manager.add_argument('b') - result_a = parser_manager.parse('python -a 1'.split()) - result_b = parser_manager.parse('python3 -b string'.split()) - assert len(result_a) == 1 - assert 'a' in result_a - assert isinstance(result_a['a'], str) - - assert len(result_b) == 1 - assert 'b' in result_b - assert isinstance(result_b['b'], str) - - -def test_add_argument_with_type_in_root_parsing_manager(): - """ - Test if adding arguments with a type have currently this type - - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a', argument_type=int) - parser_manager.add_argument('b', argument_type=bool) - - result = parser_manager.parse('python -a 1'.split()) - assert len(result) == 1 - assert 'a' in result - assert isinstance(result['a'], int) - - result = parser_manager.parse('python3 -b false'.split()) - assert len(result) == 1 - assert 'b' in result - assert isinstance(result['b'], bool) - - -def test_parsing_of_arguments_string_with_wrong_type_raise_an_exception_in_root_parsing_manager(): - """ - Test that parsing arguments with a wrong value type raises a BadTypeException - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a', argument_type=int) - - with pytest.raises(BadTypeException): - parser_manager._parse_cli('-a a'.split()) - - -# parse with Subparser tests # -def test_add_subgroup_parser_that_already_exists_raises_an_exception_in_root_parsing_manager(): - """ - Test that adding a subgroup parser that already exists raises an - AlreadyAddedSubparserException - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_subgroup(name='toto') - subparser = SubgroupConfigParsingManager('titi') - subparser.add_argument('n', 'name') - parser_manager.add_subgroup_parser('toto', subparser) - - repeated_subparser = SubgroupConfigParsingManager('titi') - repeated_subparser.add_argument('n', 'name') - - with pytest.raises(AlreadyAddedSubparserException): - parser_manager.add_subgroup_parser('toto', repeated_subparser) - - -def test_parsing_of_arguments_string_with_subgroup_parser_with_long_and_short_arguments_names_in_root_parsing_manager(): - """ - Tests that parsing arguments of a subgroup parser with long and short names arguments - only binds parser results to the long name - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_subgroup(name='sub') - subparser = SubgroupConfigParsingManager('titi') - subparser.add_argument('a', 'aaa', is_flag=True, action=store_true, default_value=False) - subparser.add_argument('c', 'ttt', is_flag=False, action=store_val, argument_type=int) - subparser.add_argument('n', 'name') - parser_manager.add_subgroup_parser('sub', subparser) - check_parse_cli_result(parser_manager, '--sub titi -a --name tutu -c 15', - {'sub': {'tutu': {'aaa': True, 'type': 'titi', 'ttt': 15}}}) - - -def test_add_subgroup_parser_without_name_argument_raise_an_exception_in_root_parsing_manager(): - """ - Test that adding a subgroup parser with no argument 'name' raises a - SubgroupParserWithoutNameArgumentException - """ - parser = RootConfigParsingManager() - subparser = SubgroupConfigParsingManager('titi') - - with pytest.raises(SubgroupParserWithoutNameArgumentException): - parser.add_subgroup_parser('toto', subparser) - - -def test_parsing_empty_string_return_empty_configuration_in_root_parsing_manager(): - """ - Test that the result of parsing an empty string is a empty dict - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('a', default_value=1) - parser_manager.add_argument('xxx', default_value='val') - result = parser_manager._parse_cli(''.split()) - assert len(result) == 0 - - -def test_validate_empty_dict_return_default_values_of_arguments_in_root_parsing_manager(): - """ - Test that the result of parsing an empty dict is a dict of arguments with their default value - """ - parser_manager = RootConfigParsingManager() - parser_manager.add_argument('c', argument_type=int, default_value=1) - parser_manager.add_argument('hello', argument_type=str, default_value="world") - - default_dic = {} - expected_dic = {'c': 1, 'hello': 'world'} - - assert parser_manager.validate(default_dic) == expected_dic - - -def test_parsing_configuration_file_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration is correctly parsed - """ - config_file = 'root_manager_basic_configuration.json' - expected_dict = load_configuration_from_json_file(config_file) - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=f'--config-file {test_files_path}/{config_file}'.split()) - assert result == expected_dict - - -def test_parsing_configuration_file_with_long_and_short_names_for_arguments_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration with long and short names for arguments is correctly parsed - """ - config_file = 'root_manager_basic_configuration_with_long_and_short_names.json' - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['argumento2'] = expected_dict.pop('2') - expected_dict['arg5'] = expected_dict.pop('5') - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=f'--config-file {test_files_path}/{config_file}'.split()) - assert result == expected_dict - - -def test_parsing_configuration_file_with_no_argument_with_default_value_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration with no values for arguments with default values - is correctly parsed - """ - config_file = 'root_manager_basic_configuration_with_no_argument_with_default_value.json' - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['arg5'] = 'default value' - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=f'--config-file {test_files_path}/{config_file}'.split()) - - assert result == expected_dict - - -def test_parsing_configuration_file_with_unknown_argument_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration with unknown arguments stops execution of the application - """ - config_file = 'root_manager_basic_configuration_with_unknown_argument.json' - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=f'--config-file {test_files_path}/{config_file}'.split()) - - assert result.type is SystemExit - assert result.value.code == -1 - - -def test_parsing_configuration_file_with_wrong_argument_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a json file containing a configuration with unknown arguments stops execution of the application - """ - config_file = 'root_manager_basic_configuration_with_argument_type_value.json' - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse(args=('--config-file ' + test_files_path + '/' + config_file).split()) - - assert result.type is SystemExit - assert result.value.code == -1 - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration.json']) -def test_parsing_cli_configuration_in_root_parsing_manager( - config_file, - root_config_parsing_manager_with_mandatory_and_optional_arguments, - cli_configuration): - """ - Test that a list of strings containing a configuration is correctly parsed - """ - expected_dict = load_configuration_from_json_file(config_file) - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - assert result == expected_dict - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_long_and_short_names.json']) -def test_parsing_cli_configuration_with_long_and_short_names_for_arguments_in_root_parsing_manager( - config_file, cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that a list of strings containing a configuration with long and short names for arguments is correctly parsed - """ - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['argumento2'] = expected_dict.pop('2') - expected_dict['arg5'] = expected_dict.pop('5') - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - assert result == expected_dict - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_no_argument_with_default_value.json']) -def test_parsing_cli_configuration_with_no_argument_with_default_value_in_root_parsing_manager( - config_file, cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that a list of strings containing a configuration with no values for arguments with default values - is correctly parsed - """ - expected_dict = load_configuration_from_json_file(config_file) - - expected_dict['arg5'] = 'default value' - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_unknown_argument.json']) -def test_parsing_cli_configuration_with_unknown_argument_terminate_execution_in_root_parsing_manager( - config_file, cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path): - """ - Test that a list of strings containing a configuration with unknown arguments stops execution of the application - """ - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result.type is SystemExit - assert result.value.code == -1 - - -def test_parsing_environment_variables_configuration_in_root_parsing_manager( - empty_cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that a list of environment variables containing a configuration is correctly parsed - """ - config_file = 'root_manager_basic_configuration.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_unknown_argument_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that a list of environment variables containing a configuration is correctly parsed - """ - config_file = 'root_manager_basic_configuration_with_unknown_argument.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result.type is SystemExit - assert result.value.code == -1 - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_long_and_short_names_for_arguments_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a configuration defined via environment variables with long and short names for arguments is correctly - parsed - """ - config_file = 'root_manager_basic_configuration_with_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['argumento2'] = expected_dict.pop('2') - expected_dict['arg5'] = expected_dict.pop('5') - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_no_argument_with_default_value_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that the parsing of a configuration defined via environment variables missing arguments with - default values results in a dict with the default values for those arguments - """ - config_file = 'root_manager_basic_configuration_with_no_argument_with_default_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['arg5'] = 'default value' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_no_argument_with_default_value.json']) -def test_configuration_priority_between_cli_and_environment_variables_in_root_parsing_manager( - config_file, cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments): - """ - Test that arguments values defined via the CLI are preserved regarding values defined via environment variables - """ - config_file_environment_variables = 'root_manager_basic_configuration.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict["arg5"] = "this is a value" # This value is not defined by the CLI but it has to be present - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_no_argument_with_default_value.json']) -def test_configuration_priority_between_cli_and_configuration_file_in_root_parsing_manager(config_file, - cli_configuration, - root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, - monkeypatch): - """ - Test that arguments values defined via the CLI are preserved regarding values defined via a configuration file - """ - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_basic_configuration.json']) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict["arg5"] = "this is a value" # This value is not defined by the CLI but it has to be present - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - -def test_configuration_priority_between_environment_variables_and_configuration_file_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): - """ - Test that arguments values defined via environment variables are preserved regarding values defined via - a configuration file - """ - config_file_environment_variables = 'root_manager_basic_configuration_with_no_argument_with_default_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_basic_configuration.json']) - - expected_dict = load_configuration_from_json_file(config_file_environment_variables) - expected_dict["arg5"] = "this is a value" # This value is not defined by the CLI but it has to be present - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', ['root_manager_basic_configuration_with_no_argument_with_default_value.json']) -def test_configuration_priority_between_cli_environment_variables_and_configuration_file_in_root_parsing_manager( - config_file, cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): - """ - Test the following argument definition priority: - 1. CLI - 2. Environment Variables - 3. Configuration file - """ - - config_file_environment_variables = 'root_manager_basic_configuration_with_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_basic_configuration.json']) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict["arg5"] = "this is a value 3" - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_subgroups_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a configuration defined via environment variables with subgroups is correctly parsed - """ - config_file = 'root_manager_configuration_with_subgroups.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_subgroups_and_long_and_short_names_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a configuration defined via environment variables with subgroups is correctly parsed - """ - config_file = 'root_manager_configuration_with_subgroups_and_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file('root_manager_configuration_with_subgroups.json') - expected_dict['input']['in1']['name'] = 'i1_name' - expected_dict['output']['o1']['model'] = 'o1_model_x' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_subgroups_and_unknown_arguments_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a configuration defined via environment variables with subgroups and unknown arguments terminates - the execution - """ - config_file = 'root_manager_configuration_with_subgroups_and_unknown_arguments.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result.type is SystemExit - assert result.value.code == -1 - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_subgroups_and_no_arguments_with_default_value_in_root_parsing_manager( - empty_cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a configuration defined via environment variables with subgroups without variables with default values - is correctly parsed - """ - config_file = 'root_manager_configuration_with_subgroups_and_no_argument_default_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['input']['in1']['name'] = 'my_i1_instance' - expected_dict['output']['o1']['name'] = 'my_o1_instance' - expected_dict['output']['o2']['name'] = 'my_o2_instance' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_parsing_environment_variables_with_subgroups_and_wrong_type_terminate_execution_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, test_files_path): - """ - Test that a configuration defined via environment variables with subgroups and wrong argument type terminates - the execution - """ - config_file = 'root_manager_configuration_with_subgroups_and_wrong_argument_type_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result.type is SystemExit - assert result.value.code == -1 - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', - ['root_manager_configuration_with_subgroups_and_no_argument_default_value.json']) -def test_config_priority_between_cli_environ_variables_and_configuration_file_with_subgroups_in_root_parsing_manager( - config_file, cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): - """ - Test the following argument definition priority for a configuration with subgroups: - 1. CLI - 2. Environment Variables - 3. Configuration file - """ - - config_file_environment_variables = 'root_manager_configuration_with_subgroups_and_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_configuration_with_subgroups.json']) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['input']['in1']['name'] = 'i1_name' - expected_dict['output']['o1']['name'] = 'o1_name' - expected_dict['output']['o2']['name'] = 'o2_name' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', - ['root_manager_configuration_with_subgroups_and_no_argument_default_value.json']) -def test_config_priority_between_cli_and_environ_variables_with_subgroups_in_root_parsing_manager( - config_file, cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path): - """ - Test that arguments values defined via the CLI are preserved regarding values defined via environment variables - with subgroups in configuration - """ - - config_file_environment_variables = 'root_manager_configuration_with_subgroups_and_long_and_short_names.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - expected_dict = load_configuration_from_json_file(file_name=config_file) - expected_dict['input']['in1']['name'] = 'i1_name' - expected_dict['output']['o1']['name'] = 'o1_name' - expected_dict['output']['o2']['name'] = 'o2_name' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -@pytest.mark.parametrize('config_file', - ['root_manager_configuration_with_subgroups_and_no_argument_default_value.json']) -def test_config_priority_between_cli_and_configuration_file_with_subgroups_in_root_parsing_manager( - config_file, cli_configuration, root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): - """ - Test that arguments values defined via the CLI are preserved regarding values defined via a config file - with subgroups in configuration - """ - monkeypatch.setattr(sys, 'argv', [*sys.argv, '--config-file', test_files_path + '/root_manager_configuration_with_subgroups.json']) - - expected_dict = load_configuration_from_json_file(config_file) - expected_dict['input']['in1']['name'] = 'in1_name' - expected_dict['output']['o1']['name'] = 'o1_name' - expected_dict['output']['o2']['name'] = 'o2_name' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - -def test_config_priority_between_environ_variables_and_configuration_file_with_subgroups_in_root_parsing_manager( - root_config_parsing_manager_with_mandatory_and_optional_arguments, - test_files_path, monkeypatch): - """ - Test that arguments values defined via the environment variables are preserved regarding values defined via a config - file with subgroups in configuration - """ - - config_file_environment_variables = 'root_manager_configuration_with_subgroups_and_no_argument_default_value.json' - created_environment_variables = define_environment_variables_configuration_from_json_file( - file_name=config_file_environment_variables, - simple_argument_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - arguments_prefix[0], - group_arguments_prefix=root_config_parsing_manager_with_mandatory_and_optional_arguments.cli_parser. - get_groups_prefixes()) - - monkeypatch.setattr(sys, 'argv', ['--config-file', test_files_path + '/root_manager_configuration_with_subgroups_and_long_and_short_names.json']) - - expected_dict = load_configuration_from_json_file(config_file_environment_variables) - expected_dict['input']['in1']['name'] = 'i1_name' - expected_dict['output']['o1']['name'] = 'o1_name' - expected_dict['output']['o2']['name'] = 'o2_name' - - result = root_config_parsing_manager_with_mandatory_and_optional_arguments.parse() - - assert result == expected_dict - - remove_environment_variables_configuration(variables_names=created_environment_variables) - - -def test_add_subgroup_in_root_parsing_manager(): - """ - Test that a subgroup is correctly added to a parsing manager - """ - parser_manager = RootConfigParsingManager() - - assert len(parser_manager.cli_parser.subgroup_parsers) == 0 - - parser_manager.add_subgroup(name='sub') - - assert len(parser_manager.cli_parser.subgroup_parsers) == 1 - - parser_manager.add_subgroup(name='sub1') - - assert len(parser_manager.cli_parser.subgroup_parsers) == 2 - - parser_manager.add_subgroup(name='sub3') - - assert len(parser_manager.cli_parser.subgroup_parsers) == 3 - - -def test_add_repeated_subgroup_terminate_execution_in_root_parsing_manager(root_config_parsing_manager): - """ - Test that adding a repeated terminates the execution - """ - with pytest.raises(SystemExit) as result: - _ = root_config_parsing_manager.add_subgroup(name='sub') - - assert result.type is SystemExit - assert result.value.code == -1 diff --git a/tests/unit/cli/__init__.py b/tests/unit/config/__init__.py similarity index 100% rename from tests/unit/cli/__init__.py rename to tests/unit/config/__init__.py diff --git a/tests/unit/config/conftest.py b/tests/unit/config/conftest.py new file mode 100644 index 00000000..ca72e00e --- /dev/null +++ b/tests/unit/config/conftest.py @@ -0,0 +1,92 @@ +# Copyright (c) 2023, Inria +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest + +from tests.utils.cli.base_config_parser import load_configuration_from_json_file + + +@pytest.fixture +def several_inputs_outputs_stream_config(): + """ + Configuration with several inputs and outputs and stream mode enabled + """ + return load_configuration_from_json_file('several_inputs_outputs_stream_mode_enabled_configuration.json') + + +@pytest.fixture +def several_inputs_outputs_postmortem_config(several_inputs_outputs_stream_config): + """ + Configuration with several inputs and outputs and stream mode disabled. + """ + several_inputs_outputs_stream_config['stream'] = False + return several_inputs_outputs_stream_config + + +@pytest.fixture +def single_input_multiple_outputs_with_different_report_type(): + """ + Configuration with several inputs and outputs and stream mode enabled + """ + return load_configuration_from_json_file('single_input_multiple_outputs_with_different_report_type_configuration.json') + + +@pytest.fixture +def output_input_configuration(): + """ + Return a dictionary containing bindings with a processor + """ + return load_configuration_from_json_file(file_name='output_input_configuration.json') + + +@pytest.fixture(params=['k8s_pre_processor_complete_configuration.json']) +def pre_processor_complete_configuration(request): + """ + Return a dictionary containing a configuration with pre-processor + """ + return load_configuration_from_json_file(file_name=request.param) + + +@pytest.fixture +def empty_pre_processor_config(pre_processor_complete_configuration): + """ + Return a configuration with bindings but without processors + """ + + pre_processor_complete_configuration.pop('pre-processor') + + return pre_processor_complete_configuration + + +@pytest.fixture(params=['k8s_pre_processor_with_non_existing_puller_configuration.json']) +def pre_processor_with_unexisting_puller_configuration(request): + """ + Return a dictionary containing a pre-processor with a puller that doesn't exist + """ + return load_configuration_from_json_file(request.param) diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/config/test_binding_manager.py similarity index 98% rename from tests/unit/cli/test_binding_manager.py rename to tests/unit/config/test_binding_manager.py index 97b8f7e2..738cac3a 100644 --- a/tests/unit/cli/test_binding_manager.py +++ b/tests/unit/config/test_binding_manager.py @@ -32,8 +32,8 @@ import pytest from powerapi.actor import ActorProxy -from powerapi.cli.binding_manager import PreProcessorBindingManager -from powerapi.cli.generator import PreProcessorGenerator, PullerGenerator +from powerapi.config.binding_manager import PreProcessorBindingManager +from powerapi.config.generator import PreProcessorGenerator, PullerGenerator from powerapi.dispatcher import DispatcherActor from powerapi.exception import UnexistingActorException, UnsupportedActorTypeException, TargetActorAlreadyUsed from powerapi.filter import BroadcastReportFilter diff --git a/tests/unit/config/test_cli_parser.py b/tests/unit/config/test_cli_parser.py new file mode 100644 index 00000000..269f787b --- /dev/null +++ b/tests/unit/config/test_cli_parser.py @@ -0,0 +1,318 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse + +import pytest + +from powerapi.config import cli_parser +from powerapi.config.cli_parser import CLIArgumentParser, CLIParseException +from powerapi.config.config_parser import ( + ComponentSchema, + ConfigurationSchema, + ConfigurationSectionSchema, +) + + +def create_parser() -> CLIArgumentParser: + """ + Create a CLI parser backed by an empty configuration schema. + :return: CLI parser accepting only built-in options. + """ + return CLIArgumentParser(ConfigurationSchema()) + + +def test_configuration_help_uses_plain_theme_without_argparse_theme(monkeypatch): + """ + Test that configuration help falls back to plain text when argparse has no theme. + """ + monkeypatch.setattr(cli_parser.argparse, 'RawDescriptionHelpFormatter', lambda _: object()) + + assert cli_parser._get_argparse_theme() == cli_parser._HelpTheme() + + +def test_cli_argument_parser_preserves_equals_in_assignment_value(): + """ + Test that assignment values preserve equals signs after the first separator. + """ + result = create_parser().parse([ + '-C', 'input.mongo.uri=mongodb://localhost/?option=value', + ]) + + assert result.configuration == { + 'input': {'mongo': {'uri': 'mongodb://localhost/?option=value'}}, + } + + +@pytest.mark.parametrize('expression', [ + 'input.mongo.uri', + '=value', + 'input..uri=value', + '.input=value', + 'input.sensor=value', + 'input.sensor.connection.uri=value', +]) +def test_cli_argument_parser_rejects_invalid_assignment(expression): + """ + Test that malformed or unsupported assignment paths are rejected. + """ + with pytest.raises(CLIParseException): + create_parser().parse(['-C', expression]) + + +def test_cli_argument_parser_builds_nested_configuration(): + """ + Test that root and component assignments build a nested configuration. + """ + result = create_parser().parse([ + '-C', 'input.sensor.type=socket', + '-C', 'input.sensor.port=9080', + '-C', 'stream=false', + ]) + + assert result.configuration == { + 'input': { + 'sensor': { + 'type': 'socket', + 'port': '9080', + }, + }, + 'stream': 'false', + } + + +def test_cli_argument_parser_uses_last_repeated_value(): + """ + Test that the last assignment to a path replaces previous values. + """ + result = create_parser().parse([ + '-C', 'input.sensor.port=9080', + '-C', 'input.sensor.port=9090', + ]) + + assert result.configuration['input']['sensor']['port'] == '9090' + + +@pytest.mark.parametrize('expressions', [ + ['input=value', 'input.sensor.port=9090'], + ['input.sensor.port=9090', 'input=value'], +]) +def test_cli_argument_parser_rejects_structural_conflict(expressions): + """ + Test that scalar and nested assignments cannot target the same path. + """ + arguments = [argument for expression in expressions for argument in ('-C', expression)] + + with pytest.raises(CLIParseException): + create_parser().parse(arguments) + + +def test_cli_argument_parser_parses_repeatable_config_assignments(): + """ + Test repeatable assignment options and configuration-file selection. + """ + parser = create_parser() + + result = parser.parse([ + '-C', 'input.sensor.type=socket', + '--set-config', 'input.sensor.port=9080', + '--config-file', '/tmp/powerapi.json', + ]) + + assert result.config_file == '/tmp/powerapi.json' + assert result.configuration == { + 'input': { + 'sensor': { + 'type': 'socket', + 'port': '9080', + }, + }, + } + + +def test_cli_argument_parser_keeps_internal_values_separate_from_root_arguments(): + """ + Test that internal parser destinations do not collide with schema properties. + """ + schema = ConfigurationSchema() + schema.add_argument('config_assignments') + schema.add_argument('config_file') + parser = CLIArgumentParser(schema) + + result = parser.parse([ + '--config_assignments', 'root-value', + '--config_file', 'another-root-value', + '-C', 'stream=false', + '--config-file', '/tmp/powerapi.json', + ]) + + assert result.config_file == '/tmp/powerapi.json' + assert result.configuration == { + 'config_assignments': 'root-value', + 'config_file': 'another-root-value', + 'stream': 'false', + } + + +def test_cli_argument_parser_reports_argument_conflict_as_cli_error(): + """ + Test that schema option conflicts are reported as CLIParseException. + """ + schema = ConfigurationSchema() + schema.add_argument('config-file') + parser = CLIArgumentParser(schema) + + with pytest.raises(CLIParseException) as result: + parser.parse([]) + + assert result.value.msg == 'Failed to add argument: argument --config-file: conflicting option string: --config-file' + assert isinstance(result.value.__cause__, argparse.ArgumentError) + + +def test_cli_argument_parser_does_not_add_absent_root_argument(): + """ + Test that omitted root options are absent from the raw CLI configuration. + """ + schema = ConfigurationSchema() + schema.add_argument('stream', is_flag=True) + parser = CLIArgumentParser(schema) + + result = parser.parse([]) + + assert 'stream' not in result.configuration + + +def test_cli_argument_parser_parses_registered_root_flag(): + """ + Test that a registered root flag produces its configuration property. + """ + schema = ConfigurationSchema() + schema.add_argument('stream', is_flag=True) + parser = CLIArgumentParser(schema) + + result = parser.parse(['--stream']) + + assert result.configuration == {'stream': True} + + +def test_cli_argument_parser_keeps_non_flag_root_value_raw(): + """ + Test that non-flag root values remain raw until schema validation. + """ + schema = ConfigurationSchema() + schema.add_argument('port') + parser = CLIArgumentParser(schema) + + result = parser.parse(['--port', '9080']) + + assert result.configuration == {'port': '9080'} + + +def test_cli_argument_parser_preserves_root_argument_underscores(): + """ + Test that underscores in root option names are preserved. + """ + schema = ConfigurationSchema() + schema.add_argument('some_value') + parser = CLIArgumentParser(schema) + + result = parser.parse(['--some_value', 'test']) + + assert result.configuration == {'some_value': 'test'} + + +def test_cli_argument_parser_rejects_contextual_argument(): + """ + Test that legacy contextual component options are rejected. + """ + parser = create_parser() + + with pytest.raises(CLIParseException) as result: + parser.parse(['--input', 'socket']) + + assert result.value.msg == 'Failed to parse CLI: unrecognized arguments: --input socket' + assert isinstance(result.value.__cause__, argparse.ArgumentError) + + +def test_cli_argument_parser_prints_help(capsys): + """ + Test that --help prints usage information and exits successfully. + """ + parser = create_parser() + + with pytest.raises(SystemExit) as result: + parser.parse(['--help']) + + help_message = capsys.readouterr().out + + assert result.value.code == 0 + assert help_message.startswith('usage:') + assert '-C PATH=VALUE' in help_message + assert '--config-file FILE' in help_message + + +def test_cli_argument_parser_help_contains_schema_paths(capsys): + """ + Test that help exposes canonical schema paths and supported component types. + """ + schema = ConfigurationSchema() + schema.add_argument('verbose', is_flag=True) + schema.add_group('input') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int) + schema.add_component('input', component) + schema.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + schema.add_section('formula', 'smartwatts', smartwatts) + + with pytest.raises(SystemExit): + CLIArgumentParser(schema).parse(['--help']) + + help_lines = { + line.strip() + for line in capsys.readouterr().out.splitlines() + if line.strip() + } + expected_lines = { + 'verbose', + 'input.NAME.type', + 'Supported types: socket', + 'input.NAME.type=socket', + 'input.NAME.port', + 'formula.smartwatts.learn-error-window-size', + } + excluded_lines = { + 'v', + 'input.NAME.p', + 'formula.smartwatts.w', + 'formula.smartwatts.type', + } + + assert expected_lines.issubset(help_lines) + assert excluded_lines.isdisjoint(help_lines) diff --git a/tests/unit/cli/test_common_cli_parsing_manager.py b/tests/unit/config/test_common_cli_parsing_manager.py similarity index 52% rename from tests/unit/cli/test_common_cli_parsing_manager.py rename to tests/unit/config/test_common_cli_parsing_manager.py index ca15d68d..5e445c28 100644 --- a/tests/unit/cli/test_common_cli_parsing_manager.py +++ b/tests/unit/config/test_common_cli_parsing_manager.py @@ -28,10 +28,15 @@ import pytest -from powerapi.cli.common_cli_parsing_manager import CommonCLIParsingManager, PullerConfigParsingManager, \ - PusherConfigParsingManager, PreProcessorConfigParsingManager, generate_env_prefix -from powerapi.cli.config_parser import store_true -from powerapi.exception import NoNameSpecifiedForSubgroupException +from powerapi.config.cli_parser import CLIParseException +from powerapi.config.common_cli_parsing_manager import ( + CommonCLIParsingManager, + PreProcessorSchema, + PullerSchema, + PusherSchema, + generate_env_prefix, +) +from powerapi.exception import ConfigurationError def test_generate_env_prefix_with_no_component(): @@ -76,69 +81,36 @@ def test_generate_env_prefix_with_custom_root_prefix(): assert generate_env_prefix('INPUT', root_prefix='MYAPP') == 'MYAPP_INPUT_' -def test_puller_config_parser_registers_shared_name_argument(): +def test_puller_config_schema_registers_default_model_argument(): """ - Test that PullerConfigParsingManager registers the shared subgroup name argument. + Test that PullerSchema registers the default report model argument. """ - parser = PullerConfigParsingManager('pytest') - name_argument = parser.cli_parser.get_arguments()['name'] + schema = PullerSchema('pytest') + model_argument = schema.arguments['model'] - assert name_argument.names == ['n', 'name'] - assert name_argument.is_mandatory is False - - -def test_puller_config_parser_registers_default_model_argument(): - """ - Test that PullerConfigParsingManager registers the default report model argument. - """ - parser = PullerConfigParsingManager('pytest') - model_argument = parser.cli_parser.get_arguments()['model'] - - assert model_argument.names == ['m', 'model'] + assert model_argument.name == 'model' assert model_argument.default_value == 'HWPCReport' -def test_pusher_config_parser_registers_shared_name_argument(): - """ - Test that PusherConfigParsingManager registers the shared subgroup name argument. - """ - parser = PusherConfigParsingManager('pytest') - name_argument = parser.cli_parser.get_arguments()['name'] - - assert name_argument.names == ['n', 'name'] - assert name_argument.is_mandatory is False - - -def test_pusher_config_parser_registers_default_model_argument(): +def test_pusher_config_schema_registers_default_model_argument(): """ - Test that PusherConfigParsingManager registers the default report model argument. + Test that PusherSchema registers the default report model argument. """ - parser = PusherConfigParsingManager('pytest') - model_argument = parser.cli_parser.get_arguments()['model'] + schema = PusherSchema('pytest') + model_argument = schema.arguments['model'] - assert model_argument.names == ['m', 'model'] + assert model_argument.name == 'model' assert model_argument.default_value == 'PowerReport' -def test_pre_processor_config_parser_registers_shared_name_argument(): +def test_pre_processor_config_schema_registers_mandatory_puller_argument(): """ - Test that PreProcessorConfigParsingManager registers the shared subgroup name argument. + Test that PreProcessorSchema registers a mandatory puller argument. """ - parser = PreProcessorConfigParsingManager('pytest') - name_argument = parser.cli_parser.get_arguments()['name'] - - assert name_argument.names == ['n', 'name'] - assert name_argument.is_mandatory is False - + schema = PreProcessorSchema('pytest') + puller_argument = schema.arguments['puller'] -def test_pre_processor_config_parser_registers_mandatory_puller_argument(): - """ - Test that PreProcessorConfigParsingManager registers a mandatory puller argument. - """ - parser = PreProcessorConfigParsingManager('pytest') - puller_argument = parser.cli_parser.get_arguments()['puller'] - - assert puller_argument.names == ['p', 'puller'] + assert puller_argument.name == 'puller' assert puller_argument.is_mandatory is True @@ -146,32 +118,32 @@ def test_common_cli_manager_registers_root_environment_prefix(): """ Test that CommonCLIParsingManager registers the root PowerAPI environment prefix. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert parser_manager.cli_parser.arguments_prefix == ['POWERAPI_'] + assert manager.schema.arguments_prefix == ['POWERAPI_'] -def test_common_cli_manager_registers_subgroup_environment_prefixes(): +def test_common_cli_manager_registers_group_environment_prefixes(): """ - Test that CommonCLIParsingManager registers every subgroup environment prefix. + Test that CommonCLIParsingManager registers every group environment prefix. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert parser_manager.cli_parser.get_groups_prefixes() == [ - 'POWERAPI_INPUT_', - 'POWERAPI_OUTPUT_', - 'POWERAPI_PRE_PROCESSOR_', - 'POWERAPI_POST_PROCESSOR_', - ] + assert {name: group.prefix for name, group in manager.schema.groups.items() if group.prefix} == { + 'input': 'POWERAPI_INPUT_', + 'output': 'POWERAPI_OUTPUT_', + 'pre-processor': 'POWERAPI_PRE_PROCESSOR_', + 'post-processor': 'POWERAPI_POST_PROCESSOR_', + } -def test_common_cli_manager_registers_top_level_subgroups(): +def test_common_cli_manager_registers_top_level_groups(): """ - Test that CommonCLIParsingManager registers every top-level subgroup. + Test that CommonCLIParsingManager registers every top-level group. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert set(parser_manager.cli_parser.subgroup_parsers) == { + assert set(manager.schema.groups) == { 'input', 'output', 'pre-processor', @@ -179,13 +151,13 @@ def test_common_cli_manager_registers_top_level_subgroups(): } -def test_common_cli_manager_registers_input_parsers(): +def test_common_cli_manager_registers_input_schemas(): """ - Test that CommonCLIParsingManager registers every built-in input parser. + Test that CommonCLIParsingManager registers every built-in input schema. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert set(parser_manager.subparser['input']) == { + assert set(manager.schema.groups['input'].components) == { 'mongodb', 'socket', 'csv', @@ -193,13 +165,13 @@ def test_common_cli_manager_registers_input_parsers(): } -def test_common_cli_manager_registers_output_parsers(): +def test_common_cli_manager_registers_output_schemas(): """ - Test that CommonCLIParsingManager registers every built-in output parser. + Test that CommonCLIParsingManager registers every built-in output schema. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert set(parser_manager.subparser['output']) == { + assert set(manager.schema.groups['output'].components) == { 'mongodb', 'prometheus', 'csv', @@ -209,14 +181,14 @@ def test_common_cli_manager_registers_output_parsers(): } -def test_common_cli_manager_registers_pre_processor_parsers(): +def test_common_cli_manager_registers_pre_processor_schemas(): """ - Test that CommonCLIParsingManager registers every built-in pre-processor parser. + Test that CommonCLIParsingManager registers every built-in pre-processor schema. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - assert set(parser_manager.subparser['pre-processor']) == { - 'k8s', + assert set(manager.schema.groups['pre-processor'].components) == { + 'kubernetes', 'openstack', } @@ -225,12 +197,11 @@ def test_common_cli_manager_registers_verbose_argument(): """ Test that CommonCLIParsingManager registers the verbose root argument. """ - parser_manager = CommonCLIParsingManager() - verbose_argument = parser_manager.cli_parser.get_arguments()['verbose'] + manager = CommonCLIParsingManager() + verbose_argument = manager.schema.arguments['verbose'] - assert verbose_argument.names == ['v', 'verbose'] + assert verbose_argument.name == 'verbose' assert verbose_argument.is_flag is True - assert verbose_argument.action is store_true assert verbose_argument.default_value is False @@ -238,12 +209,11 @@ def test_common_cli_manager_registers_stream_argument(): """ Test that CommonCLIParsingManager registers the stream root argument. """ - parser_manager = CommonCLIParsingManager() - stream_argument = parser_manager.cli_parser.get_arguments()['stream'] + manager = CommonCLIParsingManager() + stream_argument = manager.schema.arguments['stream'] - assert stream_argument.names == ['s', 'stream'] + assert stream_argument.name == 'stream' assert stream_argument.is_flag is True - assert stream_argument.action is store_true assert stream_argument.default_value is False @@ -251,7 +221,7 @@ def test_common_cli_manager_validates_output_config_without_name(): """ Test that output config validation can use the output key as the pusher name. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() config = { 'output': { 'powerrep': { @@ -262,7 +232,7 @@ def test_common_cli_manager_validates_output_config_without_name(): }, } - result = parser_manager.validate(config) + result = manager.validate(config) assert result['output']['powerrep']['type'] == 'json' assert result['output']['powerrep']['model'] == 'PowerReport' @@ -270,33 +240,33 @@ def test_common_cli_manager_validates_output_config_without_name(): assert result['output']['powerrep']['compression'] == 'auto' -def test_common_cli_manager_requires_name_for_cli_subgroup(): +def test_common_cli_manager_rejects_legacy_contextual_configuration(): """ - Test that CLI subgroup parsing still requires -n/--name. + Test that the breaking CLI no longer accepts contextual component arguments. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - with pytest.raises(NoNameSpecifiedForSubgroupException): - parser_manager._parse_cli([ + with pytest.raises(CLIParseException) as result: + manager.parse([ '--output', 'json', '--filepath', '/tmp/powerapi-output.jsonl', ]) + assert 'unrecognized arguments' in result.value.msg + def test_common_cli_manager_parse_cli_configuration(): """ Test that CommonCLIParsingManager parses a representative CLI configuration. """ - parser_manager = CommonCLIParsingManager() + manager = CommonCLIParsingManager() - result = parser_manager._parse_cli([ + result = manager.parse([ '--verbose', - '--input', 'csv', - '--name', 'pytest-puller', - '--files', 'a.csv,b.csv', - '--output', 'json', - '--name', 'pytest-pusher', - '--filepath', '/tmp/pytest-powerapi.jsonl' + '-C', 'input.pytest-puller.type=csv', + '-C', 'input.pytest-puller.files=a.csv,b.csv', + '-C', 'output.pytest-pusher.type=json', + '-C', 'output.pytest-pusher.filepath=/tmp/pytest-powerapi.jsonl', ]) assert result['verbose'] is True @@ -304,3 +274,71 @@ def test_common_cli_manager_parse_cli_configuration(): assert result['input']['pytest-puller']['files'] == ['a.csv', 'b.csv'] assert result['output']['pytest-pusher']['type'] == 'json' assert result['output']['pytest-pusher']['filepath'] == '/tmp/pytest-powerapi.jsonl' + + +def test_common_cli_manager_resolves_component_type_after_merging_environment(monkeypatch): + """ + Test that a partial CLI override inherits its component type from the environment. + """ + monkeypatch.setenv('POWERAPI_INPUT_sensor_TYPE', 'socket') + manager = CommonCLIParsingManager() + + result = manager.parse([ + 'powerapi', + '-C', 'input.sensor.port=9090', + ]) + + assert result['input']['sensor'] == { + 'type': 'socket', + 'model': 'HWPCReport', + 'host': 'localhost', + 'port': 9090, + } + + +def test_common_cli_manager_rejects_unknown_dotted_component_property(): + """ + Test that an unknown dotted component property is rejected with its full path. + """ + manager = CommonCLIParsingManager() + + with pytest.raises(ConfigurationError) as result: + manager.parse([ + 'powerapi', + '-C', 'input.sensor.type=socket', + '-C', 'input.sensor.unknown=value', + ]) + + assert result.value.path == 'input.sensor.unknown' + + +@pytest.mark.parametrize(('property_name', 'value'), [ + ('p', 9090), + ('name', 'legacy-name'), +]) +def test_common_cli_manager_rejects_legacy_component_properties(property_name, value): + """ + Test that removed shortened component property names are rejected. + """ + manager = CommonCLIParsingManager() + config = {'input': {'sensor': {'type': 'socket', property_name: value}}} + + with pytest.raises(ConfigurationError) as result: + manager.validate(config) + + assert result.value.path == f'input.sensor.{property_name}' + + +def test_common_cli_manager_requires_component_type_after_merging(): + """ + Test that a component type is required after all configuration sources are merged. + """ + manager = CommonCLIParsingManager() + + with pytest.raises(ConfigurationError) as result: + manager.parse([ + 'powerapi', + '-C', 'input.sensor.port=9090', + ]) + + assert result.value.path == 'input.sensor.type' diff --git a/tests/unit/config/test_config_parser.py b/tests/unit/config/test_config_parser.py new file mode 100644 index 00000000..48fe2465 --- /dev/null +++ b/tests/unit/config/test_config_parser.py @@ -0,0 +1,583 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json + +import pytest + +from powerapi.config.config_loader import EnvironmentConfigLoader, JSONConfigLoader +from powerapi.config.config_parser import ( + ComponentSchema, + ConfigurationSchema, + ConfigurationSectionSchema, +) +from powerapi.exception import ConfigurationError + + +def test_schema_registers_argument_definition(): + """ + Test that a configuration property is registered with its definition. + """ + schema = ConfigurationSectionSchema() + + schema.add_argument('port', argument_type=int, default_value=9080) + + definition = schema.arguments['port'] + assert definition.name == 'port' + assert definition.argument_type is int + assert definition.default_value == 9080 + + +def test_schema_rejects_duplicate_property(): + """ + Test that a configuration property cannot be registered more than once. + """ + schema = ConfigurationSectionSchema() + schema.add_argument('port') + + with pytest.raises(ValueError, match='already registered'): + schema.add_argument('port') + +def test_schema_casts_and_applies_defaults(): + """ + Test that schema validation casts canonical property values and applies defaults. + """ + schema = ConfigurationSectionSchema() + schema.add_argument('port', argument_type=int) + schema.add_argument('enabled', argument_type=bool, default_value=False) + schema.add_argument('tags', argument_type=list) + + result = schema.validate({'port': '9080', 'tags': 'host,socket'}) + + assert result == { + 'port': 9080, + 'enabled': False, + 'tags': ['host', 'socket'], + } + + +def test_schema_copies_mutable_defaults(): + """ + Test that mutating a validated default does not modify later configurations. + """ + schema = ConfigurationSectionSchema() + schema.add_argument('tags', argument_type=list, default_value=[]) + + first_result = schema.validate({}) + first_result['tags'].append('sensor') + + assert schema.validate({}) == {'tags': []} + + +def test_schema_rejects_invalid_boolean_value(): + """ + Test that an unrecognized textual boolean value is rejected. + """ + schema = ConfigurationSectionSchema() + schema.add_argument('enabled', argument_type=bool) + + with pytest.raises(ConfigurationError) as result: + schema.validate({'enabled': 'invalid'}) + + assert result.value.path == 'enabled' + assert result.value.reason == 'Expected bool' + + +def test_schema_rejects_missing_mandatory_property(): + """ + Test that a missing mandatory property is reported with its path. + """ + schema = ConfigurationSectionSchema() + schema.add_argument('uri', is_mandatory=True) + + with pytest.raises(ConfigurationError) as result: + schema.validate({}) + + assert result.value.path == 'uri' + assert result.value.reason == 'Missing required value' + + +@pytest.mark.parametrize(('argument_type', 'value', 'expected'), [ + (str, '', ''), + (list, '', []), +]) +def test_schema_accepts_empty_mandatory_property(argument_type, value, expected): + """ + Test that mandatory properties require presence but may contain empty values. + """ + schema = ConfigurationSectionSchema() + schema.add_argument('value', argument_type=argument_type, is_mandatory=True) + + assert schema.validate({'value': value}) == {'value': expected} + + +def test_schema_rejects_unknown_property(): + """ + Test that an unknown property is reported with its path. + """ + with pytest.raises(ConfigurationError) as result: + ConfigurationSectionSchema().validate({'unknown': 'value'}) + + assert result.value.path == 'unknown' + assert result.value.reason == 'Unknown property' + + +def test_schema_rejects_cli_alias_as_configuration_property(): + """ + Test that schema validation only accepts canonical configuration property names. + """ + schema = ConfigurationSectionSchema() + schema.add_argument('port', argument_type=int) + + with pytest.raises(ConfigurationError) as result: + schema.validate({'p': '9080'}) + + assert result.value.path == 'p' + assert result.value.reason == 'Unknown property' + + +def test_schema_reports_bad_type(): + """ + Test that an invalid property value reports the expected type and path. + """ + schema = ConfigurationSectionSchema() + schema.add_argument('port', argument_type=int) + + with pytest.raises(ConfigurationError) as result: + schema.validate({'port': 'not-an-integer'}) + + assert result.value.path == 'port' + assert result.value.reason == 'Expected int' + + +def test_root_schema_validates_component_without_synthetic_name_property(): + """ + Test that component names come from group keys rather than synthetic properties. + """ + schema = ConfigurationSchema() + schema.add_group('input', prefix='POWERAPI_INPUT_') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int, default_value=9080) + schema.add_component('input', component) + + result = schema.validate({'input': {'sensor': {'type': 'socket'}}}) + + assert result == {'input': {'sensor': {'type': 'socket', 'port': 9080}}} + + +def test_root_schema_prefixes_component_validation_error(): + """ + Test that component validation errors contain the complete dotted path. + """ + schema = ConfigurationSchema() + schema.add_group('input') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int) + schema.add_component('input', component) + + with pytest.raises(ConfigurationError) as result: + schema.validate({'input': {'sensor': {'type': 'socket', 'port': 'invalid'}}}) + + assert result.value.path == 'input.sensor.port' + assert result.value.reason == 'Expected int' + assert result.value.msg == 'Invalid configuration at "input.sensor.port": Expected int' + + +@pytest.mark.parametrize('configuration', [ + {'input': {'sensor': {}}}, + {'input': {'sensor': {'type': 'unknown'}}}, + {'input': {'sensor': {'type': {}}}}, +]) +def test_root_schema_requires_known_component_type(configuration): + """ + Test that components require a registered component type. + """ + schema = ConfigurationSchema() + schema.add_group('input') + + with pytest.raises(ConfigurationError) as result: + schema.validate(configuration) + + assert result.value.path == 'input.sensor.type' + + +def test_root_schema_validates_fixed_section_without_component_type(): + """ + Test that a fixed group section is validated without a component type. + """ + schema = ConfigurationSchema() + schema.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + schema.add_section('formula', 'smartwatts', smartwatts) + + result = schema.validate({ + 'formula': { + 'smartwatts': { + 'learn-error-window-size': '10', + }, + }, + }) + + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, + } + + +def test_root_schema_applies_defaults_from_fixed_section(): + """ + Test that fixed section defaults are applied when the section is omitted. + """ + schema = ConfigurationSchema() + schema.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int, default_value=10) + schema.add_section('formula', 'smartwatts', smartwatts) + + result = schema.validate({}) + + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, + } + + +def test_root_schema_rejects_component_type_in_fixed_section(): + """ + Test that a fixed group section does not accept a component type selector. + """ + schema = ConfigurationSchema() + schema.add_group('formula') + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) + + with pytest.raises(ConfigurationError) as result: + schema.validate({'formula': {'smartwatts': {'type': 'smartwatts'}}}) + + assert result.value.path == 'formula.smartwatts.type' + assert result.value.reason == 'Unknown property' + + +def test_root_schema_rejects_unregistered_type_property(): + """ + Test that the component type selector is not accepted as a root property. + """ + with pytest.raises(ConfigurationError) as result: + ConfigurationSchema().validate({'type': 'socket'}) + + assert result.value.path == 'type' + assert result.value.reason == 'Unknown property' + + +@pytest.mark.parametrize(('configuration', 'path'), [ + ({'input': []}, 'input'), + ({'input': {'sensor': []}}, 'input.sensor'), +]) +def test_root_schema_rejects_non_dictionary_group_values(configuration, path): + """ + Test that configuration groups and their entries must be dictionaries. + """ + schema = ConfigurationSchema() + schema.add_group('input') + + with pytest.raises(ConfigurationError) as result: + schema.validate(configuration) + + assert result.value.path == path + assert result.value.reason == 'Expected dict' + + +def test_json_loader_rejects_shortened_property_names(tmp_path): + """ + Test that JSON loading only accepts registered configuration property names. + """ + schema = ConfigurationSchema() + schema.add_argument('port', argument_type=int, default_value=9080) + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(json.dumps({'p': '9090'}), encoding='utf-8') + + configuration = JSONConfigLoader().load(str(config_file)) + with pytest.raises(ConfigurationError) as result: + schema.validate(configuration) + + assert result.value.path == 'p' + assert result.value.reason == 'Unknown property' + + +def test_json_loader_loads_fixed_section_without_component_type(tmp_path): + """ + Test that JSON loading preserves a fixed group section without a type. + """ + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(json.dumps({ + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, + }), encoding='utf-8') + + result = JSONConfigLoader().load(str(config_file)) + + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, + } + + +def test_json_loader_returns_empty_configuration_without_file(): + """ + Test that JSON loading returns an empty configuration when no file is selected. + """ + assert JSONConfigLoader().load(None) == {} + + +def test_json_loader_reports_invalid_json_as_configuration_error(tmp_path): + """ + Test that invalid JSON is exposed as a ConfigurationError with decoder details. + """ + config_file = tmp_path / 'powerapi-pytest.json' + invalid_json = '{"stream": true,}' + config_file.write_text(invalid_json, encoding='utf-8') + + with pytest.raises(json.JSONDecodeError) as decode_error: + json.loads(invalid_json) + + with pytest.raises(ConfigurationError) as result: + JSONConfigLoader().load(str(config_file)) + + assert result.value.path is None + assert result.value.reason == f'Invalid JSON in configuration file "{config_file}": {decode_error.value}' + + +@pytest.mark.parametrize('content', ['[]', 'null', '"value"']) +def test_json_loader_rejects_non_object_root(content, tmp_path): + """ + Test that a JSON configuration must contain an object at its root. + """ + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(content, encoding='utf-8') + + with pytest.raises(ConfigurationError) as result: + JSONConfigLoader().load(str(config_file)) + + assert result.value.reason == 'Expected a JSON object' + + +def test_environment_loader_preserves_root_and_component_format(monkeypatch): + """ + Test that environment loading preserves raw root and nested component values. + """ + schema = ConfigurationSchema() + schema.add_argument_prefix('POWERAPI_') + schema.add_argument('stream', argument_type=bool) + schema.add_group('input', prefix='POWERAPI_INPUT_') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int) + schema.add_component('input', component) + monkeypatch.setenv('POWERAPI_STREAM', 'true') + monkeypatch.setenv('POWERAPI_INPUT_SENSOR_TYPE', 'socket') + monkeypatch.setenv('POWERAPI_INPUT_SENSOR_PORT', '9080') + + result = EnvironmentConfigLoader(schema).load() + + assert result == { + 'stream': 'true', + 'input': {'sensor': {'type': 'socket', 'port': '9080'}}, + } + + +def test_environment_loader_ignores_group_without_prefix(monkeypatch): + """ + Test that a prefixless group does not hide root values or inspect unrelated environment variables. + """ + schema = ConfigurationSchema() + schema.add_argument_prefix('POWERAPI_') + schema.add_argument('stream', argument_type=bool) + schema.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + schema.add_section('formula', 'smartwatts', smartwatts) + monkeypatch.setenv('POWERAPI_STREAM', 'true') + monkeypatch.setenv('SMARTWATTS_LEARN_ERROR_WINDOW_SIZE', '10') + + result = EnvironmentConfigLoader(schema).load() + + assert result == {'stream': 'true'} + + +def test_environment_loader_preserves_fixed_section_without_component_type(monkeypatch): + """ + Test that environment loading preserves a raw fixed group section. + """ + schema = ConfigurationSchema() + schema.add_group('formula', prefix='POWERAPI_FORMULA_') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + schema.add_section('formula', 'smartwatts', smartwatts) + monkeypatch.setenv('POWERAPI_FORMULA_SMARTWATTS_LEARN_ERROR_WINDOW_SIZE', '10') + + result = EnvironmentConfigLoader(schema).load() + + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': '10', + }, + }, + } + + +def test_environment_loader_rejects_component_type_in_fixed_section(monkeypatch): + """ + Test that an environment fixed section does not accept a component type selector. + """ + schema = ConfigurationSchema() + schema.add_group('formula', prefix='POWERAPI_FORMULA_') + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) + monkeypatch.setenv('POWERAPI_FORMULA_SMARTWATTS_TYPE', 'smartwatts') + + configuration = EnvironmentConfigLoader(schema).load() + with pytest.raises(ConfigurationError) as result: + schema.validate(configuration) + + assert result.value.path == 'formula.smartwatts.type' + assert result.value.reason == 'Unknown property' + + +def test_environment_loader_preserves_component_without_type(monkeypatch): + """ + Test that environment loading preserves a partial component for later merging. + """ + schema = ConfigurationSchema() + schema.add_group('input', prefix='POWERAPI_INPUT_') + component = ComponentSchema('socket') + component.add_argument('port', argument_type=int) + schema.add_component('input', component) + monkeypatch.setenv('POWERAPI_INPUT_SENSOR_PORT', '9080') + + assert EnvironmentConfigLoader(schema).load() == { + 'input': {'sensor': {'port': '9080'}}, + } + + +def test_root_schema_rejects_overlapping_environment_prefixes(): + """ + Test that overlapping root environment prefixes are rejected. + """ + schema = ConfigurationSchema() + schema.add_argument_prefix('POWERAPI_') + + with pytest.raises(ValueError, match='conflicts with'): + schema.add_argument_prefix('POWERAPI_INPUT_') + + +def test_root_schema_rejects_duplicate_group(): + """ + Test that a configuration group cannot be registered more than once. + """ + schema = ConfigurationSchema() + schema.add_group('input') + + with pytest.raises(ValueError, match='already registered'): + schema.add_group('input') + + +def test_root_schema_rejects_group_matching_property(): + """ + Test that a group cannot reuse a registered root property name. + """ + schema = ConfigurationSchema() + schema.add_argument('input') + + with pytest.raises(ValueError, match='already registered as a property'): + schema.add_group('input') + + +def test_root_schema_rejects_property_matching_group(): + """ + Test that a root property cannot reuse a registered group name. + """ + schema = ConfigurationSchema() + schema.add_group('input') + + with pytest.raises(ValueError, match='already registered as a group'): + schema.add_argument('input') + + +def test_root_schema_rejects_duplicate_component_type(): + """ + Test that a component type cannot be registered twice in one group. + """ + schema = ConfigurationSchema() + schema.add_group('input') + schema.add_component('input', ComponentSchema('socket')) + + with pytest.raises(ValueError, match='already registered'): + schema.add_component('input', ComponentSchema('socket')) + + +def test_root_schema_rejects_component_for_unknown_group(): + """ + Test that a component cannot be registered in an unknown group. + """ + schema = ConfigurationSchema() + + with pytest.raises(ValueError, match='is not registered'): + schema.add_component('input', ComponentSchema('socket')) + + +def test_root_schema_rejects_section_for_unknown_group(): + """ + Test that a fixed section cannot be registered in an unknown group. + """ + schema = ConfigurationSchema() + + with pytest.raises(ValueError, match='is not registered'): + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) + + +def test_root_schema_rejects_duplicate_section(): + """ + Test that a fixed section name cannot be registered twice in one group. + """ + schema = ConfigurationSchema() + schema.add_group('formula') + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) + + with pytest.raises(ValueError, match='already registered'): + schema.add_section('formula', 'smartwatts', ConfigurationSectionSchema()) diff --git a/tests/unit/config/test_generator.py b/tests/unit/config/test_generator.py new file mode 100644 index 00000000..74b56c5a --- /dev/null +++ b/tests/unit/config/test_generator.py @@ -0,0 +1,332 @@ +# Copyright (c) 2023, Inria +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +from copy import deepcopy + +import pytest + +from powerapi.config.generator import ( + PreProcessorGenerator, + PullerGenerator, + PusherGenerator, +) +from powerapi.database.csv.driver import CSVInputFactory, CSVOutputFactory +from powerapi.database.json.driver import JsonInputFactory, JsonOutputFactory +from powerapi.database.socket.driver import SocketInputFactory +from powerapi.exception import ConfigurationError, PowerAPIException +from powerapi.filter import BroadcastReportFilter +from powerapi.puller import PullerActor +from powerapi.pusher import PusherActor +from powerapi.report import FormulaReport, HWPCReport, PowerReport + + +def _unavailable_factory(_: dict): + """ + Simulate a component factory whose optional dependency is unavailable. + """ + raise ImportError + + +def test_generate_puller_from_empty_config_dict_raise_an_exception(): + """ + Test that PullerGenerator raises a PowerAPIException when there is no input argument. + """ + conf = {} + generator = PullerGenerator(BroadcastReportFilter()) + + with pytest.raises(PowerAPIException): + generator.generate(conf) + + +@pytest.mark.parametrize('input_type', ['csv', 'json']) +def test_generate_file_puller_in_stream_mode_raise_an_exception(several_inputs_outputs_stream_config, input_type): + """ + Test that PullerGenerator rejects input types that do not support stream mode. + """ + config = deepcopy(several_inputs_outputs_stream_config) + config['input'] = {name: value for name, value in config['input'].items() if value['type'] == input_type} + generator = PullerGenerator(BroadcastReportFilter()) + + with pytest.raises(ConfigurationError) as raised_exception: + generator.generate(config) + + assert raised_exception.value.path == 'stream' + assert raised_exception.value.reason == f'Stream mode cannot be used with a {input_type} input' + + +def test_generate_several_pullers_from_config(several_inputs_outputs_postmortem_config): + """ + Test that several inputs are correctly used to generate the related actors. + """ + generator = PullerGenerator(BroadcastReportFilter()) + pullers = generator.generate(several_inputs_outputs_postmortem_config) + + assert len(pullers) == len(several_inputs_outputs_postmortem_config['input']) + + for puller_name, current_puller_infos in several_inputs_outputs_postmortem_config['input'].items(): + assert puller_name in pullers + assert isinstance(pullers[puller_name], PullerActor) + + db_factory = pullers[puller_name].database_factory + + if current_puller_infos['type'] == 'csv': + assert isinstance(db_factory, CSVInputFactory) + assert db_factory.input_files == current_puller_infos['files'] + elif current_puller_infos['type'] == 'socket': + assert isinstance(db_factory, SocketInputFactory) + assert db_factory.host == current_puller_infos['host'] + assert db_factory.port == current_puller_infos['port'] + elif current_puller_infos['type'] == 'json': + assert isinstance(db_factory, JsonInputFactory) + assert db_factory.output_filepath == current_puller_infos['filepath'] + else: + pytest.fail(f'Unsupported puller type: {current_puller_infos["type"]}') + + +def test_generate_streaming_puller_preserves_runtime_settings(several_inputs_outputs_stream_config): + """ + Test that a streaming puller receives its filter, stream mode, and logging level. + """ + config = deepcopy(several_inputs_outputs_stream_config) + config['input'] = {'puller3': config['input']['puller3']} + report_filter = BroadcastReportFilter() + + puller = PullerGenerator(report_filter).generate(config)['puller3'] + + assert puller.report_filter is report_filter + assert puller.stream_mode is True + assert puller.logging_level == logging.DEBUG + + +def test_generate_puller_with_registered_report_model(several_inputs_outputs_postmortem_config): + """ + Test that a registered report model is resolved when generating a puller. + """ + config = deepcopy(several_inputs_outputs_postmortem_config) + config['input'] = {'puller2': config['input']['puller2']} + config['input']['puller2']['model'] = 'CustomReport' + generator = PullerGenerator(BroadcastReportFilter()) + generator.add_report_class('CustomReport', HWPCReport) + + puller = generator.generate(config)['puller2'] + + assert puller.database_factory.report_type is HWPCReport + + +def test_register_existing_report_model_raises_value_error(): + """ + Test that a report model cannot be registered more than once. + """ + generator = PullerGenerator(BroadcastReportFilter()) + + with pytest.raises(ValueError, match='Report model "HWPCReport" is already registered'): + generator.add_report_class('HWPCReport', HWPCReport) + + +def test_register_existing_database_type_raises_value_error(): + """ + Test that a database type cannot be registered more than once. + """ + generator = PullerGenerator(BroadcastReportFilter()) + + with pytest.raises(ValueError, match='Database type "csv" is already registered'): + generator.add_db_factory('csv', generator.database_factories['csv']) + + +def test_generate_puller_with_unknown_report_model_raises_an_exception(several_inputs_outputs_postmortem_config): + """ + PullerGenerator should raise an exception when the model of an input is not defined. + """ + config = deepcopy(several_inputs_outputs_postmortem_config) + next(iter(config['input'].values()))['model'] = 'UnknownReport' + generator = PullerGenerator(BroadcastReportFilter()) + + with pytest.raises(PowerAPIException, match='Configuration error: Unknown report model "UnknownReport"'): + generator.generate(config) + + +def test_generate_puller_with_unknown_database_type_raises_an_exception(several_inputs_outputs_postmortem_config): + """ + PullerGenerator should raise an exception when the database of an input is not defined. + """ + config = deepcopy(several_inputs_outputs_postmortem_config) + next(iter(config['input'].values()))['type'] = 'unknown' + generator = PullerGenerator(BroadcastReportFilter()) + + with pytest.raises(PowerAPIException, match='Configuration error: Invalid database type: unknown'): + generator.generate(config) + + +def test_generate_puller_with_unavailable_database_dependency_raises_an_exception(several_inputs_outputs_postmortem_config): + """ + Test that a missing database dependency is reported as a configuration error. + """ + config = deepcopy(several_inputs_outputs_postmortem_config) + config['input'] = {'puller2': config['input']['puller2']} + config['input']['puller2']['type'] = 'unavailable' + generator = PullerGenerator(BroadcastReportFilter()) + generator.add_db_factory('unavailable', _unavailable_factory) + + with pytest.raises(PowerAPIException, match='Dependencies for unavailable database are not installed'): + generator.generate(config) + + +def test_generate_does_not_modify_configuration(several_inputs_outputs_postmortem_config): + """ + Test that generating pullers and pushers repeatedly preserves the canonical configuration. + """ + expected = deepcopy(several_inputs_outputs_postmortem_config) + puller_generator = PullerGenerator(BroadcastReportFilter()) + pusher_generator = PusherGenerator() + + puller_generator.generate(several_inputs_outputs_postmortem_config) + pusher_generator.generate(several_inputs_outputs_postmortem_config) + puller_generator.generate(several_inputs_outputs_postmortem_config) + pusher_generator.generate(several_inputs_outputs_postmortem_config) + + assert several_inputs_outputs_postmortem_config == expected + + +def test_generate_pusher_from_empty_config_dict_raises_an_exception(): + """ + Test that PusherGenerator raises an exception when there is no output argument. + """ + conf = {} + generator = PusherGenerator() + + with pytest.raises(PowerAPIException): + generator.generate(conf) + + +def test_generate_several_pushers_from_config(several_inputs_outputs_stream_config): + """ + Test that several outputs are correctly used to generate the related actors. + + """ + generator = PusherGenerator() + pushers = generator.generate(several_inputs_outputs_stream_config) + + assert len(pushers) == len(several_inputs_outputs_stream_config['output']) + + for pusher_name, current_pusher_infos in several_inputs_outputs_stream_config['output'].items(): + assert pusher_name in pushers + assert isinstance(pushers[pusher_name], PusherActor) + + db_factory = pushers[pusher_name].database_factory + pusher_type = current_pusher_infos['type'] + + if pusher_type == 'csv': + assert isinstance(db_factory, CSVOutputFactory) + assert db_factory.output_directory == current_pusher_infos['directory'] + elif pusher_type == 'json': + assert isinstance(db_factory, JsonOutputFactory) + assert db_factory.output_filepath == current_pusher_infos['filepath'] + else: + pytest.fail(f'Unsupported pusher type: {pusher_type}') + + +def test_generate_pusher_report_type_to_actor_mapping(single_input_multiple_outputs_with_different_report_type): + """ + Test generating a report type to actor mapping from a configuration having multiple outputs for different report types. + """ + config = single_input_multiple_outputs_with_different_report_type + generator = PusherGenerator() + actors = generator.generate(config) + report_mapping = generator.generate_report_mapping(config, actors) + + assert set(report_mapping.keys()) == {PowerReport, FormulaReport} + assert [proxy.actor_name for proxy in report_mapping[PowerReport]] == ['powerrep1', 'powerrep2'] + assert [proxy.actor_type for proxy in report_mapping[PowerReport]] == [PusherActor, PusherActor] + assert [proxy.actor_name for proxy in report_mapping[FormulaReport]] == ['formularep'] + assert [proxy.actor_type for proxy in report_mapping[FormulaReport]] == [PusherActor] + + +def test_generate_pusher_report_mapping_without_output_group_raises_an_exception(): + """ + Test that report mapping requires the output component group. + """ + generator = PusherGenerator() + + with pytest.raises(PowerAPIException, match='Configuration error: Component "output" is not defined'): + generator.generate_report_mapping({}, {}) + + +def test_generate_pusher_report_mapping_with_missing_actor_raises_an_exception(): + """ + Test that report mapping rejects an output without a generated actor. + """ + config = {'output': {'missing': {'model': 'PowerReport'}}} + generator = PusherGenerator() + + with pytest.raises(PowerAPIException, match='Actor "missing" is not defined'): + generator.generate_report_mapping(config, {}) + + +def test_generate_pre_processor_from_empty_config_dict_raises_an_exception(): + """ + Test that PreProcessGenerator raises an exception when there is no processor argument. + """ + conf = {} + generator = PreProcessorGenerator() + + with pytest.raises(PowerAPIException): + generator.generate(conf) + + +def test_register_existing_processor_type_raises_value_error(): + """ + Test that a processor type cannot be registered more than once. + """ + generator = PreProcessorGenerator() + + with pytest.raises(ValueError, match='Processor type "kubernetes" is already registered'): + generator.add_processor_factory('kubernetes', generator.processor_factories['kubernetes']) + + +def test_generate_unknown_processor_type_raises_an_exception(): + """ + Test that generating an unknown processor type raises a configuration error. + """ + config = {'verbose': False, 'pre-processor': {'processor': {'type': 'unknown'}}} + + with pytest.raises(PowerAPIException, match='Configuration error: Invalid processor type: unknown'): + PreProcessorGenerator().generate(config) + + +def test_generate_processor_with_unavailable_dependency_raises_an_exception(): + """ + Test that a missing processor dependency is reported as a configuration error. + """ + config = {'verbose': False, 'pre-processor': {'processor': {'type': 'unavailable'}}} + generator = PreProcessorGenerator() + generator.add_processor_factory('unavailable', _unavailable_factory) + + with pytest.raises(PowerAPIException, match='Dependencies for unavailable processor are not installed'): + generator.generate(config) diff --git a/tests/unit/cli/test_generator_clickhouse.py b/tests/unit/config/test_generator_clickhouse.py similarity index 84% rename from tests/unit/cli/test_generator_clickhouse.py rename to tests/unit/config/test_generator_clickhouse.py index ca6746df..2f8551b2 100644 --- a/tests/unit/cli/test_generator_clickhouse.py +++ b/tests/unit/config/test_generator_clickhouse.py @@ -28,8 +28,7 @@ import pytest -from powerapi.cli.generator import PusherGenerator -from powerapi.exception import PowerAPIException +from powerapi.config.generator import PusherGenerator from powerapi.pusher import PusherActor pytest.importorskip('powerapi.database.clickhouse.driver') # The ClickHouse driver requires external dependencies to work. @@ -81,16 +80,3 @@ def test_pusher_generator_with_valid_clickhouse_config(clickhouse_config): assert db_factory.username == expected_db_attributes['username'] assert db_factory.password == expected_db_attributes['password'] assert db_factory.database_name == expected_db_attributes['database'] - - -@pytest.mark.parametrize('missing_arg', ['model', 'host', 'port', 'username', 'password', 'database']) -def test_pusher_generator_with_missing_arguments_in_clickhouse_config(clickhouse_config, missing_arg): - """ - PusherGenerator should raise an exception when a required argument is missing from the ClickHouse config. - """ - generator = PusherGenerator() - - clickhouse_config['output']['pytest-clickhouse-pusher'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(clickhouse_config) diff --git a/tests/unit/cli/test_generator_influxdb2.py b/tests/unit/config/test_generator_influxdb2.py similarity index 83% rename from tests/unit/cli/test_generator_influxdb2.py rename to tests/unit/config/test_generator_influxdb2.py index 4acc96ce..90e30a03 100644 --- a/tests/unit/cli/test_generator_influxdb2.py +++ b/tests/unit/config/test_generator_influxdb2.py @@ -28,8 +28,7 @@ import pytest -from powerapi.cli.generator import PusherGenerator -from powerapi.exception import PowerAPIException +from powerapi.config.generator import PusherGenerator from powerapi.pusher import PusherActor pytest.importorskip("powerapi.database.influxdb2.driver") # The InfluxDB2 driver requires external dependencies to work. @@ -79,16 +78,3 @@ def test_pusher_generator_with_valid_influxdb2_config(influxdb2_config): assert db_factory.org == expected_db_attributes['org'] assert db_factory.token == expected_db_attributes['token'] assert db_factory.bucket == expected_db_attributes['bucket'] - - -@pytest.mark.parametrize('missing_arg', ['model', 'uri', 'org', 'token', 'bucket']) -def test_pusher_generator_with_missing_arguments_in_influxdb2_config(influxdb2_config, missing_arg): - """ - PusherGenerator should raise an exception when a required argument is missing from the InfluxDB2 config. - """ - generator = PusherGenerator() - - influxdb2_config['output']['pytest-influxdb2-pusher'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(influxdb2_config) diff --git a/tests/unit/cli/test_generator_k8s.py b/tests/unit/config/test_generator_k8s.py similarity index 82% rename from tests/unit/cli/test_generator_k8s.py rename to tests/unit/config/test_generator_k8s.py index 8c5eb5c3..7a92b140 100644 --- a/tests/unit/cli/test_generator_k8s.py +++ b/tests/unit/config/test_generator_k8s.py @@ -30,8 +30,7 @@ pytest.importorskip('kubernetes') -from powerapi.cli.generator import PreProcessorGenerator -from powerapi.exception import PowerAPIException +from powerapi.config.generator import PreProcessorGenerator from powerapi.processor.pre.k8s.actor import KubernetesPreProcessorActor @@ -45,10 +44,11 @@ def k8s_processor_config(): 'verbose': True, 'pre-processor': { 'pytest-k8s-preprocessor': { - 'type': 'k8s', + 'type': 'kubernetes', 'api-mode': 'manual', 'api-host': 'https://127.0.0.1:36599', 'api-key': 'pytest-token-powerapi', + 'labels': ['app.kubernetes.io/name'], 'puller': 'pytest-json-puller' } } @@ -72,16 +72,6 @@ def test_preprocessor_generator_with_valid_k8s_config(k8s_processor_config): assert preprocessor.monitor_config.api_mode == expected_preprocessor_attributes['api-mode'] assert preprocessor.monitor_config.api_key == expected_preprocessor_attributes['api-key'] assert preprocessor.monitor_config.api_host == expected_preprocessor_attributes['api-host'] - - -@pytest.mark.parametrize('missing_arg', ['api-mode']) -def test_preprocessor_generator_with_missing_arguments_in_k8s_config(k8s_processor_config, missing_arg): - """ - PreProcessorGenerator should raise an exception when a required argument is missing from the MongoDB config. - """ - generator = PreProcessorGenerator() - - k8s_processor_config['pre-processor']['pytest-k8s-preprocessor'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(k8s_processor_config) + assert preprocessor.monitor_config.label_mapping == { + 'app.kubernetes.io/name': 'k8s_pod_label_app_kubernetes_io_name', + } diff --git a/tests/unit/cli/test_generator_mongodb.py b/tests/unit/config/test_generator_mongodb.py similarity index 79% rename from tests/unit/cli/test_generator_mongodb.py rename to tests/unit/config/test_generator_mongodb.py index 22d524f9..1c0b1411 100644 --- a/tests/unit/cli/test_generator_mongodb.py +++ b/tests/unit/config/test_generator_mongodb.py @@ -28,8 +28,7 @@ import pytest -from powerapi.cli.generator import PusherGenerator, PullerGenerator -from powerapi.exception import PowerAPIException +from powerapi.config.generator import PusherGenerator, PullerGenerator from powerapi.filter import BroadcastReportFilter from powerapi.puller import PullerActor from powerapi.pusher import PusherActor @@ -89,20 +88,6 @@ def test_puller_generator_with_valid_mongodb_config(mongodb_config): assert db_factory.database_name == expected_db_attributes['db'] assert db_factory.collection_name == expected_db_attributes['collection'] - -@pytest.mark.parametrize('missing_arg', ['model', 'uri']) -def test_puller_generator_with_missing_arguments_in_mongodb_config(mongodb_config, missing_arg): - """ - PullerGenerator should raise an exception when a required argument is missing from the MongoDB config. - """ - generator = PullerGenerator(BroadcastReportFilter()) - - mongodb_config['input']['pytest-mongodb-puller'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(mongodb_config) - - def test_pusher_generator_with_valid_mongodb_config(mongodb_config): """ PusherGenerator should generate a PusherActor with a MongoDB database driver. @@ -123,16 +108,3 @@ def test_pusher_generator_with_valid_mongodb_config(mongodb_config): assert db_factory.uri == expected_db_attributes['uri'] assert db_factory.database_name == expected_db_attributes['db'] assert db_factory.collection_name == expected_db_attributes['collection'] - - -@pytest.mark.parametrize('missing_arg', ['model', 'uri', 'db', 'collection']) -def test_pusher_generator_with_missing_arguments_in_mongodb_config(mongodb_config, missing_arg): - """ - PusherGenerator should raise an exception when a required argument is missing from the MongoDB config. - """ - generator = PusherGenerator() - - mongodb_config['output']['pytest-mongodb-pusher'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(mongodb_config) diff --git a/tests/unit/cli/test_generator_openstack.py b/tests/unit/config/test_generator_openstack.py similarity index 91% rename from tests/unit/cli/test_generator_openstack.py rename to tests/unit/config/test_generator_openstack.py index 53dc6848..f8fab8b4 100644 --- a/tests/unit/cli/test_generator_openstack.py +++ b/tests/unit/config/test_generator_openstack.py @@ -30,7 +30,7 @@ pytest.importorskip('openstack') -from powerapi.cli.generator import PreProcessorGenerator +from powerapi.config.generator import PreProcessorGenerator from powerapi.processor.pre.openstack.actor import OpenStackPreProcessorActor @@ -46,7 +46,8 @@ def openstack_config(): 'pytest-openstack-preprocessor': { 'type': 'openstack', 'puller': 'pytest-json-puller', - 'polling-interval': 10.0 + 'polling-interval': 10.0, + 'metadata': ['environment'], } } } @@ -67,3 +68,6 @@ def test_preprocessor_generator_with_valid_openstack_config(openstack_config): expected_preprocessor_attributes = openstack_config['pre-processor']['pytest-openstack-preprocessor'] assert preprocessor.monitor_config.polling_interval == expected_preprocessor_attributes['polling-interval'] + assert preprocessor.monitor_config.metadata_mapping == { + 'environment': 'openstack_metadata_environment', + } diff --git a/tests/unit/cli/test_generator_prometheus.py b/tests/unit/config/test_generator_prometheus.py similarity index 84% rename from tests/unit/cli/test_generator_prometheus.py rename to tests/unit/config/test_generator_prometheus.py index 4c1b360b..f6b1d10a 100644 --- a/tests/unit/cli/test_generator_prometheus.py +++ b/tests/unit/config/test_generator_prometheus.py @@ -28,8 +28,7 @@ import pytest -from powerapi.cli.generator import PusherGenerator -from powerapi.exception import PowerAPIException +from powerapi.config.generator import PusherGenerator from powerapi.pusher import PusherActor pytest.importorskip("powerapi.database.prometheus.driver") # The Prometheus driver requires external dependencies to work. @@ -76,16 +75,3 @@ def test_pusher_generator_with_valid_prometheus_config(prometheus_config): assert db_factory.listen_addr == expected_db_attributes['addr'] assert db_factory.listen_port == expected_db_attributes['port'] assert set(db_factory.tags) == {'powerapi_example_tag1', 'powerapi_example_tag2'} - - -@pytest.mark.parametrize('missing_arg', ['model', 'addr', 'port']) -def test_pusher_generator_with_missing_arguments_in_mongodb_config(prometheus_config, missing_arg): - """ - PusherGenerator should raise an exception when a required argument is missing from the Prometheus config. - """ - generator = PusherGenerator() - - prometheus_config['output']['pytest-prometheus-pusher'].pop(missing_arg) - - with pytest.raises(PowerAPIException): - generator.generate(prometheus_config) diff --git a/tests/unit/config/test_parsing_manager.py b/tests/unit/config/test_parsing_manager.py new file mode 100644 index 00000000..db33be67 --- /dev/null +++ b/tests/unit/config/test_parsing_manager.py @@ -0,0 +1,231 @@ +# Copyright (c) 2021, INRIA +# Copyright (c) 2021, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +import sys + +import pytest + +from powerapi.config.cli_parser import CLIParseException +from powerapi.config.config_parser import ComponentSchema, ConfigurationSectionSchema +from powerapi.config.parsing_manager import ConfigurationParsingManager +from powerapi.exception import ConfigurationError + + +@pytest.fixture +def parsing_manager() -> ConfigurationParsingManager: + """ + Create a parsing manager with representative root and component configuration. + :return: Configured parsing manager. + """ + manager = ConfigurationParsingManager() + manager.add_argument_prefix('TEST_POWERAPI_') + manager.add_argument('verbose', is_flag=True, default_value=False) + manager.add_argument('interval', argument_type=int, default_value=10) + manager.add_group('input', prefix='TEST_POWERAPI_INPUT_') + + socket = ComponentSchema('socket') + socket.add_argument('host', default_value='localhost') + socket.add_argument('port', argument_type=int, default_value=9080) + socket.add_argument('tags', argument_type=list) + manager.add_component('input', socket) + + return manager + + +def test_parse_merges_cli_assignments_and_validates_schema(parsing_manager): + """ + Test that CLI values are merged, cast, and completed with schema defaults. + """ + result = parsing_manager.parse([ + 'powerapi', + '--verbose', + '-C', 'input.sensor.type=socket', + '-C', 'input.sensor.port=9090', + '-C', 'input.sensor.tags=host,pod', + ]) + + assert result == { + 'verbose': True, + 'interval': 10, + 'input': { + 'sensor': { + 'type': 'socket', + 'host': 'localhost', + 'port': 9090, + 'tags': ['host', 'pod'], + }, + }, + } + + +def test_parse_validates_fixed_section_without_component_type(): + """ + Test that CLI assignments configure a fixed group section without a type. + """ + manager = ConfigurationParsingManager() + manager.add_group('formula') + smartwatts = ConfigurationSectionSchema() + smartwatts.add_argument('learn-error-window-size', argument_type=int) + manager.add_section('formula', 'smartwatts', smartwatts) + + result = manager.parse([ + 'powerapi', + '-C', 'formula.smartwatts.learn-error-window-size=10', + ]) + + assert result == { + 'formula': { + 'smartwatts': { + 'learn-error-window-size': 10, + }, + }, + } + + +@pytest.mark.parametrize('args', [ + ['powerapi', '--interval', '12'], + ['--interval', '12'], +]) +def test_parse_accepts_arguments_with_or_without_executable(parsing_manager, args): + """ + Test that parsing accepts argument lists with or without an executable name. + """ + assert parsing_manager.parse(args) == {'verbose': False, 'interval': 12} + + +def test_parse_uses_sys_argv_by_default(parsing_manager, monkeypatch): + """ + Test that parsing uses the process arguments when no argument list is provided. + """ + monkeypatch.setattr(sys, 'argv', ['powerapi', '--interval', '12']) + + assert parsing_manager.parse() == {'verbose': False, 'interval': 12} + + +def test_parse_applies_defaults_when_sources_are_empty(parsing_manager): + """ + Test that parsing an empty configuration applies root defaults. + """ + assert parsing_manager.parse([]) == {'verbose': False, 'interval': 10} + + +def test_parse_merges_sources_with_cli_then_environment_then_file_precedence(parsing_manager, monkeypatch, tmp_path): + """ + Test CLI, environment, and file precedence for root and component values. + """ + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(json.dumps({ + 'interval': 1, + 'input': { + 'sensor': { + 'type': 'socket', + 'host': 'file', + 'port': 1001, + 'tags': ['file'], + }, + }, + }), encoding='utf-8') + monkeypatch.setenv('TEST_POWERAPI_INTERVAL', '2') + monkeypatch.setenv('TEST_POWERAPI_INPUT_SENSOR_HOST', 'environment') + monkeypatch.setenv('TEST_POWERAPI_INPUT_SENSOR_PORT', '2002') + + result = parsing_manager.parse([ + 'powerapi', + '--config-file', str(config_file), + '-C', 'interval=3', + '-C', 'input.sensor.port=3003', + ]) + + assert result == { + 'verbose': False, + 'interval': 3, + 'input': { + 'sensor': { + 'type': 'socket', + 'host': 'environment', + 'port': 3003, + 'tags': ['file'], + }, + }, + } + + +def test_parse_resolves_environment_component_type_for_partial_file_configuration(parsing_manager, monkeypatch, tmp_path): + """ + Test that a partial file component inherits its type from the environment after merging. + """ + config_file = tmp_path / 'powerapi-pytest.json' + config_file.write_text(json.dumps({ + 'input': { + 'sensor': { + 'port': 9080, + }, + }, + }), encoding='utf-8') + monkeypatch.setenv('TEST_POWERAPI_INPUT_SENSOR_TYPE', 'socket') + + result = parsing_manager.parse([ + 'powerapi', + '--config-file', str(config_file), + ]) + + assert result['input']['sensor'] == { + 'type': 'socket', + 'host': 'localhost', + 'port': 9080, + } + + +def test_parse_propagates_cli_errors(parsing_manager): + """ + Test that command-line parsing errors propagate to the caller. + """ + with pytest.raises(CLIParseException, match='unrecognized arguments'): + parsing_manager.parse(['powerapi', '--unknown']) + + +def test_parse_propagates_configuration_errors(parsing_manager): + """ + Test that schema validation errors propagate with their configuration path. + """ + with pytest.raises(ConfigurationError) as result: + parsing_manager.parse(['powerapi', '-C', 'input.sensor.type=unknown']) + + assert result.value.path == 'input.sensor.type' + + +def test_parse_propagates_missing_configuration_file(parsing_manager, tmp_path): + """ + Test that selecting a missing configuration file raises FileNotFoundError. + """ + missing_file = tmp_path / 'powerapi-pytest-missing.json' + + with pytest.raises(FileNotFoundError): + parsing_manager.parse(['powerapi', '--config-file', str(missing_file)]) diff --git a/tests/unit/config/test_utils.py b/tests/unit/config/test_utils.py new file mode 100644 index 00000000..a229b0b6 --- /dev/null +++ b/tests/unit/config/test_utils.py @@ -0,0 +1,98 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest + +from powerapi.config._utils import merge_dictionaries, string_to_bool, string_to_list + + +@pytest.mark.parametrize('value', [' YES ', 'y', 'true', 't', '1']) +def test_string_to_bool_converts_true_values(value): + """ + Test that supported textual true values are normalized and converted. + """ + assert string_to_bool(value) is True + + +@pytest.mark.parametrize('value', [' No ', 'n', 'false', 'f', '0']) +def test_string_to_bool_converts_false_values(value): + """ + Test that supported textual false values are normalized and converted. + """ + assert string_to_bool(value) is False + + +def test_string_to_bool_rejects_unknown_value(): + """ + Test that an unsupported textual boolean value is rejected. + """ + with pytest.raises(ValueError, match='Invalid boolean value: invalid'): + string_to_bool('invalid') + + +@pytest.mark.parametrize(('value', 'expected'), [ + ('', []), + ('host', ['host']), + ('host, pod', ['host', 'pod']), +]) +def test_string_to_list_converts_comma_separated_values(value, expected): + """ + Test that comma-separated values are split and stripped. + """ + assert string_to_list(value) == expected + + +def test_merge_dictionaries_uses_last_configuration_as_highest_precedence(): + """ + Test that later configurations override earlier values while preserving nested values. + """ + config_file = {'stream': False, 'input': {'sensor': {'port': 8080, 'uri': 'file'}}} + environment = {'input': {'sensor': {'port': 9080}}} + cli = {'stream': True, 'input': {'sensor': {'uri': 'cli'}}} + + result = merge_dictionaries(config_file, environment, cli) + + assert result == { + 'stream': True, + 'input': {'sensor': {'port': 9080, 'uri': 'cli'}}, + } + + +def test_merge_dictionaries_does_not_modify_or_reuse_inputs(): + """ + Test that merged dictionaries do not share mutable values with their inputs. + """ + first = {'input': {'sensor': {'port': 8080, 'tags': ['host']}}} + second = {'input': {'sensor': {'uri': 'socket'}}} + + result = merge_dictionaries(first, second) + result['input']['sensor']['port'] = 9080 + result['input']['sensor']['tags'].append('socket') + + assert first == {'input': {'sensor': {'port': 8080, 'tags': ['host']}}} + assert second == {'input': {'sensor': {'uri': 'socket'}}} diff --git a/tests/unit/database/csv/test_driver.py b/tests/unit/database/csv/test_driver.py index 06673157..6d823a61 100644 --- a/tests/unit/database/csv/test_driver.py +++ b/tests/unit/database/csv/test_driver.py @@ -32,6 +32,7 @@ import pytest from powerapi.database.csv.driver import CSVInput, CSVInputFactory, CSVOutput, CSVOutputFactory +from powerapi.database.exceptions import ConnectionFailed from powerapi.report import FormulaReport, HWPCReport, PowerReport, Report @@ -66,6 +67,16 @@ def test_csv_input_factory_is_picklable() -> None: pickle.dumps(factory) +def test_csv_input_connect_with_missing_file_raise_connection_failed(tmp_path) -> None: + """ + CSV input should report a controlled connection failure when an input file cannot be opened. + """ + csv_input = CSVInput(HWPCReport, [str(tmp_path / 'missing.csv')]) + + with pytest.raises(ConnectionFailed): + csv_input.connect() + + @pytest.mark.parametrize('report_type', [PowerReport, FormulaReport]) def test_create_csv_output(report_type: type[Report]) -> None: """ diff --git a/tests/unit/database/json/test_driver.py b/tests/unit/database/json/test_driver.py new file mode 100644 index 00000000..49e2f17c --- /dev/null +++ b/tests/unit/database/json/test_driver.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest + +from powerapi.database.exceptions import ConnectionFailed +from powerapi.database.json.driver import JsonInput +from powerapi.report import HWPCReport + + +def test_json_input_connect_with_missing_file_raise_connection_failed(tmp_path) -> None: + """ + JSON input should report a controlled connection failure when its input file cannot be opened. + """ + json_input = JsonInput(HWPCReport, str(tmp_path / 'missing.jsonl'), 'auto') + + with pytest.raises(ConnectionFailed): + json_input.connect() diff --git a/tests/utils/cli/base_config_parser.py b/tests/utils/cli/base_config_parser.py index f174bfed..2c4f8000 100644 --- a/tests/utils/cli/base_config_parser.py +++ b/tests/utils/cli/base_config_parser.py @@ -78,36 +78,21 @@ def generate_cli_configuration_from_json_file(file_name: str) -> list: def generate_cli_configuration_from_dictionary(configuration: dict, group_name: str = '') -> list: """ - Generate a list with arguments defined in dictionary. The list always has arguments --arg1_name, arg1_value, - -arg2_name, arg2_value... Each - argument name has as prefix '-' if it is short (its length == 1) or '--' if it is long (its length>1) + Generate repeatable dotted CLI configuration assignments from a dictionary. :param str configuration: The dictionary with the configuration :param str group_name: The name of the group that is currently being created """ conf_as_list = [] for argument_name, argument_value in configuration.items(): - prefix = '--' - if len(argument_name) == 1: - prefix = '-' - - if not isinstance(argument_value, dict): - conf_as_list.append(prefix + argument_name) - conf_as_list.append(str(argument_value)) - + argument_path = '.'.join(filter(None, (group_name, argument_name))) + if isinstance(argument_value, dict): + conf_as_list.extend(generate_cli_configuration_from_dictionary( + configuration=argument_value, + group_name=argument_path, + )) else: - - if 'type' in argument_value: - conf_as_list.append(group_name) - conf_as_list.append(argument_value['type']) - argument_value.pop('type') - conf_as_list.append('--name') - conf_as_list.append(argument_name) - else: - group_name = prefix + argument_name - - conf_as_list.extend(generate_cli_configuration_from_dictionary(configuration=argument_value, - group_name=group_name)) + conf_as_list.extend(('-C', f'{argument_path}={argument_value}')) return conf_as_list diff --git a/tests/utils/cli/csv_input_output_stream_mode_enabled_configuration.json b/tests/utils/cli/csv_input_output_stream_mode_enabled_configuration.json index 0c0558bc..74e87e66 100644 --- a/tests/utils/cli/csv_input_output_stream_mode_enabled_configuration.json +++ b/tests/utils/cli/csv_input_output_stream_mode_enabled_configuration.json @@ -5,16 +5,14 @@ "puller": { "model": "HWPCReport", "type": "csv", - "files": ["/tmp/rapl.csv", "/tmp/msr.csv"], - "name": "my_puller" + "files": ["/tmp/rapl.csv", "/tmp/msr.csv"] } }, "output": { "pusher_power": { "type": "csv", "model": "PowerReport", - "directory": "/tmp/formula_results", - "name": "my_pusher" + "directory": "/tmp/formula_results" } } -} \ No newline at end of file +} diff --git a/tests/utils/cli/k8s_pre_processor_complete_configuration.json b/tests/utils/cli/k8s_pre_processor_complete_configuration.json index 151bc026..f81cb817 100644 --- a/tests/utils/cli/k8s_pre_processor_complete_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_complete_configuration.json @@ -18,7 +18,7 @@ }, "pre-processor": { "my_processor": { - "type": "k8s", + "type": "kubernetes", "api-mode": "manual", "puller": "one_puller" } diff --git a/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json index 5fd5213a..06c7787c 100644 --- a/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json @@ -18,7 +18,7 @@ }, "pre-processor": { "my_processor": { - "type": "k8s", + "type": "kubernetes", "api-mode": "manual", "puller": "non_existent_puller" } diff --git a/tests/utils/cli/several_inputs_outputs_stream_mode_enabled_configuration.json b/tests/utils/cli/several_inputs_outputs_stream_mode_enabled_configuration.json index 6010e4b8..11a8ec00 100644 --- a/tests/utils/cli/several_inputs_outputs_stream_mode_enabled_configuration.json +++ b/tests/utils/cli/several_inputs_outputs_stream_mode_enabled_configuration.json @@ -5,15 +5,13 @@ "puller2": { "model": "HWPCReport", "type": "csv", - "files": ["/tmp/rapl.csv", "/tmp/msr.csv"], - "name": "puller2" + "files": ["/tmp/rapl.csv", "/tmp/msr.csv"] }, "puller3": { "model": "HWPCReport", "type": "socket", "host": "localhost", - "port": 1111, - "name": "puller3" + "port": 1111 }, "puller4": { "model": "HWPCReport",