diff --git a/doc/source/mutest.rst b/doc/source/mutest.rst index f07c357..6565774 100644 --- a/doc/source/mutest.rst +++ b/doc/source/mutest.rst @@ -33,6 +33,15 @@ topology with the given configuration file and execute each test on the resulting topology. The munet topology is launched at the start and brought down at the end of each test script. +Alternatively, mutests can be run from the munet CLI using the ``test`` command. +Note that tests with side effects might fundamentally change the state of the +topology. + +.. code-block:: console + + $ sudo munet + > test mutest_*.py + Log Files --------- diff --git a/munet/cli.py b/munet/cli.py index b7fc7ba..a588f86 100644 --- a/munet/cli.py +++ b/munet/cli.py @@ -12,6 +12,7 @@ import logging import multiprocessing import os +from pathlib import Path import pty import re import readline @@ -294,6 +295,7 @@ def make_help_str(unet): cli :: open a secondary CLI window help :: this help hosts :: list hosts + test :: run mutests quit :: quit the cli HOST can be a host or one of the following: @@ -456,7 +458,7 @@ async def run_command( outf.write(f"------- End: {host} ------\n") -cli_builtins = ["cli", "help", "hosts", "quit"] +cli_builtins = ["cli", "help", "hosts", "test", "quit"] class Completer: @@ -470,11 +472,34 @@ def complete(self, text, state): tokens = line.split() # print(f"\nXXX: tokens: {tokens} text: '{text}' state: {state}'\n") - first_token = not tokens or (text and len(tokens) == 1) - # If we have already have a builtin command we are done - if tokens and tokens[0] in cli_builtins: - return [None] + if tokens: + if tokens[0] == "test": + return self.complete_test(tokens, text, state) + if tokens[0] in cli_builtins: + return [None] + return self.complete_cmd(tokens, text, state) + + def complete_test(self, tokens, text, state): + file_select = "mutest_*.py" + config_dir = self.unet.config_dirname + tests_glob = Path(config_dir).rglob(file_select) + tests = {str(x.relative_to(Path(config_dir))) for x in tests_glob} + + completes = {x + " " for x in tests if x.startswith(text)} + + # remove any completions already present + if text: + done_set = set(tokens[:-1]) + else: + done_set = set(tokens) + completes -= done_set + completes = sorted(completes) + [None] + + return completes[state] + + def complete_cmd(self, tokens, text, state): + first_token = not tokens or (text and len(tokens) == 1) cli_run_cmds = set(self.unet.cli_run_cmds.keys()) top_run_cmds = {x for x in cli_run_cmds if self.unet.cli_run_cmds[x][3]} @@ -543,6 +568,41 @@ async def doline( background, ) return True + if cmd == "test": + from .mutest.__main__ import execute_test # pylint: disable=import-outside-toplevel + + tnum = 1 + exec_path = unet.rundir.joinpath("munet-test-exec.log") + exec_path.parent.mkdir(parents=True, exist_ok=True) + exec_handler = logging.FileHandler(exec_path, "w") + exec_formatter = logging.Formatter( + "%(asctime)s %(levelname)5s: %(name)s: %(message)s" + ) + exec_handler.setFormatter(exec_formatter) + + for file_select in nline.split(): + tests_glob = Path(unet.config_dirname).glob(file_select) + tests = {str(x) for x in tests_glob} + + for test in tests: + # Get test case loggers + cli_logger = logging.getLogger(f"cli.mutest.output.{test}") + cli_reslog = logging.getLogger(f"cli.mutest.results.{test}") + cli_logger.addHandler(exec_handler) + cli_reslog.addHandler(exec_handler) + + try: + await execute_test( + unet, + Path(test), + {}, # Use defaults + tnum, + cli_logger, + cli_reslog, + ) + finally: + tnum += 1 + return True # # In window commands diff --git a/munet/logconf.yaml b/munet/logconf.yaml index 430ee20..0c6a33d 100644 --- a/munet/logconf.yaml +++ b/munet/logconf.yaml @@ -4,6 +4,9 @@ formatters: format: '%(asctime)s: %(levelname)s: %(message)s' precise: format: '%(asctime)s %(levelname)s: %(name)s: %(message)s' + result_color: + class: munet.mulog.ResultColorFormatter + format: '%(levelname)5s: %(message)s' handlers: console: @@ -17,16 +20,33 @@ handlers: level: DEBUG filename: munet-exec.log mode: w + info_console: + level: INFO + class: logging.StreamHandler + formatter: result_color + stream: ext://sys.stderr root: level: DEBUG handlers: [ "console", "file" ] -# these are some loggers that get used. -# loggers: -# munet: -# level: DEBUG -# propagate: true -# munet.base.commander -# level: DEBUG -# propagate: true +loggers: + # these are some loggers that get used. + # munet: + # level: DEBUG + # propagate: true + # munet.base.commander + # level: DEBUG + # propagate: true + cli.mutest.output: + level: DEBUG + # No default handlers. A file handler must be specified in code. + handlers: [] + propagate: false + cli.mutest.results: + level: DEBUG + # Don't log to munet-exec.log, etc. by default in order to separate the test + # logging from the munet logging. Instead, default to only the console. For + # test logging, a file handler must be specified in code. + handlers: [ "info_console" ] + propagate: false diff --git a/munet/mutest/__main__.py b/munet/mutest/__main__.py index 684d67c..a3e8aee 100644 --- a/munet/mutest/__main__.py +++ b/munet/mutest/__main__.py @@ -33,7 +33,6 @@ from munet.parser import async_build_topology from munet.parser import get_config - # We want all but critical to fit in 5 characters for alignment logging.addLevelName(logging.WARNING, "WARN") root_logger = logging.getLogger("") @@ -197,9 +196,10 @@ async def collect(args: Namespace): async def execute_test( unet: Munet, test: Path, - args: Namespace, + args: dict, test_num: int, - exec_handler: logging.Handler, + logger: logging.Logger, + reslog: logging.Logger, ) -> (int, int, int, Exception): """Execute a test case script. @@ -209,18 +209,13 @@ async def execute_test( Args: unet: a running topology. test: path to the test case script file. - args: argparse results. + args: argparse results as a dict. test_num: the number of this test case in the run. - exec_handler: exec file handler to add to test loggers which do not propagate. + logger: logger to record test run info. + reslog: logger to record test results. """ test_name = testname_from_path(test) - # Get test case loggers - logger = logging.getLogger(f"mutest.output.{test_name}") - reslog = logging.getLogger(f"mutest.results.{test_name}") - logger.addHandler(exec_handler) - reslog.addHandler(exec_handler) - # We need to send an info level log to cause the speciifc handler to be # created, otherwise all these debug ones don't get through reslog.info("") @@ -232,7 +227,7 @@ async def execute_test( targets["."] = unet tc = uapi.TestCase( - str(test_num), test_name, test, targets, args, logger, reslog, args.full_summary + str(test_num), test_name, test, targets, args, logger, reslog, args.get('full_summary', False) ) try: passed, failed, e = tc.execute() @@ -323,8 +318,14 @@ async def run_tests(args): print_header(reslog, unet) printed_header = True + # Get test case loggers + logger = logging.getLogger(f"mutest.output.{test_name}") + reslog = logging.getLogger(f"mutest.results.{test_name}") + logger.addHandler(exec_handler) + reslog.addHandler(exec_handler) + passed, failed, e = await execute_test( - unet, test, args, tnum, exec_handler + unet, test, vars(args), tnum, logger, reslog ) except KeyboardInterrupt as error: errlog.warning("KeyboardInterrupt while running test %s", test_name) diff --git a/munet/mutest/userapi.py b/munet/mutest/userapi.py index b50d082..04604d4 100644 --- a/munet/mutest/userapi.py +++ b/munet/mutest/userapi.py @@ -126,14 +126,14 @@ def pause_test(desc=""): def act_on_result(success, args, desc=""): - if args.pause: + if args.get('pause', False): pause_test(desc) elif success or len(desc) == 0: # No success on description-less steps are not considered errors. return - if args.cli_on_error: + if args.get('cli_on_error', False): raise CLIOnErrorError(desc) - if args.pause_on_error: + if args.get('pause_on_error', False): pause_test(desc) @@ -201,7 +201,7 @@ def __init__( name: str, path: Path, targets: dict, - args: Namespace, + args: dict, output_logger: logging.Logger = None, result_logger: logging.Logger = None, full_summary: bool = False,