munet: run mutests from the munet cli

doc/source/mutest.rst

@@ -33,6 +33,15 @@ topology with the given configuration file and execute each test on the
 resulting topology. The munet topology is launched at the start and brought down
 at the end of each test script.
 
+Alternatively, mutests can be run from the munet CLI using the ``test`` command.
+Note that tests with side effects might fundamentally change the state of the
+topology.
+
+.. code-block:: console
+
+   $ sudo munet
+   > test mutest_*.py
+
 Log Files
 ---------
 

munet/cli.py

@@ -12,6 +12,7 @@
 import logging
 import multiprocessing
 import os
+from pathlib import Path
 import pty
 import re
 import readline
@@ -294,6 +295,7 @@ def make_help_str(unet):
   cli   :: open a secondary CLI window
   help  :: this help
   hosts :: list hosts
+  test  :: run mutests
   quit  :: quit the cli
 
   HOST can be a host or one of the following:
@@ -456,7 +458,7 @@ async def run_command(
             outf.write(f"------- End: {host} ------\n")
 
 
-cli_builtins = ["cli", "help", "hosts", "quit"]
+cli_builtins = ["cli", "help", "hosts", "test", "quit"]
 
 
 class Completer:
@@ -470,11 +472,34 @@ def complete(self, text, state):
         tokens = line.split()
         # print(f"\nXXX: tokens: {tokens} text: '{text}' state: {state}'\n")
 
-        first_token = not tokens or (text and len(tokens) == 1)
-
         # If we have already have a builtin command we are done
-        if tokens and tokens[0] in cli_builtins:
-            return [None]
+        if tokens:
+            if tokens[0] == "test":
+                return self.complete_test(tokens, text, state)
+            if tokens[0] in cli_builtins:
+                return [None]
+        return self.complete_cmd(tokens, text, state)
+
+    def complete_test(self, tokens, text, state):
+        file_select = "mutest_*.py"
+        config_dir = self.unet.config_dirname
+        tests_glob = Path(config_dir).rglob(file_select)
+        tests = {str(x.relative_to(Path(config_dir))) for x in tests_glob}
+
+        completes = {x + " " for x in tests if x.startswith(text)}
+
+        # remove any completions already present
+        if text:
+            done_set = set(tokens[:-1])
+        else:
+            done_set = set(tokens)
+        completes -= done_set
+        completes = sorted(completes) + [None]
+
+        return completes[state]
+
+    def complete_cmd(self, tokens, text, state):
+        first_token = not tokens or (text and len(tokens) == 1)
 
         cli_run_cmds = set(self.unet.cli_run_cmds.keys())
         top_run_cmds = {x for x in cli_run_cmds if self.unet.cli_run_cmds[x][3]}
@@ -543,6 +568,41 @@ async def doline(
             background,
         )
         return True
+    if cmd == "test":
+        from .mutest.__main__ import execute_test  # pylint: disable=import-outside-toplevel
+
+        tnum = 1
+        exec_path = unet.rundir.joinpath("munet-test-exec.log")
+        exec_path.parent.mkdir(parents=True, exist_ok=True)
+        exec_handler = logging.FileHandler(exec_path, "w")
+        exec_formatter = logging.Formatter(
+            "%(asctime)s %(levelname)5s: %(name)s: %(message)s"
+        )
+        exec_handler.setFormatter(exec_formatter)
+
+        for file_select in nline.split():
+            tests_glob = Path(unet.config_dirname).glob(file_select)
+            tests = {str(x) for x in tests_glob}
+
+            for test in tests:
+                # Get test case loggers
+                cli_logger = logging.getLogger(f"cli.mutest.output.{test}")
+                cli_reslog = logging.getLogger(f"cli.mutest.results.{test}")
+                cli_logger.addHandler(exec_handler)
+                cli_reslog.addHandler(exec_handler)
+
+                try:
+                    await execute_test(
+                        unet,
+                        Path(test),
+                        {},  # Use defaults
+                        tnum,
+                        cli_logger,
+                        cli_reslog,
+                    )
+                finally:
+                    tnum += 1
+        return True
 
     #
     # In window commands

munet/logconf.yaml

@@ -4,6 +4,9 @@ formatters:
     format: '%(asctime)s: %(levelname)s: %(message)s'
   precise:
     format: '%(asctime)s %(levelname)s: %(name)s: %(message)s'
+  result_color:
+    class: munet.mulog.ResultColorFormatter
+    format: '%(levelname)5s: %(message)s'
 
 handlers:
   console:
@@ -17,16 +20,33 @@ handlers:
     level: DEBUG
     filename: munet-exec.log
     mode: w
+  info_console:
+    level: INFO
+    class: logging.StreamHandler
+    formatter: result_color
+    stream: ext://sys.stderr
 
 root:
   level: DEBUG
   handlers: [ "console", "file" ]
 
-# these are some loggers that get used.
-# loggers:
-#   munet:
-#     level: DEBUG
-#     propagate: true
-#   munet.base.commander
-#     level: DEBUG
-#      propagate: true
+loggers:
+  # these are some loggers that get used.
+  #   munet:
+  #     level: DEBUG
+  #     propagate: true
+  #   munet.base.commander
+  #     level: DEBUG
+  #     propagate: true
+  cli.mutest.output:
+    level: DEBUG
+    # No default handlers. A file handler must be specified in code.
+    handlers: []
+    propagate: false
+  cli.mutest.results:
+    level: DEBUG
+    # Don't log to munet-exec.log, etc. by default in order to separate the test
+    # logging from the munet logging. Instead, default to only the console. For
+    # test logging, a file handler must be specified in code.
+    handlers: [ "info_console" ]
+    propagate: false

munet/mutest/__main__.py

@@ -33,7 +33,6 @@
 from munet.parser import async_build_topology
 from munet.parser import get_config
 
-
 # We want all but critical to fit in 5 characters for alignment
 logging.addLevelName(logging.WARNING, "WARN")
 root_logger = logging.getLogger("")
@@ -197,9 +196,10 @@ async def collect(args: Namespace):
 async def execute_test(
     unet: Munet,
     test: Path,
-    args: Namespace,
+    args: dict,
     test_num: int,
-    exec_handler: logging.Handler,
+    logger: logging.Logger,
+    reslog: logging.Logger,
 ) -> (int, int, int, Exception):
     """Execute a test case script.
 
@@ -209,18 +209,13 @@ async def execute_test(
     Args:
         unet: a running topology.
         test: path to the test case script file.
-        args: argparse results.
+        args: argparse results as a dict.
         test_num: the number of this test case in the run.
-        exec_handler: exec file handler to add to test loggers which do not propagate.
+        logger: logger to record test run info.
+        reslog: logger to record test results.
     """
     test_name = testname_from_path(test)
 
-    # Get test case loggers
-    logger = logging.getLogger(f"mutest.output.{test_name}")
-    reslog = logging.getLogger(f"mutest.results.{test_name}")
-    logger.addHandler(exec_handler)
-    reslog.addHandler(exec_handler)
-
     # We need to send an info level log to cause the speciifc handler to be
     # created, otherwise all these debug ones don't get through
     reslog.info("")
@@ -232,7 +227,7 @@ async def execute_test(
     targets["."] = unet
 
     tc = uapi.TestCase(
-        str(test_num), test_name, test, targets, args, logger, reslog, args.full_summary
+        str(test_num), test_name, test, targets, args, logger, reslog, args.get('full_summary', False)
     )
     try:
         passed, failed, e = tc.execute()
@@ -323,8 +318,14 @@ async def run_tests(args):
                             print_header(reslog, unet)
                             printed_header = True
 
+                        # Get test case loggers
+                        logger = logging.getLogger(f"mutest.output.{test_name}")
+                        reslog = logging.getLogger(f"mutest.results.{test_name}")
+                        logger.addHandler(exec_handler)
+                        reslog.addHandler(exec_handler)
+
                         passed, failed, e = await execute_test(
-                            unet, test, args, tnum, exec_handler
+                            unet, test, vars(args), tnum, logger, reslog
                         )
                 except KeyboardInterrupt as error:
                     errlog.warning("KeyboardInterrupt while running test %s", test_name)

munet/mutest/userapi.py

@@ -126,14 +126,14 @@ def pause_test(desc=""):
 
 
 def act_on_result(success, args, desc=""):
-    if args.pause:
+    if args.get('pause', False):
         pause_test(desc)
     elif success or len(desc) == 0:
         # No success on description-less steps are not considered errors.
         return
-    if args.cli_on_error:
+    if args.get('cli_on_error', False):
         raise CLIOnErrorError(desc)
-    if args.pause_on_error:
+    if args.get('pause_on_error', False):
         pause_test(desc)
 
 
@@ -201,7 +201,7 @@ def __init__(
         name: str,
         path: Path,
         targets: dict,
-        args: Namespace,
+        args: dict,
         output_logger: logging.Logger = None,
         result_logger: logging.Logger = None,
         full_summary: bool = False,