Merge pull request #15 from tomasohara/tom-dev

integrate tom-dev

Author: tomasohara

.github/workflows/tests.yml

@@ -27,13 +27,18 @@ jobs:
     ## TODO?: runs-on: ubuntu:act-20.04
     runs-on: ubuntu-20.04
     ## OLD: runs-on: ubuntu-latest
-    
+
+    container:
+      image: my-shell-scripts
+      options: --user=root
+
     steps:
     - uses: actions/checkout@v3
 
     - name: Build docker image
       run: |
-        docker build -t name:shell-scripts-dev -f- . <Dockerfile
+        ## OLD: docker build -t name:shell-scripts-dev -f- . <Dockerfile
+        docker build  --tag my-shell-scripts  -f- . <Dockerfile
 
     ## TEST
     - name: Configure Environment
@@ -46,4 +51,6 @@ jobs:
         ## OLD: docker run --env "DEBUG_LEVEL=5" --rm --mount type=bind,source="$(pwd)",target=/home/shell-scripts shell-scripts-dev
         ## TODO???: put repo under /mnt/local-shell-scripts and installed version under /home/shell-scripts
         ## OLD: docker run --env "DEBUG_LEVEL=5" --rm --mount type=bind,source="$(pwd)",target=/home/shell-scripts shell-scripts-dev
-        docker run --env "DEBUG_LEVEL=5" --rm --mount type=bind,source="$(pwd)",target=/home/shell-scripts ubuntu:act-20.04
+        ## OLD: docker run --env "DEBUG_LEVEL=5" --rm --mount type=bind,source="$(pwd)",target=/home/shell-scripts ubuntu:act-20.04
+        ## TEST:
+        docker run --env "DEBUG_LEVEL=5" --rm --mount type=bind,source="$(pwd)",target=/home/shell-scripts my-shell-scripts

check_errors.perl

@@ -96,8 +96,9 @@ BEGIN
     # TODO: rework error in line test to omit files
     # NOTE: It can be easier to add special-case rules rather than devise a general regex;
     # ex: 'error' occuring within a line even at word boundaries can be too broad.
-    elsif (## &debug_print(7, "here\n", 7) &&
-	   /^(ERROR|Error)\b/	   
+    elsif (## DEBUG: &debug_print(7, "here\n", 7) &&
+	   ## OLD: /^(ERROR|Error)\b/	   
+	   /^(Error)\b/i
 	   ## OLD: || /command not found/i
 	   ## NOTE: maldito modules package polutes environment and man page not clear about disabling
 	   || (/command not found/i && (! /Cannot switch to Modules/))

examples/extract-all-git-versions.bash

@@ -77,8 +77,10 @@ while read -r LINE; do
     COMMIT_DATE=$(echo "$LINE" | cut -d ' ' -f 1)
     COMMIT_SHA=$(echo "$LINE" | cut -d ' ' -f 2)
     ## DEBUG: echo "COUNT=$COUNT LINE=$LINE COMMIT_DATE=$COMMIT_DATE COMMIT_SHA=$COMMIT_SHA"
-    $verbose && printf '.'
-    git cat-file -p "$COMMIT_SHA:$REL_GIT_PATH_TO_FILE" > "$EXPORT_TO/$GIT_SHORT_FILENAME.$COUNT.$COMMIT_DATE"
+    ## OLD: $verbose && printf '.'
+    output_file="$EXPORT_TO/$GIT_SHORT_FILENAME.$COUNT.$COMMIT_DATE"
+    git cat-file -p "$COMMIT_SHA:$REL_GIT_PATH_TO_FILE" > "$output_file"
+    $verbose && echo "$output_file"
 done <"$info"
 
 # return success code

git-aliases.bash

@@ -772,8 +772,8 @@ function git-misc-alias-usage() {
     ## OLD:
     ## echo "   git mv --verbose old-file new-file"
     ## echo "   GIT_MESSAGE='renamed' GIT_SKIP_ADD=1 git-update-commit-push old-file new-file"
-    echo "    GIT_MESSAGE='renamed' git-rename-file old-file new-file"
-    echo "    GIT_MESSAGE='moved' git-move-to-dir dir file1 file2"
+    echo "    GIT_MESSAGE='renamed' git-rename-file OLD NEW-file"
+    echo "    GIT_MESSAGE='moved' git-move-to-dir DIR file1 file2"
     echo ""
     echo "To delete files (mucho cuidado):"
     echo "   git rm old-file"

local-workflows.sh

@@ -70,21 +70,25 @@ if [ "${RUN_BUILD:-1}" = "1" ]; then
     #    https://stackoverflow.com/questions/42297387/docker-build-with-build-arg-with-multiple-arguments
     ## TODO: docker build --build-arg "DEBUG_LEVEL=$DEBUG_LEVEL" --build-arg "GIT_BRANCH=$GIT_BRANCH" --platform linux/x86_64 --tag "$IMAGE_NAME" .
     # shellcheck disable=SC2086
-    docker build --build-arg "GIT_BRANCH=$GIT_BRANCH" --platform linux/x86_64 $BUILD_OPTS --tag "name:$IMAGE_NAME" .
+    ## OLD: docker build --build-arg "GIT_BRANCH=$GIT_BRANCH" --platform linux/x86_64 $BUILD_OPTS --tag "name:$IMAGE_NAME" .
+    ## TEST
+    docker build --build-arg "GIT_BRANCH=$GIT_BRANCH" --platform linux/x86_64 $BUILD_OPTS --tag my-shell-scripts .
 fi
 
 # Run the Github Actions workflow locally
 # TODO: get this to effing work [maldito act]
 if [ "${RUN_WORKFLOW:-1}" = "1" ]; then
     file="${WORKFLOW_FILE:-act.yml}"
-    ## TEST (Pass along debug level in evvironment)
-    echo "Running Github Actions locally"
+    ## TEST (Pass along debug level in environment)
+    echo "Running Github Actions locally w/ $file"
     # shellcheck disable=SC2086
     ## TODO: act --env DEBUG_LEVEL="$DEBUG_LEVEL" --container-architecture linux/amd64 --pull="$ACT_PULL" -platform "$ACT_WORKFLOW" --workflows ./.github/workflows/"$file" $RUN_OPTS
-    act --verbose --env DEBUG_LEVEL="$DEBUG_LEVEL" --container-architecture linux/amd64 --pull="$ACT_PULL" -platform "$ACT_WORKFLOW" --workflows ./.github/workflows/"$file" $RUN_OPTS
+    act --verbose --env DEBUG_LEVEL="$DEBUG_LEVEL" --container-architecture linux/amd64 --pull="$ACT_PULL" -platform my-shell-scripts --workflows ./.github/workflows/"$file" $RUN_OPTS
 fi
 
+# Run via docker directly
 if [ "${RUN_DOCKER:-1}" = "1" ]; then
    echo "Running Tests via Docker"
-   docker run -it --env DEBUG_LEVEL="$DEBUG_LEVEL" --mount type=bind,source="$(pwd)",target=/home/shell-scripts shell-scripts-dev
+   ## OLD: docker run -it --env DEBUG_LEVEL="$DEBUG_LEVEL" --mount type=bind,source="$(pwd)",target=/home/shell-scripts shell-scripts-dev
+   docker run -it --env DEBUG_LEVEL="$DEBUG_LEVEL" --mount type=bind,source="$(pwd)",target=/home/shell-scripts my-shell-scripts
 fi

simple_batspp.py

@@ -665,7 +665,7 @@ def _convert_to_bats(self, test):
                       ## TEST: ("" if not GLOBAL_SETUP.strip() else ("\t" + GLOBAL_SETUP + ";\n")) +
                       f'\ttestfolder="{TEMP_DIR}/{unspaced_title}"\n' +
                       f'\tmkdir --parents "$testfolder"\n' +
-                      (f'\tcommand cp ./*.* "$testfolder"\n' if COPY_DIR else '') +
+                      (f'\tcommand cp -R ./. "$testfolder"\n' if COPY_DIR else '') +
                       # note: warning added for sake of shellcheck
                       f'\tbuiltin cd "$testfolder" || echo Warning: Unable to "cd $testfolder"\n')
         setup_sans_prompt = my_re.sub(r'^\s*\$', '\t', setup, flags=my_re.MULTILINE)

tests/batspp_report.py

@@ -4,14 +4,17 @@
 # DESCRIPTION: Test automation & report generation for ipynb test files (for BatsPP 1.5.X).
 # BatsPP files are generated by default (stored at ./batspp-only).
 #
+# TODO2:
+# - use gh.form_path instead of f"{dir}/{file}" (for sake of Windows users).
+#
 
 """
   Test automation & report generation for BatsPP test files for Bash.
   The files can be generated from Jupyter .ipynb files (see ./batspp-only).
 """
 
 # Standard modules
-## TODO: import json
+import math
 
 # Installed modules
 import yaml
@@ -71,6 +74,8 @@
                                  "Regex for tests to include; ex: 'c.*' for debugging")
 SINGLE_STORE = system.getenv_bool("SINGLE_STORE", False,
                                   f"Whether to just use {BATSPP_OUTPUT_STORE} for all store dirs except kcov")
+CLEAN_DEFAULT = system.getenv_bool("CLEAN_OUTPUT", not debug.detailed_debugging(),
+                                   f"Whether to clean existing output by remove entire directories")
 ## NOTE: the code needs to be thoroughly revamped (e.g., currently puts .batspp in same place as .bats)
 if SINGLE_STORE:
     BATSPP_STORE = BATS_STORE = TXT_STORE = BATSPP_OUTPUT_STORE
@@ -112,17 +117,21 @@ def main():
     BATSPP_SWITCH_ARG = "switch"
     FORCE_ARG = "force"
     CLEAN_ARG = "clean"
+    DEFINITIONS_ARG = "definitions"
     main_app = Main(
         description=__doc__.format(script=gh.basename(__file__)).strip("\n"),
         boolean_options=[
             (NO_REPORTS_ARG, "No reports are generated, testfiles are shown"),
             (KCOV_REPORTS_ARG, f"KCOV (HTML based) reports generated, stored at {KCOV_STORE}"),
             (TEXT_REPORTS_ARG, f"Textfile based reports generated, stored at {TXT_STORE}"),
-            (ALL_REPORTS_ARG, "Generates report for all available testfiles (NOBATSPP testfiles were ignored by default)"),
+            (ALL_REPORTS_ARG, "Generates report for all available testfiles (n.b., NOBATSPP testfiles ignored by default)"),
             (FORCE_ARG, "Force running under admin-like account"),
             (CLEAN_ARG, "Remove output from previous runs; *** warning: this removes entire subdirectories"),
             (BATSPP_SWITCH_ARG, "Uses batspp library instead of ../simple_batspp.py script"),
         ],
+        text_options=[
+            (DEFINITIONS_ARG, "Script with alias definitions to be sourced"),
+        ],
         skip_input=False,
         manual_input=False,
         short_options=True,
@@ -134,10 +143,12 @@ def main():
     KCOV_OPTION = main_app.get_parsed_option(KCOV_REPORTS_ARG)
     ALL_OPTION = main_app.get_parsed_option(ALL_REPORTS_ARG)
     FORCE_OPTION = main_app.get_parsed_option(FORCE_ARG, UNDER_DOCKER)
-    CLEAN_OPTION = main_app.get_parsed_option(CLEAN_ARG)
+    CLEAN_OPTION = main_app.get_parsed_option(CLEAN_ARG, CLEAN_DEFAULT)
     BATSPP_SWITCH_OPTION = main_app.get_parsed_option(BATSPP_SWITCH_ARG)
     USE_SIMPLE_BATSPP = (not BATSPP_SWITCH_OPTION)
-    debug.trace_expr(4, NO_OPTION, TXT_OPTION, KCOV_OPTION, FORCE_OPTION, CLEAN_OPTION, BATSPP_SWITCH_OPTION, USE_SIMPLE_BATSPP)
+    DEFINITIONS_SCRIPT = main_app.get_parsed_option(DEFINITIONS_ARG)
+    debug.trace_expr(4, NO_OPTION, TXT_OPTION, KCOV_OPTION, FORCE_OPTION, CLEAN_OPTION, BATSPP_SWITCH_OPTION, USE_SIMPLE_BATSPP, DEFINITIONS_SCRIPT)
+    RUN_BATS = (TXT_OPTION or not NO_OPTION)
 
     # Do check for adminstrative user and exit unless --force
     is_admin = my_re.search(r"root|admin|adm", gh.run("groups"))
@@ -148,9 +159,9 @@ def main():
 
     # Cleanup up previous rusn
     # Warning: 'rm -rf' is a very dangerous command:
-    # it should only be done in temporary directories (i.e., not under
+    # it should only be done in temporary directories (e.g., not under repo)
     if CLEAN_OPTION:
-        if not NO_OPTION:
+        if RUN_BATS:
             gh.run(f"rm -rf {BATSPP_STORE}/*")
             gh.run(f"rm -rf {BATS_STORE}/*")
 
@@ -167,12 +178,14 @@ def run_batspp(input_file, output_file):
         real_output_file = output_file + ".out"
         log_file = output_file + ".log"
         debug.trace(5, f"run_batspp{(input_file, output_file)}")
+        source_spec = (f"--source '{DEFINITIONS_SCRIPT}'" if DEFINITIONS_SCRIPT else "")
         if USE_SIMPLE_BATSPP:
             # note: adds sentinels around paragraph segments for simpler parsing;
-            # also uses Bash instead of Bats and copies ./tests files into bats test dir (under temp)
-            run_output = gh.run(f"MATCH_SENTINELS=1 PARA_BLOCKS=1 BASH_EVAL=1 COPY_DIR=1 FORCE_RUN={FORCE_OPTION} python3 ../simple_batspp.py {input_file} --output {output_file} > {real_output_file} 2> {log_file}")
+            # uses Bash instead of Bats (to bypass need for global setup sections)
+            # and copies ./tests files into bats test dir (under temp).
+            run_output = gh.run(f"MATCH_SENTINELS=1 PARA_BLOCKS=1 BASH_EVAL=1 COPY_DIR=1 FORCE_RUN={FORCE_OPTION} python3 ../simple_batspp.py {input_file} --output {output_file} {source_spec} > {real_output_file} 2> {log_file}")
         else:
-            run_output = gh.run(f"batspp {input_file} --save {output_file} 2> {log_file}")
+            run_output = gh.run(f"batspp {input_file} --save {output_file} {source_spec} 2> {log_file}")
         debug.code(4, lambda: gh.run(f"check_errors.perl {log_file}"))
         debug.assertion(not run_output.strip())
         real_output = system.read_file(real_output_file)
@@ -207,15 +220,16 @@ def run_batspp(input_file, output_file):
         is_ipynb = file.endswith(IPYNB)
         if is_ipynb:
             if TEST_REGEX and not my_re.match(fr"{TEST_REGEX}", file):
-                debug.trace(3, f"Ignoring {file}")
+                debug.trace(3, f"FYI: Ignoring {file} not mathing TEST_REGEX ({TEST_REGEX})")
                 continue
             if not ALL_OPTION and NOBATSPP in file:
+                debug.trace(4, f"FYI: Ignoring NOBATSPP file: {file}")
                 print(f"NOBATSPP File Found [{i}]: {file}")
                 avoid_array.append(file)
                 avoid_count += 1
-            else:
-                print(f"JUPYTER Testfile Found [{i}]: {file}")
-                ipynb_array.append(file)
+                continue
+            print(f"JUPYTER Testfile Found [{i}]: {file}")
+            ipynb_array.append(file)
             i += 1
 
     print(f"\nIPYNB Files Found (Total - NOBATSPP): {i-1} - {avoid_count} = {i-avoid_count-1}")
@@ -233,7 +247,7 @@ def run_batspp(input_file, output_file):
                                              gh.basename(testfile.replace(IPYNB, BATSPP)))
             print(f"IPYNB TESTFILE [{i}]: {testfile} => {batspp_from_ipynb}")
             log_file = f"{batspp_from_ipynb}.log"
-            gh.run(f"python3 ../jupyter_to_batspp.py {testfile} --output {batspp_from_ipynb}")
+            gh.run(f"python3 ../jupyter_to_batspp.py {testfile} --output {batspp_from_ipynb} 2> {log_file}")
             # note: uses call to avoid issue with lambda function argument binding
             debug.call(4, gh.run, f"check_errors.perl {log_file}", **{"output": True})
             batspp_array.append(batspp_from_ipynb)
@@ -244,9 +258,14 @@ def run_batspp(input_file, output_file):
 
     # 3) Executing batspp files & storing them as bats
     print(f"\n\n==========BATS GENERATED==========\n")
+    # TODO3: rename i => num_test_files, total_count_ok => total_ok_tests, and total_count_total to total_num_tests
     i = 1
+    total_count_ok = 0
+    total_count_total = 0
+    total_success_rate = 0
+    total_num_successful = 0
 
-    if NO_OPTION:
+    if not RUN_BATS:
         print(f">> SKIPPING BATSPP CHECK (-n ARGUMENT PROVIDED)\n")
     else:
         for batsppfile_path in batspp_array:
@@ -266,34 +285,41 @@ def run_batspp(input_file, output_file):
             i += 1
 
             if TXT_OPTION:
-                bats_output = run_batspp(batsppfile_path, f"{BATSPP_OUTPUT_STORE}/{output_from_batspp}")
+                output_from_batspp_path = gh.form_path(BATSPP_OUTPUT_STORE, output_from_batspp)
+                bats_output = run_batspp(batsppfile_path, output_from_batspp_path)
 
                 output_lines = bats_output.splitlines()
                 output_lines_filtered = [item for item in output_lines if not item.startswith("#")]
                 debug.trace_expr(5, output_lines_filtered)
                 if output_lines_filtered:
                     # Ignore the line given the number of tests (e.g., "1..5")
                     header_line = output_lines_filtered.pop(0)
-                    debug.assertion(my_re.search(r"^1\.\.\d+", header_line))
+                    debug.trace_expr(5, header_line)
+                    debug.assertion(my_re.search(r"^1\.\.\d+", header_line) or (header_line == "0..0"),
+                                    f"Unexpected header line for {output_from_batspp_path}")
                 debug.assertion(len(output_lines_filtered))
 
                 count_ok = len([item for item in output_lines_filtered if item.startswith("ok")])
                 count_bad = len([item for item in output_lines_filtered if item.startswith("not ok")])
                 count_total = (count_ok + count_bad)
-                count_success_rate = (round((count_ok / count_total)*100, 2) if count_total else 0)
+                success_rate = (round((count_ok / count_total)*100, 2) if count_total else 0)
                 min_score = system.to_float(thresholds.get(ipynb_from_batspp, DEFAULT_MIN_SCORE))
-                count_success_rate_bool = (count_success_rate >= min_score)
-                debug.trace_expr(4, min_score, count_ok, count_bad, count_total, count_success_rate, count_success_rate_bool)
-                SUMMARY_TEXT = f"{count_ok} out of {count_total} successful ({count_success_rate}%)\nSuccess: {count_success_rate_bool}"
+                successful = (success_rate >= min_score)
+                debug.trace_expr(4, min_score, count_ok, count_bad, count_total, success_rate, successful)
+                SUMMARY_TEXT = f"{count_ok} out of {count_total} successful ({success_rate}%)\nSuccess: {successful}"
                 gh.write_file(f"{TXT_STORE}/{txt_from_batspp}", SUMMARY_TEXT)
                 print(f"{test_extensionless}: {SUMMARY_TEXT}")
+                total_count_ok += count_ok
+                total_count_total += count_total
+                total_success_rate += success_rate
+                total_num_successful += int(successful)
 
                 # Categorizing Tests if they are successful or not
                 txt_option_JSON = {}
                 txt_option_JSON["test_name"] = test_extensionless
                 txt_option_JSON["test_min_score"] = min_score
-                txt_option_JSON["test_success_rate"] = count_success_rate
-                if count_success_rate_bool:
+                txt_option_JSON["test_success_rate"] = success_rate
+                if successful:
                     success_test_array.append(txt_option_JSON)
                 else:
                     failure_test_array.append(txt_option_JSON)
@@ -326,24 +352,40 @@ def run_batspp(input_file, output_file):
     error_testfiles = [tf for tf in ipynb_array if tf not in set_wt]
     faulty_count = ipynb_count - batspp_count
 
+    NaN = math.nan
     print(f"\n======================================================")
     print(f"SUMMARY STATISTICS:\n")
     print(f"simple_batspp.py used: {bool(BATSPP_SWITCH_OPTION)}")
     print(f"No. of IPYNB testfiles: {ipynb_count + avoid_count}")
-    print(f"No. of BATSPP files (generated): {batspp_count if TXT_OPTION or NO_OPTION != 1 else 'NaN'}")
-    print(f"No. of FAULTY testfiles: {faulty_count if TXT_OPTION or NO_OPTION != 1 else 'NaN'}")
+    print(f"No. of BATSPP files (generated): {batspp_count if RUN_BATS else NaN}")
+    print(f"No. of FAULTY testfiles: {faulty_count if RUN_BATS else NaN}")
     print(f"No. of AVOIDED testfiles: {avoid_count}")
+    print(f"Total no. of good tests: {total_count_ok}")
+    print(f"Total no. of individual tests: {total_count_total}")
+    # Note: macro-average is mean of success score, whereas micro-average is based on global counts.
+    # See https://datascience.stackexchange.com/questions/15989/micro-average-vs-macro-average-performance-in-a-multiclass-classification-settin
+    avg_successful = macro_success_rate = micro_success_rate = NaN
+    if batspp_count:
+        avg_successful = total_num_successful / batspp_count * 100
+        macro_success_rate = total_success_rate / batspp_count
+        micro_success_rate = total_count_ok / total_count_total * 100
+    print(f"Total no. files OK w/ threshold: {total_num_successful}")
+    print(f"Average no. files OK / threshold: {system.round_num(avg_successful)}%")
+    print(f"Macro success score: {system.round_num(macro_success_rate)}%")
+    print(f"Micro success score: {system.round_num(micro_success_rate)}%")
+    print("    where successful macro is mean of individual scores and micro is global metric")
 
     print(f"\nFAULTY TESTFILES:")
     if faulty_count == 0:
-        print("NaN")
+        print("n/a")
     else:
+        ## TODO3: what is the intention here (e.g., the '>>')?
         for tf in error_testfiles:
             print(f">> {tf}")
 
     print("\nAVOIDED TESTFILES:")
     if avoid_count == 0:
-        print("NaN")
+        print("n/a")
     else:
         for tf in avoid_array:
             print(f">> {tf}")
@@ -352,6 +394,7 @@ def run_batspp(input_file, output_file):
         print("\nTEST SUCCESS (--txt ENABLED):")
         print(f"No. of Successful Tests:", len(success_test_array))
         print(f"No. of Failure Tests:", len(failure_test_array))
+        debug.assertion(len(success_test_array) == total_num_successful)
 
         def print_test_array(arr):
             """Print summary of test results in ARR"""
@@ -371,7 +414,7 @@ def print_test_array(arr):
     print(f"======================================================")
 
     # Return number of failed tests as statues (i.e., OK if 0 failed)
-    code = len(failure_test_array)
+    code = (len(failure_test_array) if success_test_array else -1)
     system.exit(status_code=code)
 
 # -------------------------------------------------------------------------------