diff --git a/.github/linters/.ruff.toml b/.github/linters/.ruff.toml index de9a8356..9cb5f660 100644 --- a/.github/linters/.ruff.toml +++ b/.github/linters/.ruff.toml @@ -1,5 +1,6 @@ cache-dir = "/tmp/.ruff_cache" line-length = 88 +output-format = "grouped" [lint] # Allow unused variables when underscore-prefixed. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5419cc74..d91c7944 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -42,8 +42,6 @@ jobs: # Once the codebase is perfectly linted/formatted, switch to validating only changed files # VALIDATE_ALL_CODEBASE: false # lint and format new or changed files only FILTER_REGEX_EXCLUDE: (.*[.]conf.py|pull_request_template.md) - FIX_PYTHON_RUFF_FORMAT: true - FIX_PYTHON_RUFF: true IGNORE_GITIGNORED_FILES: true VALIDATE_BASH: true VALIDATE_CHECKOV: true diff --git a/fcm_bdiff/fcm_bdiff.py b/fcm_bdiff/fcm_bdiff.py index f6def457..74a84c2a 100644 --- a/fcm_bdiff/fcm_bdiff.py +++ b/fcm_bdiff/fcm_bdiff.py @@ -39,7 +39,7 @@ class FCMBase: Note that the version for Git has a small handful of methods, mostly internal and some propeties. These are kept as close as possible to version in git_bdiff.py. - + Attributes used to navigate the horros of FCM and thus used in this package are therefore preceded with an '_' and shouldn't be what is being referred to outwith this class. Nor should the original @@ -60,7 +60,7 @@ def __init__(self, parent=None, repo=None): kind of 'discovery' that was necessary for FCM that is hoped to become outdated with Git. """ - + # use_mirror checks for SOURCE_UM_MIRROR env var, and if set # redirects the branch to that value and sets retries. # Otherwise it returns the branch unchanged and retries=0 @@ -88,7 +88,7 @@ def get_branch_name(self): For now, define it to be the contants of the URL after .*/main/ has been stripped off. i.e. it will start with trunk/... or branches/... """ - + pattern = rf"{self.get_repository_root()}/main/(.*)$" match = re.match(pattern, self._branch_url) if match: @@ -101,7 +101,7 @@ def run_fcm_command(self, command, max_retries, snooze): """ Run an fcm command, optionally retrying on failure. """ - + retries = 0 while True: result = subprocess.run( @@ -156,7 +156,7 @@ def get_branch_parent(self): Given the raw output from an fcm binfo command - which can be retrieved by calling get_branch_info() - returns the Branch Parent Field """ - + parent = re.search( r"^Branch Parent:\s*(?P.*)$", self._branch_info, @@ -182,7 +182,7 @@ def get_url(self): Given the raw output from an fcm binfo command - which can be retrieved by calling get_branch_info() - returns the URL field """ - + url = re.search(r"^URL:\s*(?P.*)$", self._branch_info, flags=re.MULTILINE) if url: url = url.group("url") @@ -195,7 +195,7 @@ def is_trunk_test(self, url): Given an FCM url, returns True if it appears to be pointing to the UM main trunk """ - + search = re.search( r""" (svn://fcm\d+/\w+_svn/\w+/trunk| @@ -212,7 +212,7 @@ def get_repository_root(self): Given the raw output from an fcm binfo command - which can be retrieved by calling get_branch_info() - returns the Repository Root field """ - + repos_root = re.search( r"^Repository Root:\s*(?P.*)\s*$", self._branch_info, @@ -229,7 +229,7 @@ def get_latest_commit(self): Given the raw output from an fcm binfo command - which can be retrieved by calling get_branch_info() - returns the Last Changed Rev """ - + repos_rev = re.search( r"^Last Changed Rev:\s*(?P.*)\s*$", self._branch_info, @@ -263,7 +263,7 @@ def has_diverged(self): Bit vague here, so we're going to check to see if 'parent' had an '@' in it denoting it's a branch of """ - + match = re.match(r".*@(\d+)$", self.parent) if match: return True @@ -272,7 +272,7 @@ def has_diverged(self): def files(self): """Iterate over files changed on the branch.""" - + dem_danged_files = self._get_files() for line in dem_danged_files: if line != "": @@ -335,7 +335,7 @@ def get_bdiff_summarize(self, snooze=300, retries=0): (if the branch is the mirror, allow for a few retries in case it hasn't picked up the latest commit yet) """ - + command = ["fcm", "bdiff", "--summarize", self._branch] return self.run_fcm_command(command, retries, snooze) @@ -352,5 +352,5 @@ def __init__(self, branch_info: str): def is_main(self) -> bool: """Return True if the branch is the main trunk.""" - + return self.is_trunk_test(self._branch_url) diff --git a/gh_review_project/cr_deadline.py b/gh_review_project/cr_deadline.py index 0ab4f8f0..3064d7ff 100644 --- a/gh_review_project/cr_deadline.py +++ b/gh_review_project/cr_deadline.py @@ -37,8 +37,7 @@ def parse_args(): testfile_path = Path(__file__).parent / "test" parser = argparse.ArgumentParser( - "Changes to the Simulation System projects required at the code" - "review deadline." + "Changes to the Simulation System projects required at the codereview deadline." ) parser.add_argument("--milestone", help="Milestone being released") @@ -79,7 +78,6 @@ def parse_args(): def main( milestone: str, test: bool, capture_project: bool, file: Path, dry: bool ) -> None: - # Get milestone data if test: issue_data = ProjectData.from_file(ISSUE_ID, file / "issue.json") diff --git a/gh_review_project/finish_milestone.py b/gh_review_project/finish_milestone.py index 5f3b9c58..b48b92b9 100644 --- a/gh_review_project/finish_milestone.py +++ b/gh_review_project/finish_milestone.py @@ -13,6 +13,7 @@ * Remaining open PRs and issues against this milestone * Closed PRs against this milestone """ + from pathlib import Path import argparse from review_project import ProjectData, REVIEW_ID, ISSUE_ID @@ -186,7 +187,6 @@ def parse_args(): def main( milestone: str, test: bool, capture_project: bool, file: Path, dry: bool ) -> None: - # Get milestone data if test: review_data = ProjectData.from_file(REVIEW_ID, file / "pr.json") diff --git a/gh_review_project/review_project.py b/gh_review_project/review_project.py index a5d9ee36..1c07b74d 100644 --- a/gh_review_project/review_project.py +++ b/gh_review_project/review_project.py @@ -8,6 +8,7 @@ Classes and functions for interacting with the Simulation Systems Review Tracker Project. """ + from __future__ import annotations import json @@ -248,7 +249,6 @@ def get_milestone(self, milestone: str, status: str = "all") -> dict: or (status == "open" and item.status in item.open_states) or (status == "closed" and item.status not in item.open_states) ): - milestone_data[item.repo].append(item) return milestone_data @@ -357,7 +357,7 @@ def modify_milestone(self, milestone: str, dry_run: bool = False) -> None: if milestone: command += f" --milestone='{milestone}'" else: - command += f" --remove-milestone" + command += " --remove-milestone" message = f"Changing milestone for #{self.number} in {self.repo}" diff --git a/gh_review_project/workload.py b/gh_review_project/workload.py index 7597b180..63748adf 100644 --- a/gh_review_project/workload.py +++ b/gh_review_project/workload.py @@ -202,7 +202,6 @@ def parse_args(): def main(total: bool, test: bool, capture_project: bool, file: Path): - # Extract data from github about the reviews and team members. if test: data = ProjectData.from_file(REVIEW_ID, file) diff --git a/kgo_updates/kgo_update/kgo_update.py b/kgo_updates/kgo_update/kgo_update.py index 901d0fe4..bc6ce86e 100755 --- a/kgo_updates/kgo_update/kgo_update.py +++ b/kgo_updates/kgo_update/kgo_update.py @@ -31,6 +31,7 @@ """ + import argparse import os import re @@ -208,7 +209,6 @@ def group_comparisons_by_dir(comparisons, skip=False): """ kgo_dirs = defaultdict(dict) for _, kgo_file, suite_file, status, _ in comparisons: - # If the kgo_file isn't set, move on (in some cases the test is not # comparing two files at all if kgo_file is None: @@ -237,17 +237,13 @@ def group_comparisons_by_dir(comparisons, skip=False): # If the above goes wrong it will eventually hit root; if this happens # we cannot continue as something is seriously not right if basedir == "/": - msg = ( - "Problem locating KGO directory - " - "is this actually a KGO file?\n {0}" - ) + msg = "Problem locating KGO directory - is this actually a KGO file?\n {0}" sys.exit(msg.format(kgo_file)) # Otherwise add the result to the list - for each entry we store the # relative path to the KGO file and the full path to the file in the # suite which should be used to update it (if it needs updating) if status.strip() in UPDATE_CRITERIA: - # Here we could check if this update has already been lodged, # and if it has perhaps we can do some additional checking # (for instance, are the changes in answers the same?) @@ -501,7 +497,7 @@ def _split_lines(self, text, width): "-E", "--extension", default=".cylc", - help="The extension of the variables file, either .rc " "or .cylc", + help="The extension of the variables file, either .rc or .cylc", ) args = parser.parse_args() @@ -607,8 +603,7 @@ def _split_lines(self, text, width): # Open the file for viewing in user's editor if interactive: print( - f"\n\nOpening {script_path}\nHit Return to Step through, " - "q to print all\n\n" + f"\n\nOpening {script_path}\nHit Return to Step through, q to print all\n\n" ) with open(script_path, "r") as f: line_count = 0 diff --git a/lfric_macros/files/template_versions.py b/lfric_macros/files/template_versions.py index fca00431..63fe7648 100644 --- a/lfric_macros/files/template_versions.py +++ b/lfric_macros/files/template_versions.py @@ -1,6 +1,7 @@ import sys -from metomi.rose.upgrade import MacroUpgrade + +from metomi.rose.upgrade import MacroUpgrade # noqa: F401 class UpgradeError(Exception): diff --git a/lfric_macros/tests/test_lfric_apps_dir/test_read_python_imports.py b/lfric_macros/tests/test_lfric_apps_dir/test_read_python_imports.py index bc1333d0..77fb6869 100644 --- a/lfric_macros/tests/test_lfric_apps_dir/test_read_python_imports.py +++ b/lfric_macros/tests/test_lfric_apps_dir/test_read_python_imports.py @@ -1,4 +1,4 @@ -import os -from shlex import split -from pathlib import Path, PurePath -import networkx as nx +import os # noqa: F401 +from shlex import split # noqa: F401 +from pathlib import Path, PurePath # noqa: F401 +import networkx as nx # noqa: F401 diff --git a/lfric_styling/styling_keywords.py b/lfric_styling/styling_keywords.py index 82444565..3fbc4e3b 100644 --- a/lfric_styling/styling_keywords.py +++ b/lfric_styling/styling_keywords.py @@ -9,6 +9,7 @@ """ Newer keywords used during physics forking. """ + NEW_KEYWORDS = [ "abort", "abs", diff --git a/script_copyright_checker/bin/copyright_checker.py b/script_copyright_checker/bin/copyright_checker.py index 0b841c00..65b5efab 100755 --- a/script_copyright_checker/bin/copyright_checker.py +++ b/script_copyright_checker/bin/copyright_checker.py @@ -8,6 +8,7 @@ Script which tests code files within the UM repository to ensure they contain a recognised copyright notice. """ + import argparse import os import re @@ -208,9 +209,7 @@ def parse_options(): "--full_trunk", action="store_true", default=False, - help=( - "run on use the full file list when trunk, " "else run on fcm branch-diff" - ), + help=("run on use the full file list when trunk, else run on fcm branch-diff"), ) excl_group.add_argument( "files", diff --git a/script_umdp3_checker/checker_dispatch_tables.py b/script_umdp3_checker/checker_dispatch_tables.py index fe01d5a8..51aac77f 100644 --- a/script_umdp3_checker/checker_dispatch_tables.py +++ b/script_umdp3_checker/checker_dispatch_tables.py @@ -35,53 +35,32 @@ def get_diff_dispatch_table_fortran(self) -> Dict[str, Callable]: return { # 'Captain Daves doomed test of destruction': # self.umdp3_checker.capitulated_keywords, - "Lowercase Fortran keywords not permitted": - self.umdp3_checker.capitalised_keywords, - "OpenMP sentinels not in column one": - self.umdp3_checker.openmp_sentinels_in_column_one, - "Omitted optional space in keywords": - self.umdp3_checker.unseparated_keywords, + "Lowercase Fortran keywords not permitted": self.umdp3_checker.capitalised_keywords, + "OpenMP sentinels not in column one": self.umdp3_checker.openmp_sentinels_in_column_one, + "Omitted optional space in keywords": self.umdp3_checker.unseparated_keywords, "GO TO other than 9999": self.umdp3_checker.go_to_other_than_9999, - "WRITE without format": - self.umdp3_checker.write_using_default_format, - "Lowercase or CamelCase variable names only": - self.umdp3_checker.lowercase_variable_names, - "Use of dimension attribute": - self.umdp3_checker.dimension_forbidden, - "Continuation lines shouldn't start with &": - self.umdp3_checker.ampersand_continuation, - "Use of EQUIVALENCE or PAUSE": - self.umdp3_checker.forbidden_keywords, - "Use of older form of relational operator (.GT. etc.)": - self.umdp3_checker.forbidden_operators, - "Line longer than 80 characters": - self.umdp3_checker.line_over_80chars, + "WRITE without format": self.umdp3_checker.write_using_default_format, + "Lowercase or CamelCase variable names only": self.umdp3_checker.lowercase_variable_names, + "Use of dimension attribute": self.umdp3_checker.dimension_forbidden, + "Continuation lines shouldn't start with &": self.umdp3_checker.ampersand_continuation, + "Use of EQUIVALENCE or PAUSE": self.umdp3_checker.forbidden_keywords, + "Use of older form of relational operator (.GT. etc.)": self.umdp3_checker.forbidden_operators, + "Line longer than 80 characters": self.umdp3_checker.line_over_80chars, "Line includes tab character": self.umdp3_checker.tab_detection, - "USEd printstatus_mod instead of umPrintMgr": - self.umdp3_checker.printstatus_mod, - "Used PRINT rather than umMessage and umPrint": - self.umdp3_checker.printstar, - "Used WRITE(6) rather than umMessage and umPrint": - self.umdp3_checker.write6, - "Used um_fort_flush rather than umPrintFlush": - self.umdp3_checker.um_fort_flush, - "Used Subversion keyword substitution which is prohibited": - self.umdp3_checker.svn_keyword_subst, - "Used !OMP instead of !$OMP": - self.umdp3_checker.omp_missing_dollar, - "Used #ifdef/#ifndef rather than #if defined() or #if !defined()": - self.umdp3_checker.cpp_ifdef, - "Presence of fortran comment in CPP directive": - self.umdp3_checker.cpp_comment, - "Used an archaic fortran intrinsic function": - self.umdp3_checker.obsolescent_fortran_intrinsic, - "EXIT statements should be labelled": - self.umdp3_checker.exit_stmt_label, - "Intrinsic modules must be USEd with an INTRINSIC " + - "keyword specifier": self.umdp3_checker.intrinsic_modules, - "READ statements should have an explicit UNIT= as " + - "their first argument": - self.umdp3_checker.read_unit_args, + "USEd printstatus_mod instead of umPrintMgr": self.umdp3_checker.printstatus_mod, + "Used PRINT rather than umMessage and umPrint": self.umdp3_checker.printstar, + "Used WRITE(6) rather than umMessage and umPrint": self.umdp3_checker.write6, + "Used um_fort_flush rather than umPrintFlush": self.umdp3_checker.um_fort_flush, + "Used Subversion keyword substitution which is prohibited": self.umdp3_checker.svn_keyword_subst, + "Used !OMP instead of !$OMP": self.umdp3_checker.omp_missing_dollar, + "Used #ifdef/#ifndef rather than #if defined() or #if !defined()": self.umdp3_checker.cpp_ifdef, + "Presence of fortran comment in CPP directive": self.umdp3_checker.cpp_comment, + "Used an archaic fortran intrinsic function": self.umdp3_checker.obsolescent_fortran_intrinsic, + "EXIT statements should be labelled": self.umdp3_checker.exit_stmt_label, + "Intrinsic modules must be USEd with an INTRINSIC " + + "keyword specifier": self.umdp3_checker.intrinsic_modules, + "READ statements should have an explicit UNIT= as " + + "their first argument": self.umdp3_checker.read_unit_args, } def get_file_dispatch_table_fortran( @@ -89,64 +68,48 @@ def get_file_dispatch_table_fortran( ) -> Dict[str, Callable]: """Get dispatch table for Fortran file tests""" return { - "Warning - used an if-def due for retirement": - self.umdp3_checker.retire_if_def, - "File is missing at least one IMPLICIT NONE": - self.umdp3_checker.implicit_none, + "Warning - used an if-def due for retirement": self.umdp3_checker.retire_if_def, + "File is missing at least one IMPLICIT NONE": self.umdp3_checker.implicit_none, "Never use STOP or CALL abort": self.umdp3_checker.forbidden_stop, - "Use of Fortran function as a variable name": - self.umdp3_checker.intrinsic_as_variable, - "File missing crown copyright statement or agreement reference": - self.umdp3_checker.check_crown_copyright, - "File missing correct code owner comment": - self.umdp3_checker.check_code_owner, - "Used (/ 1,2,3 /) form of array initialisation, rather than " + - "[1,2,3] form": self.umdp3_checker.array_init_form, + "Use of Fortran function as a variable name": self.umdp3_checker.intrinsic_as_variable, + "File missing crown copyright statement or agreement reference": self.umdp3_checker.check_crown_copyright, + "File missing correct code owner comment": self.umdp3_checker.check_code_owner, + "Used (/ 1,2,3 /) form of array initialisation, rather than " + + "[1,2,3] form": self.umdp3_checker.array_init_form, } def get_diff_dispatch_table_c(self) -> Dict[str, Callable]: """Get dispatch table for C diff tests""" return { - "Line longer than 80 characters": - self.umdp3_checker.line_over_80chars, + "Line longer than 80 characters": self.umdp3_checker.line_over_80chars, "Line includes tab character": self.umdp3_checker.tab_detection, - "Fixed-width Integer format specifiers must have a space " + - "between themselves and the string delimiter (the \" character)": - self.umdp3_checker.c_integral_format_specifiers, + "Fixed-width Integer format specifiers must have a space " + + 'between themselves and the string delimiter (the " character)': self.umdp3_checker.c_integral_format_specifiers, } def get_file_dispatch_table_c(self) -> Dict[str, Callable]: """Get dispatch table for C file tests""" return { - "Warning - used an if-def due for retirement": - self.umdp3_checker.retire_if_def, + "Warning - used an if-def due for retirement": self.umdp3_checker.retire_if_def, "Used a deprecated C identifier": self.umdp3_checker.c_deprecated, - "File missing crown copyright statement or agreement reference": - self.umdp3_checker.check_crown_copyright, - "File missing correct code owner comment": - self.umdp3_checker.check_code_owner, - "Used an _OPENMP if-def without also testing against " + - "SHUM_USE_C_OPENMP_VIA_THREAD_UTILS. (Or _OPENMP does " + - "not come first in the test.)": - self.umdp3_checker.c_openmp_define_pair_thread_utils, - "Used an _OPENMP && SHUM_USE_C_OPENMP_VIA_THREAD_UTILS if-def " + - "test in a logical combination with a third macro": - self.umdp3_checker.c_openmp_define_no_combine, - "Used !defined(_OPENMP) rather than defined(_OPENMP) " + - "with #else branch": self.umdp3_checker.c_openmp_define_not, - "Used an omp #pragma (or #include ) without " + - "protecting it with an _OPENMP if-def": - self.umdp3_checker.c_protect_omp_pragma, - "Used the #ifdef style of if-def, rather than the #if " + - "defined() style": - self.umdp3_checker.c_ifdef_defines, - "C Unit does not end with a final newline character": - self.umdp3_checker.c_final_newline, + "File missing crown copyright statement or agreement reference": self.umdp3_checker.check_crown_copyright, + "File missing correct code owner comment": self.umdp3_checker.check_code_owner, + "Used an _OPENMP if-def without also testing against " + + "SHUM_USE_C_OPENMP_VIA_THREAD_UTILS. (Or _OPENMP does " + + "not come first in the test.)": self.umdp3_checker.c_openmp_define_pair_thread_utils, + "Used an _OPENMP && SHUM_USE_C_OPENMP_VIA_THREAD_UTILS if-def " + + "test in a logical combination with a third macro": self.umdp3_checker.c_openmp_define_no_combine, + "Used !defined(_OPENMP) rather than defined(_OPENMP) " + + "with #else branch": self.umdp3_checker.c_openmp_define_not, + "Used an omp #pragma (or #include ) without " + + "protecting it with an _OPENMP if-def": self.umdp3_checker.c_protect_omp_pragma, + "Used the #ifdef style of if-def, rather than the #if " + + "defined() style": self.umdp3_checker.c_ifdef_defines, + "C Unit does not end with a final newline character": self.umdp3_checker.c_final_newline, } def get_file_dispatch_table_all(self) -> Dict[str, Callable]: """Get dispatch table for universal file tests""" return { - "Line includes trailing whitespace character(s)": - self.umdp3_checker.line_trail_whitespace, + "Line includes trailing whitespace character(s)": self.umdp3_checker.line_trail_whitespace, } diff --git a/script_umdp3_checker/tests/test_umdp3.py b/script_umdp3_checker/tests/test_umdp3.py index 0fa51329..2f4a5db8 100644 --- a/script_umdp3_checker/tests/test_umdp3.py +++ b/script_umdp3_checker/tests/test_umdp3.py @@ -25,7 +25,7 @@ def test_basic_functionality(): """Test basic UMDP3 functionality""" - + print("Testing basic UMDP3 functionality...") # Initialize UMDP3Checker @@ -80,7 +80,7 @@ def test_basic_functionality(): def test_dispatch_tables(): """Test dispatch tables""" - + print("\nTesting dispatch tables...") dispatch = CheckerDispatchTables() @@ -104,7 +104,7 @@ def test_dispatch_tables(): def test_fortran_specific(): """Test Fortran-specific checks""" - + print("\nTesting Fortran-specific checks...") umdp3_checker = UMDP3Checker() @@ -136,7 +136,7 @@ def test_fortran_specific(): def test_c_specific(): """Test C-specific checks""" - + print("\nTesting C-specific checks...") umdp3_checker = UMDP3Checker() @@ -164,7 +164,7 @@ def test_c_specific(): def create_test_files(): """Create test files for full integration test""" - + print("\nCreating test files...") # Create temporary directory @@ -233,7 +233,7 @@ def test_function(): def main(): """Main test function""" - + print("UMDP3 Python Translation Test Suite") print("=" * 40) diff --git a/script_umdp3_checker/umdp3_checker_rules.py b/script_umdp3_checker/umdp3_checker_rules.py index 02254c58..c18270da 100644 --- a/script_umdp3_checker/umdp3_checker_rules.py +++ b/script_umdp3_checker/umdp3_checker_rules.py @@ -218,7 +218,7 @@ def capitalised_keywords(self, lines: List[str]) -> TestResult: checker_name="Capitalised Keywords", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -234,7 +234,7 @@ def openmp_sentinels_in_column_one(self, lines: List[str]) -> TestResult: error_log = self.add_error_log( error_log, "OpenMP sentinel not in column 1:", count + 1 ) - output = f"Checked {count+1} lines, found {failures} failures." + output = f"Checked {count + 1} lines, found {failures} failures." return TestResult( checker_name="OpenMP sentinels not in column one", failure_count=failures, @@ -252,18 +252,16 @@ def unseparated_keywords(self, lines: List[str]) -> TestResult: if line.lstrip(" ").startswith("!"): continue clean_line = self.remove_quoted(line) - for pattern in [f"\\b{kw}\\b" for kw in - unseparated_keywords_list]: + for pattern in [f"\\b{kw}\\b" for kw in unseparated_keywords_list]: if re.search(pattern, clean_line, re.IGNORECASE): - self.add_extra_error(f"unseparated keyword in line: " - f"{line.strip()}") + self.add_extra_error(f"unseparated keyword in line: {line.strip()}") failures += 1 error_log = self.add_error_log( error_log, f"unseparated keyword in line: {line.strip()}", count + 1, ) - output = f"Checked {count+1} lines, found {failures} failures." + output = f"Checked {count + 1} lines, found {failures} failures." return TestResult( checker_name="Unseparated Keywords", failure_count=failures, @@ -281,8 +279,7 @@ def go_to_other_than_9999(self, lines: List[str]) -> TestResult: clean_line = self.remove_quoted(line) clean_line = re.sub(r"!.*$", "", clean_line) - if match := re.search(r"\bGO\s*TO\s+(\d+)", clean_line, - re.IGNORECASE): + if match := re.search(r"\bGO\s*TO\s+(\d+)", clean_line, re.IGNORECASE): label = match.group(1) if label != "9999": self.add_extra_error(f"GO TO {label}") @@ -290,7 +287,7 @@ def go_to_other_than_9999(self, lines: List[str]) -> TestResult: error_log = self.add_error_log( error_log, f"GO TO {label}", count + 1 ) - output = f"Checked {count+1} lines, found {failures} failures." + output = f"Checked {count + 1} lines, found {failures} failures." return TestResult( checker_name="GO TO other than 9999", failure_count=failures, @@ -308,13 +305,11 @@ def write_using_default_format(self, lines: List[str]) -> TestResult: clean_line = self.remove_quoted(line) clean_line = re.sub(r"!.*$", "", clean_line) - if re.search(r"\bWRITE\s*\(\s*\*\s*,\s*\*\s*\)", clean_line, - re.IGNORECASE): + if re.search(r"\bWRITE\s*\(\s*\*\s*,\s*\*\s*\)", clean_line, re.IGNORECASE): self.add_extra_error("WRITE(*,*) found") failures += 1 - error_log = self.add_error_log(error_log, "WRITE(*,*) found", - count + 1) - output = f"Checked {count+1} lines, found {failures} failures." + error_log = self.add_error_log(error_log, "WRITE(*,*) found", count + 1) + output = f"Checked {count + 1} lines, found {failures} failures." return TestResult( checker_name="WRITE using default format", failure_count=failures, @@ -356,7 +351,7 @@ def lowercase_variable_names(self, lines: List[str]) -> TestResult: error_log, "UPPERCASE variable name", count + 1 ) - output = f"Checked {count+1} lines, found {failures} failures." + output = f"Checked {count + 1} lines, found {failures} failures." return TestResult( checker_name="Lowercase or CamelCase variable names only", failure_count=failures, @@ -381,7 +376,7 @@ def dimension_forbidden(self, lines: List[str]) -> TestResult: error_log, "DIMENSION attribute used", count + 1 ) - output = f"Checked {count+1} lines, found {failures} failures." + output = f"Checked {count + 1} lines, found {failures} failures." return TestResult( checker_name="Use of dimension attribute", failure_count=failures, @@ -407,7 +402,7 @@ def ampersand_continuation(self, lines: List[str]) -> TestResult: checker_name="Continuation lines shouldn't start with &", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -423,8 +418,7 @@ def forbidden_keywords(self, lines: List[str]) -> TestResult: clean_line = self.remove_quoted(line) clean_line = re.sub(r"!.*$", "", clean_line) - if re.search(r"\b(EQUIVALENCE|PAUSE)\b", clean_line, - re.IGNORECASE): + if re.search(r"\b(EQUIVALENCE|PAUSE)\b", clean_line, re.IGNORECASE): self.add_extra_error("forbidden keyword") failures += 1 error_log = self.add_error_log( @@ -435,7 +429,7 @@ def forbidden_keywords(self, lines: List[str]) -> TestResult: checker_name="Use of forbidden keywords EQUIVALENCE or PAUSE", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -459,11 +453,10 @@ def forbidden_operators(self, lines: List[str]) -> TestResult: ) return TestResult( - checker_name="Use of older form of relational operator " + - "(.GT. etc.)", + checker_name="Use of older form of relational operator " + "(.GT. etc.)", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -476,14 +469,13 @@ def line_over_80chars(self, lines: List[str]) -> TestResult: if len(line.rstrip()) > 80: self.add_extra_error("line too long") failures += 1 - error_log = self.add_error_log(error_log, "line too long", - count + 1) + error_log = self.add_error_log(error_log, "line too long", count + 1) return TestResult( checker_name="Line longer than 80 characters", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -504,7 +496,7 @@ def tab_detection(self, lines: List[str]) -> TestResult: checker_name="Line includes tab character", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -525,7 +517,7 @@ def printstatus_mod(self, lines: List[str]) -> TestResult: checker_name="Use of printstatus_mod instead of umPrintMgr", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -541,14 +533,13 @@ def printstar(self, lines: List[str]) -> TestResult: if re.search(r"\bPRINT\s*\*", clean_line, re.IGNORECASE): self.add_extra_error("PRINT * used") failures += 1 - error_log = self.add_error_log(error_log, "PRINT * used", - count + 1) + error_log = self.add_error_log(error_log, "PRINT * used", count + 1) return TestResult( checker_name="Use of PRINT rather than umMessage and umPrint", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -564,14 +555,13 @@ def write6(self, lines: List[str]) -> TestResult: if re.search(r"\bWRITE\s*\(\s*6\s*,", clean_line, re.IGNORECASE): self.add_extra_error("WRITE(6) used") failures += 1 - error_log = self.add_error_log(error_log, "WRITE(6) used", - count + 1) + error_log = self.add_error_log(error_log, "WRITE(6) used", count + 1) return TestResult( checker_name="Use of WRITE(6) rather than umMessage and umPrint", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -591,7 +581,7 @@ def um_fort_flush(self, lines: List[str]) -> TestResult: checker_name="Use of um_fort_flush rather than umPrintFlush", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -611,7 +601,7 @@ def svn_keyword_subst(self, lines: List[str]) -> TestResult: checker_name="Subversion keyword substitution", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -621,18 +611,16 @@ def omp_missing_dollar(self, lines: List[str]) -> TestResult: error_log = {} count = -1 for count, line in enumerate(lines): - if (re.search(r"!\s*OMP\b", line) and - not re.search(r"!\$OMP", line)): + if re.search(r"!\s*OMP\b", line) and not re.search(r"!\$OMP", line): self.add_extra_error("!OMP without $") failures += 1 - error_log = self.add_error_log(error_log, "!OMP without $", - count + 1) + error_log = self.add_error_log(error_log, "!OMP without $", count + 1) return TestResult( checker_name="!OMP without $", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -653,7 +641,7 @@ def cpp_ifdef(self, lines: List[str]) -> TestResult: checker_name="#ifdef/#ifndef used", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -674,15 +662,14 @@ def cpp_comment(self, lines: List[str]) -> TestResult: self.add_extra_error("Fortran comment in CPP directive") failures += 1 error_log = self.add_error_log( - error_log, "Fortran comment in CPP directive", - count + 1 + error_log, "Fortran comment in CPP directive", count + 1 ) return TestResult( checker_name="Fortran comment in CPP directive", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -700,15 +687,14 @@ def obsolescent_fortran_intrinsic(self, lines: List[str]) -> TestResult: self.add_extra_error(f"obsolescent intrinsic: {intrinsic}") failures += 1 error_log = self.add_error_log( - error_log, f"obsolescent intrinsic: {intrinsic}", - count + 1 + error_log, f"obsolescent intrinsic: {intrinsic}", count + 1 ) return TestResult( checker_name="obsolescent intrinsic", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -732,7 +718,7 @@ def exit_stmt_label(self, lines: List[str]) -> TestResult: checker_name="unlabelled EXIT statement", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -749,10 +735,8 @@ def intrinsic_modules(self, lines: List[str]) -> TestResult: for module in intrinsic_modules: if re.search( rf"\bUSE\s+(::)*\s*{module}\b", clean_line, re.IGNORECASE - ) and not re.search(r"\bINTRINSIC\b", clean_line, - re.IGNORECASE): - self.add_extra_error( - f"intrinsic module {module} without INTRINSIC") + ) and not re.search(r"\bINTRINSIC\b", clean_line, re.IGNORECASE): + self.add_extra_error(f"intrinsic module {module} without INTRINSIC") failures += 1 error_log = self.add_error_log( error_log, @@ -764,7 +748,7 @@ def intrinsic_modules(self, lines: List[str]) -> TestResult: checker_name="intrinsic modules", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -777,8 +761,7 @@ def read_unit_args(self, lines: List[str]) -> TestResult: clean_line = self.remove_quoted(line) clean_line = re.sub(r"!.*$", "", clean_line) - if match := re.search(r"\bREAD\s*\(\s*([^,)]+)", clean_line, - re.IGNORECASE): + if match := re.search(r"\bREAD\s*\(\s*([^,)]+)", clean_line, re.IGNORECASE): first_arg = match.group(1).strip() if not first_arg.upper().startswith("UNIT="): self.add_extra_error("READ without explicit UNIT=") @@ -791,7 +774,7 @@ def read_unit_args(self, lines: List[str]) -> TestResult: checker_name="read unit args", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -820,14 +803,13 @@ def retire_if_def(self, lines: List[str]) -> TestResult: self.add_extra_error(f"retired if-def: {match.group(1)}") failures += 1 error_log = self.add_error_log( - error_log, f"retired if-def: {match.group(1)}", - count + 1 + error_log, f"retired if-def: {match.group(1)}", count + 1 ) return TestResult( checker_name="retired if-def", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -863,8 +845,7 @@ def forbidden_stop(self, lines: List[str]) -> TestResult: clean_line = self.remove_quoted(line) clean_line = re.sub(r"!.*$", "", clean_line) - if re.search(r"\b(STOP|CALL\s+abort)\b", clean_line, - re.IGNORECASE): + if re.search(r"\b(STOP|CALL\s+abort)\b", clean_line, re.IGNORECASE): self.add_extra_error("STOP or CALL abort used") failures += 1 error_log = self.add_error_log( @@ -875,7 +856,7 @@ def forbidden_stop(self, lines: List[str]) -> TestResult: checker_name="forbidden stop", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -890,9 +871,15 @@ def intrinsic_as_variable(self, lines: List[str]) -> TestResult: # as I doubt this does anything near what that did... for count, line in enumerate(lines): clean_line = self.remove_quoted(line) - check = (r"^\s*(INTEGER|REAL|LOGICAL|CHARACTER)\s*.*:" + - r":\s*(SIN|COS|LOG|EXP|TAN)\b") - if re.search(check, clean_line, re.IGNORECASE, ): + check = ( + r"^\s*(INTEGER|REAL|LOGICAL|CHARACTER)\s*.*:" + + r":\s*(SIN|COS|LOG|EXP|TAN)\b" + ) + if re.search( + check, + clean_line, + re.IGNORECASE, + ): self.add_extra_error("intrinsic function used as variable") failures += 1 error_log = self.add_error_log( @@ -903,7 +890,7 @@ def intrinsic_as_variable(self, lines: List[str]) -> TestResult: checker_name="intrinsic as variable", failure_count=failures, passed=(failures == 0), - output=f"Checked {count+1} lines, found {failures} failures.", + output=f"Checked {count + 1} lines, found {failures} failures.", errors=error_log, ) @@ -923,8 +910,7 @@ def check_crown_copyright(self, lines: List[str]) -> TestResult: found_copyright = True if not found_copyright: - self.add_extra_error( - "missing copyright or crown copyright statement") + self.add_extra_error("missing copyright or crown copyright statement") error_log = self.add_error_log( error_log, "missing copyright or crown copyright statement", 0 ) @@ -949,16 +935,14 @@ def check_code_owner(self, lines: List[str]) -> TestResult: file_content = "\n".join(lines) found_code_owner = False error_log = {} - if ("Code Owner:" in file_content or - "code owner" in file_content.lower()): + if "Code Owner:" in file_content or "code owner" in file_content.lower(): # print(f"Debug: Found {file_content.lower()}") found_code_owner = True # This is often a warning rather than an error if not found_code_owner: self.add_extra_error("missing code owner comment") - error_log = self.add_error_log(error_log, - "missing code owner comment", 0) + error_log = self.add_error_log(error_log, "missing code owner comment", 0) return TestResult( checker_name="Code Owner Comment", failure_count=0 if found_code_owner else 1, @@ -1030,8 +1014,7 @@ def c_deprecated(self, lines: List[str]) -> int: for line in lines: for identifier in deprecated_c_identifiers: if re.search(rf"\b{identifier}\b", line): - self.add_extra_error( - f"deprecated C identifier: {identifier}") + self.add_extra_error(f"deprecated C identifier: {identifier}") failures += 1 return failures @@ -1058,8 +1041,7 @@ def c_openmp_define_no_combine(self, lines: List[str]) -> int: ) or re.search( r"&&.*_OPENMP.*&&.*SHUM_USE_C_OPENMP_VIA_THREAD_UTILS", line ): - self.add_extra_error( - "OpenMP defines combined with third macro") + self.add_extra_error("OpenMP defines combined with third macro") failures += 1 return failures diff --git a/script_umdp3_checker/umdp3_conformance.py b/script_umdp3_checker/umdp3_conformance.py index 58ef8273..ce768ebc 100644 --- a/script_umdp3_checker/umdp3_conformance.py +++ b/script_umdp3_checker/umdp3_conformance.py @@ -11,6 +11,7 @@ # Add custom modules to Python path if needed # Add the repository root to access fcm_bdiff and git_bdiff packages import sys + sys.path.insert(0, str(Path(__file__).parent.parent)) """ @@ -28,6 +29,7 @@ class CheckResult: tests passed, and a list of TestResult objects for each test run on that file. """ + """TODO : Might be better to store number of tests run, and number passed, rather than just number failed and whether all passed.""" file_path: str = "No file provided" @@ -191,11 +193,13 @@ def __init__( else [] ) if print_volume >= 5: - print(f"UMDP3_checker initialized :\n" - f" Name : {self.name}\n" - f" Has {len(self.check_functions)} check functions\n" - f" Using {len(self.file_extensions)} file extensions\n" - f" Gives {len(self.files_to_check)} files to check.") + print( + f"UMDP3_checker initialized :\n" + f" Name : {self.name}\n" + f" Has {len(self.check_functions)} check functions\n" + f" Using {len(self.file_extensions)} file extensions\n" + f" Gives {len(self.files_to_check)} files to check." + ) def get_name(self) -> str: return self.name @@ -207,8 +211,7 @@ def check(self, file_path: Path) -> CheckResult: for check_name, check_function in self.check_functions.items(): file_results.append(check_function(lines)) - tests_failed = sum([0 if result.passed else 1 for result in - file_results]) + tests_failed = sum([0 if result.passed else 1 for result in file_results]) return CheckResult( file_path=str(file_path), tests_failed=tests_failed, @@ -245,11 +248,13 @@ def __init__( else [] ) if print_volume >= 5: - print(f"ExternalChecker initialized :\n" - f" Name : {self.name}\n" - f" Has {len(self.check_commands)} check commands\n" - f" Using {len(self.file_extensions)} file extensions\n" - f" Gives {len(self.files_to_check)} files to check.") + print( + f"ExternalChecker initialized :\n" + f" Name : {self.name}\n" + f" Has {len(self.check_commands)} check commands\n" + f" Using {len(self.file_extensions)} file extensions\n" + f" Gives {len(self.files_to_check)} files to check." + ) def get_name(self) -> str: return self.name @@ -261,8 +266,7 @@ def check(self, file_path: Path) -> CheckResult: for test_name, command in self.check_commands.items(): try: cmd = command + [str(file_path)] - result = subprocess.run(cmd, capture_output=True, - text=True, timeout=60) + result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) except subprocess.TimeoutExpired: file_results.append( TestResult( @@ -362,8 +366,7 @@ def check_files(self) -> None: self.results = results return - def print_results(self, print_volume: int = 3, - quiet_pass: bool = True) -> bool: + def print_results(self, print_volume: int = 3, quiet_pass: bool = True) -> bool: """Print results and return True if all checks passed. ========================================================""" """ @@ -388,15 +391,16 @@ def print_results(self, print_volume: int = 3, continue if print_volume >= 3 and not test_result.passed: plural = "" if test_result.failure_count == 1 else "s" - print(f" {test_result.checker_name:60s} : Found " - + f"{test_result.failure_count:3} failure{plural}.") + print( + f" {test_result.checker_name:60s} : Found " + + f"{test_result.failure_count:3} failure{plural}." + ) if test_result.errors and print_volume >= 4: print(" " * 8 + line_2(82)) for count, (title, info) in enumerate( test_result.errors.items() ): - print(" " * 8 + - f"{count + 1:2} : {title} : {info}") + print(" " * 8 + f"{count + 1:2} : {title} : {info}") print(" " * 8 + line_2(82)) elif print_volume >= 3: print(f" {test_result.checker_name:60s} : ✓ PASS") @@ -429,27 +433,26 @@ def process_arguments(): "-p", "--path", type=str, default="./", help="path to repository" ) parser.add_argument( - "--max-workers", type=int, default=8, - help="Maximum number of parallel workers" + "--max-workers", type=int, default=8, help="Maximum number of parallel workers" ) parser.add_argument( - "--fullcheck", action="store_true", + "--fullcheck", + action="store_true", help="Instead of just checking changed files, check all files in " - "the repository" + "the repository", ) parser.add_argument( - "--printpass", action="store_true", + "--printpass", + action="store_true", help="Print details of passed checks as well as failed ones.\n" - "By default, only failed checks are printed in detail." + "By default, only failed checks are printed in detail.", ) group = parser.add_mutually_exclusive_group() group.add_argument( - "-v", "--verbose", action="count", default=0, - help="Increase output verbosity" + "-v", "--verbose", action="count", default=0, help="Increase output verbosity" ) group.add_argument( - "-q", "--quiet", action="count", default=0, - help="Decrease output verbosity" + "-q", "--quiet", action="count", default=0, help="Decrease output verbosity" ) # The following are not yet implemented, but may become useful # branch and base branch could be used to configure the CMS diff @@ -527,25 +530,21 @@ def which_cms_is_it(path: str, print_volume: int = 3) -> CMSSystem: def create_style_checkers( - file_types: List[str], changed_files: List[Path], - print_volume: int = 3 + file_types: List[str], changed_files: List[Path], print_volume: int = 3 ) -> List[StyleChecker]: """Create style checkers based on requested file types.""" dispatch_tables = CheckerDispatchTables() checkers = [] if "Fortran" in file_types: - file_extensions = {".f", ".for", ".f90", ".f95", - ".f03", ".f08", ".F90"} + file_extensions = {".f", ".for", ".f90", ".f95", ".f03", ".f08", ".F90"} fortran_diff_table = dispatch_tables.get_diff_dispatch_table_fortran() fortran_file_table = dispatch_tables.get_file_dispatch_table_fortran() generic_file_table = dispatch_tables.get_file_dispatch_table_all() if print_volume >= 3: print("Configuring Fortran checkers:") - combined_checkers = fortran_diff_table | fortran_file_table | \ - generic_file_table + combined_checkers = fortran_diff_table | fortran_file_table | generic_file_table fortran_file_checker = UMDP3_checker.from_full_list( - "Fortran Checker", file_extensions, - combined_checkers, changed_files + "Fortran Checker", file_extensions, combined_checkers, changed_files ) checkers.append(fortran_file_checker) if "Python" in file_types: @@ -556,11 +555,10 @@ def create_style_checkers( # "flake 8": ["flake8", "-q"], # "black": ["black", "--check"], # "pylint": ["pylint", "-E"], - "ruff": ["ruff", "check"], + "ruff": ["ruff", "check"], } python_file_checker = ExternalChecker( - "External Python Checkers", file_extensions, - python_checkers, changed_files + "External Python Checkers", file_extensions, python_checkers, changed_files ) checkers.append(python_file_checker) if "Generic" in file_types or file_types == []: @@ -568,16 +566,16 @@ def create_style_checkers( print("Configuring Generic File Checkers:") all_file_dispatch_table = dispatch_tables.get_file_dispatch_table_all() generic_checker = UMDP3_checker( - "Generic File Checker", set(), all_file_dispatch_table, - changed_files + "Generic File Checker", set(), all_file_dispatch_table, changed_files ) checkers.append(generic_checker) return checkers -def get_files_to_check(path: str, full_check: bool, - print_volume: int = 3) -> List[Path]: +def get_files_to_check( + path: str, full_check: bool, print_volume: int = 3 +) -> List[Path]: """ Docstring for get_files_to_check : A routine to get the list of files to check based on the CMS or the full check override. @@ -599,8 +597,10 @@ def get_files_to_check(path: str, full_check: bool, if print_volume >= 1: print("Full check override enabled.") if print_volume >= 3: - print(f" Found {len(all_files)} files to " - f"check in repository at path: {path}") + print( + f" Found {len(all_files)} files to " + f"check in repository at path: {path}" + ) return all_files else: # Configure CMS, and check we've been passed a branch if print_volume >= 1: @@ -627,8 +627,7 @@ def get_files_to_check(path: str, full_check: bool, Later, could add configuration files to specify which checkers to use for each file type.""" - active_checkers = create_style_checkers(args.file_types, - full_file_paths) + active_checkers = create_style_checkers(args.file_types, full_file_paths) # TODO : Could create a conformance checker for each # file type. @@ -643,18 +642,16 @@ def get_files_to_check(path: str, full_check: bool, if log_volume >= 3: print(line_1(81)) - print("## Results :" + " "*67 + "##") + print("## Results :" + " " * 67 + "##") print(line_1(81) + "\n") else: print("Results :") - all_passed = checker.print_results(print_volume=log_volume, - quiet_pass=quiet_pass) + all_passed = checker.print_results(print_volume=log_volume, quiet_pass=quiet_pass) if log_volume >= 4: print("\n" + line_1(81)) - print("## Summary :" + " "*67 + "##") + print("## Summary :" + " " * 67 + "##") print(line_1(81)) print(f"Total files checked: {len(checker.results)}") - print(f"Total files failed: " - f"{sum(1 for r in checker.results if not r.all_passed)}") + print(f"Total files failed: {sum(1 for r in checker.results if not r.all_passed)}") exit(0 if all_passed else 1) diff --git a/suite_report.py b/suite_report.py index fb03ebfa..67148cdf 100755 --- a/suite_report.py +++ b/suite_report.py @@ -6,22 +6,22 @@ # which you should have received as part of this distribution. # *****************************COPYRIGHT******************************* """ - ## NOTE ## +## NOTE ## - This module is one of several for which the main copy is in the - SimSys_Scripts repository. When making changes, please ensure the - changes are made in the github repository. +This module is one of several for which the main copy is in the +SimSys_Scripts repository. When making changes, please ensure the +changes are made in the github repository. - Script to process the results of a suite and write a summary to file. The - summary is in Trac wiki mark-up. Any projects that do not have a local - mirror repository are assumed not to be used at that site and are - excluded from the report. +Script to process the results of a suite and write a summary to file. The +summary is in Trac wiki mark-up. Any projects that do not have a local +mirror repository are assumed not to be used at that site and are +excluded from the report. - Owner: Scientific Software Development and Deployment team - (formerly : UM System Development Team) - Cylc Suite Syntax: shutdown handler = "suite_report.py" - Command Line syntax: - suite_report.py -S [-v] [-q] [-N] [-L ] +Owner: Scientific Software Development and Deployment team + (formerly : UM System Development Team) +Cylc Suite Syntax: shutdown handler = "suite_report.py" +Command Line syntax: + suite_report.py -S [-v] [-q] [-N] [-L ] """ @@ -40,7 +40,6 @@ import traceback from argparse import ArgumentParser, ArgumentTypeError, RawDescriptionHelpFormatter from collections import defaultdict -from optparse import OptionGroup, OptionParser from fcm_bdiff import get_branch_diff_filenames @@ -1154,9 +1153,9 @@ def get_code_owners(self, code_owners): owner += " (" + deputy + ")" needed_approvals[owner].add(section) except KeyError: - needed_approvals[ - "Unknown - ensure section " "is in CodeOwners.txt" - ].add(section) + needed_approvals["Unknown - ensure section is in CodeOwners.txt"].add( + section + ) return needed_approvals @@ -1416,7 +1415,7 @@ def forced_status_sort(item_tuple): lines.append(f" || {task} || {highlight_start}{state}{highlight_end} || ") if len(lines) == 1: lines.append( - " |||| This table is deliberately empty as all tasks " "are hidden || " + " |||| This table is deliberately empty as all tasks are hidden || " ) status_summary = ["'''Suite Output'''"] @@ -1822,7 +1821,7 @@ def get_altered_files_list(mirror_loc): except Exception as err: print(err) if attempt == 4: - print("Cant get list of alterered files - returning " "empty list.") + print("Cant get list of alterered files - returning empty list.") bdiff_files = [] break @@ -2120,7 +2119,7 @@ def parse_arguments(): dest="increase_verbosity", action="count", default=0, - help="increases Verbosity level. " f"(default: {DEFAULT_VERBOSITY})", + help=f"increases Verbosity level. (default: {DEFAULT_VERBOSITY})", ) verbose.add_argument( diff --git a/umdp3_fixer/ampersands.py b/umdp3_fixer/ampersands.py index a374c28a..9d8debcb 100755 --- a/umdp3_fixer/ampersands.py +++ b/umdp3_fixer/ampersands.py @@ -26,11 +26,23 @@ the ampersand shifting, and will be flagged, optionally with a message in stdout. """ + # import sys import re + # import traceback from optparse import OptionParser -from fstring_parse import * # noqa: F403 +from fstring_parse import ( + find_commented_char, + find_quoted_char, + is_continuation, + is_pp_continuation, + is_str_continuation, + ParsingError, + replace_characters, + DQUOTE, + SQUOTE, +) # Default column in which to place ampersands. DEFAULT_COL = 80 @@ -44,9 +56,8 @@ class CharError(ParsingError): def __init__(self, char, number): self.number = number self.char = char - self.msg = ( - "There are {0:d} unquoted, uncommented " - '"{1:s}" in this line.'.format(number, char) + self.msg = 'There are {0:d} unquoted, uncommented "{1:s}" in this line.'.format( + number, char ) pass @@ -252,12 +263,10 @@ def shift_ampersand( # If the line contains an unquoted or uncommented ampersand, need # to check if it is in the right place. if amp_loc != -1: - # If there is no comment, determin if it is a leading continuation # ampersand. If it is remove it; else shift the ampersand (and remove # any trailing whitespace). if comment_loc == -1: - # determin if we are a leading ampersand beforeline = workline[lp:amp_loc].rstrip() @@ -292,7 +301,6 @@ def shift_ampersand( # if the line is too long, in which case can try to take white # space to get it down to length. elif comment_loc > amp_loc: - # If the line is too long, first make sure there is only one # space between ampersand and comment, and get rid of any # trailing whitespace. @@ -322,7 +330,6 @@ def shift_ampersand( # Try cutting down the whitespace one step at a time until # there is only one space left. for i in range(nloop): - if workline[amp_location - 1] == " ": # If there is still whitespace that can be removed, # remove it and update the ampersand location. @@ -373,7 +380,7 @@ def apply_ampersand_shift( if debug: print_message( "PARSING ERROR", - "{0:s} Ampersand shifting has not been " "applied".format(e.msg), + "{0:s} Ampersand shifting has not been applied".format(e.msg), iline + 1, line=line, fname=fname, diff --git a/umdp3_fixer/fstring_parse.py b/umdp3_fixer/fstring_parse.py index 6b3aa10c..b557cf92 100644 --- a/umdp3_fixer/fstring_parse.py +++ b/umdp3_fixer/fstring_parse.py @@ -13,6 +13,7 @@ This module contains various functions for parsing and manipulating quoted strings in Fortran """ + import re @@ -309,7 +310,6 @@ def simplify_line(lines): # Pull any continuation lines into this line while repeat_simplify: - while is_pp_continuation(line): iline += 1 line = "".join([re.sub(r"\\(\s*)$", r" \1", line), lines[iline]]) diff --git a/umdp3_fixer/indentation.py b/umdp3_fixer/indentation.py index 5ef6085a..8b3747db 100755 --- a/umdp3_fixer/indentation.py +++ b/umdp3_fixer/indentation.py @@ -15,9 +15,15 @@ Module containing various functions used to apply UMDP3 style indentation to Fortran source code """ + import re import sys -from fstring_parse import * # noqa: F403 +from fstring_parse import ( + is_continuation, + is_pp_continuation, + is_str_continuation, + simplify_line, +) # Number of spaces of indent to apply per indentation level INDENT = 2 @@ -120,7 +126,6 @@ def apply_indentation(lines, debug=False): rel_indent_pp_level_max = {} for iline, line in enumerate(lines): - if debug: print('\n{0:d}: "{1:s}"'.format(iline, line)) diff --git a/umdp3_fixer/styling.py b/umdp3_fixer/styling.py index 21d50066..98e76fdb 100755 --- a/umdp3_fixer/styling.py +++ b/umdp3_fixer/styling.py @@ -14,9 +14,22 @@ This module contains various functions for applying UMDP3 styling to Fortran source code """ + import re import sys -from fstring_parse import * # noqa: F403 +from fstring_parse import ( + blank_fcomments, + blank_fstring, + clean_str_continuation, + is_continuation, + is_pp_continuation, + is_str_continuation, + ParsingError, + partial_blank_fstring, + simplify_line, + DQUOTE, + SQUOTE, +) CODE_REPLACEMENTS = [ # Replace Fortran 77 style conditional keywords @@ -839,7 +852,7 @@ def replace_patterns(line, str_continuation): blank_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:" "".format(e.msg)) + print("{0:s} Line simplification has failed for:".format(e.msg)) print(line) exit(1) @@ -905,7 +918,7 @@ def replace_comment_patterns(line, str_continuation): match_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:" "".format(e.msg)) + print("{0:s} Line simplification has failed for:".format(e.msg)) print(line) exit(1) diff --git a/umdp3_fixer/umdp3_fixer.py b/umdp3_fixer/umdp3_fixer.py index 6b42294e..9b95dcb0 100755 --- a/umdp3_fixer/umdp3_fixer.py +++ b/umdp3_fixer/umdp3_fixer.py @@ -31,6 +31,7 @@ branch-diff version to apply to them. C files must have the .c or .h extension. """ + import os import re import sys @@ -206,7 +207,6 @@ def main(): ): if input_file.split(".")[-1] == "h": if re.search(r".*\/include\/other\/.*", input_file) is not None: - print( "Input file {0:s} not a " "Fortran include file," @@ -215,8 +215,9 @@ def main(): continue else: print( - "Input file {0:s} not a " - "Fortran file, skipping".format(input_file) + "Input file {0:s} not a Fortran file, skipping".format( + input_file + ) ) continue @@ -241,8 +242,7 @@ def main(): if len(amp_not_parsed) > 0: print( - "Ampersand Alignment Failed for:" - " {0:s}".format(input_file) + "Ampersand Alignment Failed for: {0:s}".format(input_file) ) print("failed on lines:\n") for i in amp_not_parsed: @@ -256,9 +256,7 @@ def main(): white_lines = apply_whitespace_fixes(amp_lines) if white_lines is None: - print( - "Whitespace Fixes Failed for:" " {0:s}".format(input_file) - ) + print("Whitespace Fixes Failed for: {0:s}".format(input_file)) failed = True styled_lines = None @@ -289,8 +287,7 @@ def main(): if len(amp_not_parsed) > 0: print( - "Ampersand Alignment Failed for:" - " {0:s}".format(input_file) + "Ampersand Alignment Failed for: {0:s}".format(input_file) ) print("failed on lines:\n") for i in amp_not_parsed: @@ -424,9 +421,7 @@ def main(): ) else: print("\nSkipping C files:") - print( - "\nThis utilises clang-format," " which is not available on this system" - ) + print("\nThis utilises clang-format, which is not available on this system") if failed: sys.exit(1) diff --git a/umdp3_fixer/whitespace.py b/umdp3_fixer/whitespace.py index 50d3e632..1b3cd6ab 100755 --- a/umdp3_fixer/whitespace.py +++ b/umdp3_fixer/whitespace.py @@ -9,7 +9,18 @@ # import sys import re import argparse -from fstring_parse import * # noqa: F403 +from fstring_parse import ( + blank_fcomments, + blank_fstring, + clean_str_continuation, + is_continuation, + is_pp_continuation, + is_str_continuation, + ParsingError, + partial_blank_fstring, + DQUOTE, + SQUOTE, +) # These 4 are defined globally for visibility. They are actually only used in # main or a single subroutine but actual text is not yet defined in stone... @@ -58,7 +69,6 @@ def strip_trailing_space(line): def keyword_split(line, str_continuation): - if len(line.strip()) == 0: return line @@ -118,7 +128,7 @@ def keyword_split(line, str_continuation): blank_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed " "for:".format(e.msg)) + print("{0:s} Line simplification has failed for:".format(e.msg)) print(line) exit(1) @@ -225,7 +235,7 @@ def main(): parser.add_argument( "-k", "--keywordsplit", - help="Split potentially conjoined keywords" " such as INOUT", + help="Split potentially conjoined keywords such as INOUT", action="store_true", ) parser.add_argument(