Merge remote-tracking branch 'origin/main' into skip_internal_imports

Author: mr-c

.github/dependabot.yml

@@ -1,7 +1,11 @@
 version: 2
 updates:
-- package-ecosystem: pip
-  directory: "/"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/ci-tests.yml

@@ -1,4 +1,4 @@
-name: Continous integration tests
+name: Continuous integration tests
 
 on:
   push:
@@ -15,31 +15,30 @@ jobs:
   tox:
     name: CI tests via Tox
 
-    runs-on: ubuntu-20.04
-
+    runs-on: ubuntu-20.04  # 22.04 doesn't support Python 3.6
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [6, 7, 8, 9, 10]
+        py-ver-minor: [6, 7, 8, 9, 10, 11]
         step: [lint, unit, mypy]
 
     env:
       py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}
       TOXENV: ${{ format('py{0}{1}-{2}', matrix.py-ver-major, matrix.py-ver-minor, matrix.step) }}
 
     steps:
-      - uses: actions/checkout@v2
-
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ env.py-semver }}
-
-      - name: Cache for pip
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.step }}-${{ hashFiles('requirements.txt', 'tox.ini') }}
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            test-requirements.txt
+            mypy-requirements.txt
 
       - name: Upgrade setuptools and install tox
         run: |
@@ -48,7 +47,7 @@ jobs:
 
       - name: MyPy cache
         if: ${{ matrix.step == 'mypy' }}
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: .mypy_cache/${{ env.py-semver }}
           key: mypy-${{ env.py-semver }}
@@ -58,9 +57,10 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.step == 'unit' }}
-        uses: codecov/codecov-action@v2.1.0
+        uses: codecov/codecov-action@v3
         with:
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   tox-style:
     name: CI linters via Tox
@@ -72,24 +72,19 @@ jobs:
         step: [lintreadme, pydocstyle]
 
     env:
-      py-semver: "3.10"
-      TOXENV: ${{ format('py310-{0}', matrix.step) }}
+      py-semver: "3.11"
+      TOXENV: ${{ format('py311-{0}', matrix.step) }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ env.py-semver }}
-
-      - name: Cache for pip
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.step }}-${{ hashFiles('requirements.txt') }}
+          cache: pip
 
       - name: Upgrade setuptools and install tox
         run: |
@@ -105,23 +100,20 @@ jobs:
 
   release_test:
     name: cwl-utils release test
-
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.10"  # quoted, otherwise that turns into the number 3.1
-
-      - name: Cache for pip
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-release-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
-
+          python-version: "3.11"  # quoted, otherwise that turns into the number 3.1
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            test-requirements.txt
+            mypy-requirements.txt
       - name: Install packages
         run: |
           pip install -U pip setuptools wheel

.github/workflows/codeql-analysis.yml

@@ -2,18 +2,27 @@ name: "Code scanning - action"
 
 on:
   push:
+    branches: [main]
   pull_request:
+    branches: [main]
   schedule:
     - cron: '0 1 * * 4'
 
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   CodeQL-Build:
 
     runs-on: ubuntu-latest
 
+    permissions:
+      security-events: write
+
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         # We must fetch at least the immediate parents so that if this is
         # a pull request then we can checkout the head.
@@ -26,9 +35,9 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: python
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.gitignore

@@ -28,9 +28,8 @@ venv3/
 # pycharm
 .idea/
 
-# typshed repo
-typeshed/2and3/schema_salad
-typeshed/2and3/ruamel/yaml
+# local type stubs
+mypy-stubs/ruamel/yaml
 
 
 #mypy

Makefile

@@ -32,70 +32,78 @@ DEVPKGS=diff_cover black pylint pep257 pydocstyle flake8 tox tox-pyenv \
 	-rtest-requirements.txt -rmypy-requirements.txt
 DEBDEVPKGS=pylint python3-coverage sloccount \
 	   python3-flake8 shellcheck
-VERSION=1.2.2  # please also update setup.py
+VERSION=1.2.7  # please also update setup.py
 
-## all         : default task
+## all                    : default task (install cwl-upgrader in dev mode)
 all: dev
 
-## help        : print this help message and exit
+## help                   : print this help message and exit
 help: Makefile
 	@sed -n 's/^##//p' $<
 
-## install-dep : install most of the development dependencies via pip
+## cleanup                : shortcut for "make sort_imports format flake8 diff_pydocstyle_report"
+cleanup: sort_imports format flake8 diff_pydocstyle_report
+
+## install-dep            : install most of the development dependencies via pip
 install-dep: install-dependencies
 
 install-dependencies: FORCE
 	pip install --upgrade $(DEVPKGS)
 	pip install -r requirements.txt -r mypy-requirements.txt
 
-## install     : install the ${MODULE} module and script(s)
+## install                : install the cwlupgrader package and scripts
 install: FORCE
 	pip install .$(EXTRAS)
 
-## dev     : install the ${MODULE} module in dev mode
+## dev                    : install the cwlupgrader package in dev mode
 dev: install-dep
 	pip install -e .$(EXTRAS)
 
-## dist        : create a module package for distribution
+## dist                   : create a module package for distribution
 dist: dist/${MODULE}-$(VERSION).tar.gz
 
 dist/${MODULE}-$(VERSION).tar.gz: $(SOURCES)
 	python setup.py sdist bdist_wheel
 
-## clean       : clean up all temporary / machine-generated files
+## clean                  : clean up all temporary / machine-generated files
 clean: FORCE
 	rm -f ${MODILE}/*.pyc tests/*.pyc
 	python setup.py clean --all || true
 	rm -Rf .coverage
 	rm -f diff-cover.html
 
 # Linting and code style related targets
-## sorting imports using isort: https://github.com/timothycrosley/isort
+## sort_import            : sorting imports using isort: https://github.com/timothycrosley/isort
 sort_imports: $(PYSOURCES)
-	isort $^ typeshed
+	isort $^
 
 remove_unused_imports: $(PYSOURCES)
 	autoflake --in-place --remove-all-unused-imports $^
 
 pep257: pydocstyle
-## pydocstyle      : check Python code style
+## pydocstyle             : check Python docstring style
 pydocstyle: $(PYSOURCES)
 	pydocstyle --add-ignore=D100,D101,D102,D103 $^ || true
 
 pydocstyle_report.txt: $(PYSOURCES)
 	pydocstyle setup.py $^ > $@ 2>&1 || true
 
+## diff_pydocstyle_report : check Python docstring style for changed files only
 diff_pydocstyle_report: pydocstyle_report.txt
-	diff-quality --compare-branch=main --violations=pycodestyle --fail-under=100 $^
+	diff-quality --compare-branch=main --violations=pydocstyle --fail-under=100 $^
+
+## codespell              : check for common misspellings
+codespell:
+	codespell -w $(shell git ls-files | grep -v mypy-stubs | grep -v gitignore | grep -v EDAM.owl | grep -v pre.yml | grep -v test_schema)
 
-## format      : check/fix all code indentation and formatting (runs black)
+## format                 : check/fix all code indentation and formatting (runs black)
 format:
 	black setup.py cwlupgrader tests
 
 format-check:
-	black --diff --check cwlupgrader setup.py typeshed
+	black --diff --check cwlupgrader setup.py
 
-## pylint      : run static code analysis on Python code
+## pylint                 : run static code analysis on Python code
 pylint: $(PYSOURCES)
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
                 $^ -j0|| true
@@ -105,7 +113,7 @@ pylint_report.txt: $(PYSOURCES)
 		$^ -j0> $@ || true
 
 diff_pylint_report: pylint_report.txt
-	diff-quality --compare-branch main --violations=pylint pylint_report.txt
+	diff-quality --compare-branch=main --violations=pylint pylint_report.txt
 
 .coverage: testcov
 
@@ -125,23 +133,23 @@ coverage-report: .coverage
 	coverage report
 
 diff-cover: coverage.xml
-	diff-cover --compare-branch main $^
+	diff-cover --compare-branch=main $^
 
 diff-cover.html: coverage.xml
 	diff-cover --compare-branch main $^ --html-report $@
 
-## test        : run the ${MODULE} test suite
+## test                   : run the cwlupgrader test suite
 test: $(PYSOURCES)
-	python setup.py test
+	python -m pytest -rs
 
-## testcov     : run the ${MODULE} test suite and collect coverage
+## testcov                : run the cwlupgrader test suite and collect coverage
 testcov: $(PYSOURCES)
 	python setup.py test --addopts "--cov" ${PYTEST_EXTRA}
 
 sloccount.sc: $(PYSOURCES) Makefile
 	sloccount --duplicates --wide --details $^ > $@
 
-## sloccount   : count lines of code
+## sloccount              : count lines of code
 sloccount: $(PYSOURCES) Makefile
 	sloccount $^
 
@@ -153,17 +161,17 @@ mypy3: mypy
 mypy: $(PYSOURCES)
 	if ! test -f $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \
 	then \
-		rm -Rf typeshed/ruamel/yaml ; \
+		rm -Rf mypy-stubs/ruamel/yaml ; \
 		ln -s $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
-			typeshed/ruamel/ ; \
+			mypy-stubs/ruamel/ ; \
 	fi  # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
-	MYPYPATH=$$MYPYPATH:typeshed mypy $^
+	MYPYPATH=$$MYPYPATH:mypy-stubs mypy $^
 
 pyupgrade: $(PYSOURCES)
 	pyupgrade --exit-zero-even-if-changed --py36-plus $^
 
 release-test: FORCE
-	git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false )
+	git diff-index --quiet HEAD -- || ( echo You have uncommitted changes, please commit them and try again; false )
 	./release-test.sh
 
 release: release-test