Merge remote-tracking branch 'origin/main' into skip_internal_imports

Author: mr-c

.github/workflows/ci-tests.yml

@@ -21,6 +21,13 @@ jobs:
         py-ver-major: [3]
         py-ver-minor: [6, 7, 8, 9, 10, 11]
         step: [lint, unit, mypy]
+        exclude:
+          - py-ver-major: 3
+            py-ver-minor: 6
+            step: mypy
+          - py-ver-major: 3
+            py-ver-minor: 6
+            step: lint
 
     env:
       py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}

Makefile

@@ -28,7 +28,7 @@ EXTRAS=
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard cwlupgrader/**.py tests/*.py) setup.py
 DEVPKGS=diff_cover black pylint pep257 pydocstyle flake8 tox tox-pyenv \
-	isort wheel autoflake flake8-bugbear pyupgrade bandit \
+	isort wheel autoflake flake8-bugbear pyupgrade bandit build\
 	-rtest-requirements.txt -rmypy-requirements.txt
 DEBDEVPKGS=pylint python3-coverage sloccount \
 	   python3-flake8 shellcheck
@@ -57,13 +57,14 @@ install: FORCE
 
 ## dev                    : install the cwlupgrader package in dev mode
 dev: install-dep
+	pip install -U pip setuptools wheel
 	pip install -e .$(EXTRAS)
 
 ## dist                   : create a module package for distribution
 dist: dist/${MODULE}-$(VERSION).tar.gz
 
 dist/${MODULE}-$(VERSION).tar.gz: $(SOURCES)
-	python setup.py sdist bdist_wheel
+	python -m build
 
 ## clean                  : clean up all temporary / machine-generated files
 clean: FORCE
@@ -74,7 +75,7 @@ clean: FORCE
 
 # Linting and code style related targets
 ## sort_import            : sorting imports using isort: https://github.com/timothycrosley/isort
-sort_imports: $(PYSOURCES)
+sort_imports: $(PYSOURCES) mypy-stubs
 	isort $^
 
 remove_unused_imports: $(PYSOURCES)
@@ -98,10 +99,10 @@ codespell:
 
 ## format                 : check/fix all code indentation and formatting (runs black)
 format:
-	black setup.py cwlupgrader tests
+	black setup.py cwlupgrader tests mypy-stubs
 
 format-check:
-	black --diff --check cwlupgrader setup.py
+	black --diff --check cwlupgrader setup.py mypy-stubs
 
 ## pylint                 : run static code analysis on Python code
 pylint: $(PYSOURCES)
@@ -136,11 +137,11 @@ diff-cover: coverage.xml
 	diff-cover --compare-branch=main $^
 
 diff-cover.html: coverage.xml
-	diff-cover --compare-branch main $^ --html-report $@
+	diff-cover --compare-branch=main $^ --html-report $@
 
 ## test                   : run the cwlupgrader test suite
 test: $(PYSOURCES)
-	python -m pytest -rs
+	python -m pytest -rs ${PYTEST_EXTRA}
 
 ## testcov                : run the cwlupgrader test suite and collect coverage
 testcov: $(PYSOURCES)
@@ -167,6 +168,12 @@ mypy: $(PYSOURCES)
 	fi  # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
 	MYPYPATH=$$MYPYPATH:mypy-stubs mypy $^
 
+mypy_3.6: $(filter-out setup.py,$(PYSOURCES))
+	MYPYPATH=$$MYPYPATH:mypy-stubs mypy --python-version 3.6 $^
+
+shellcheck: FORCE
+	shellcheck testing.sh release-test.sh
+
 pyupgrade: $(PYSOURCES)
 	pyupgrade --exit-zero-even-if-changed --py36-plus $^
 
@@ -181,8 +188,8 @@ release: release-test
 		twine upload testenv2/src/${MODULE}/dist/* && \
 		git tag v${VERSION} && git push --tags
 
-flake8: $(PYSOURCES)
-	flake8 $^
+flake8: FORCE
+	flake8 $(PYSOURCES)
 
 FORCE:
 

cwlupgrader/main.py

@@ -24,7 +24,7 @@
 
 yaml = ruamel.yaml.main.YAML(typ="rt")
 yaml.allow_duplicate_keys = True
-yaml.preserve_quotes = True  # type: ignore
+yaml.preserve_quotes = True
 yaml.default_flow_style = False
 
 

mypy-requirements.txt

@@ -1,2 +1,2 @@
-mypy==0.971
+mypy==1.3.0
 types-setuptools

release-test.sh

@@ -35,7 +35,7 @@ then
 	rm -f testenv1/lib/python-wheels/setuptools* \
 		&& pip install --force-reinstall -U pip==${pipver} \
 		&& pip install setuptools==${setuptoolsver} wheel
-	pip install -rtest-requirements.txt ".${extras}"
+	pip install -rtest-requirements.txt ".${extras}" build
 	make test
 	pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 	mkdir testenv1/not-${module}
@@ -63,7 +63,7 @@ rm -f lib/python-wheels/setuptools* \
 # The following can fail if you haven't pushed your commits to ${repo}
 pip install -e "git+${repo}@${HEAD}#egg=${package}${extras}"
 pushd src/${package}
-pip install -rtest-requirements.txt
+pip install -rtest-requirements.txt build
 make dist
 make test
 cp dist/${package}*tar.gz ../../../testenv3/
@@ -84,7 +84,7 @@ rm -f lib/python-wheels/setuptools* \
 	&& pip install --force-reinstall -U pip==${pipver} \
         && pip install setuptools==${setuptoolsver} wheel
 package_tar=$(find . -name "${package}*tar.gz")
-pip install "-r${DIR}/test-requirements.txt"
+pip install "-r${DIR}/test-requirements.txt" build
 pip install "${package_tar}${extras}"
 mkdir out
 tar --extract --directory=out -z -f ${package}*.tar.gz

setup.cfg

@@ -14,4 +14,4 @@ use_parentheses = True
 line_length = 88
 
 [metadata]
-license_file = LICENSE.txt
+license_files = LICENSE.txt

testing.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
 git clone https://github.com/common-workflow-language/common-workflow-language.git cwl-v1.0
-cd cwl-v1.0/v1.0
+cd cwl-v1.0/v1.0 || exit
 cp -r v1.0 v1.1
 rm v1.1/*.cwl
-for cwl in v1.0/*.cwl; do echo ${cwl}; cwl-upgrader ${cwl} > v1.1/${cwl##v1.0/}; done
+for cwl in v1.0/*.cwl; do echo "${cwl}"; cwl-upgrader "${cwl}" > v1.1/"${cwl##v1.0/}"; done
 cp conformance_test_v1.0.yaml conformance_test_v1.0_to_v1_1.yaml
 sed -i 's=v1.0/=v1.1/=g' conformance_test_v1.0_to_v1_1.yaml
-cwltest --test conformance_test_v1.0_to_v1_1.yaml --tool cwltool -j$(nproc)
+cwltest --test conformance_test_v1.0_to_v1_1.yaml --tool cwltool "-j$(nproc)"
 if 
-	find v1.1/ -type f | xargs grep cwlVersion | grep -v basename-fields-job.yml | grep v1.0
+	find v1.1/ -type f -print0 | xargs -0 grep cwlVersion | grep -v basename-fields-job.yml | grep v1.0
 then false
 else true
 fi

tox.ini

@@ -3,7 +3,7 @@ envlist =
   py{36,37,38,39,310,311}-lint,
   py{36,37,38,39,310,311}-unit,
 
-  py{36,37,38,39,310,311}-mypy,
+  py{37,38,39,310,311}-mypy,
   py39-lintreadme,
   py39-pydocstyle
 
@@ -25,7 +25,7 @@ python =
 description =
   py{36,37,38,39,310,311}-unit: Run the unit tests
   py{36,37,38,39,310,311}-lint: Lint the Python code
-  py{36,37,38,39,310,311}-mypy: Check for type safety
+  py{37,38,39,310,311}-mypy: Check for type safety
   py311-pydocstyle: docstring style checker
   py311-lintreadme: Lint the README.rst->.md conversion
 
@@ -37,22 +37,23 @@ deps =
   py{36,37,38,39,310,311}-{unit,mypy}: -rtest-requirements.txt
   py{36,37,38,39,310,311}-lint: flake8-bugbear
   py{36,37,38,39,310,311}-lint: black
-  py{36,37,38,39,310,311}-mypy: -rmypy-requirements.txt
+  py{37,38,39,310,311}-mypy: -rmypy-requirements.txt
 
 setenv =
   py{36,37,38,39,310,311}-unit: LC_ALL = C.UTF-8
 
 commands =
   py{36,37,38,39,310,311}-unit: python -m pip install -U pip setuptools wheel
-  py{36,37,38,39,310,311}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs}
+  py{36,37,38,39,310,311}-unit: make coverage-report coverage.xml PYTEST_EXTRA="{posargs}"
   py{36,37,38,39,310,311}-lint: make flake8
   py{36,37,38,39,310,311}-lint: make format-check
-  py{36,37,38,39,310,311}-mypy: make mypy
+  py{37,38,39,310,311}-mypy: make mypy
 
 allowlist_externals =
   py{36,37,38,39,310,311}-lint: flake8
   py{36,37,38,39,310,311}-lint: black
   py{36,37,38,39,310,311}-{mypy,shellcheck,lint,unit}: make
+  py37-mypy: make mypy_3.6
 
 skip_install =
   py{36,37,38,39,310,311}-lint: true