Merge pull request #121 from LUMC/release_1.5.0

Release 1.5.0

Author: rhpvorderman

.github/workflows/ci.yml

@@ -0,0 +1,91 @@
+name: Continous integration
+
+on:
+  pull_request:
+  push:
+    branches:
+      - develop
+      - master
+
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - 3.6
+    steps:
+      - uses: actions/checkout@v2.3.4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+      - name: Install tox
+        run: pip install tox
+      - name: Lint
+        run: tox -e lint
+  docs:
+    needs: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2.3.4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+      - name: Install tox
+        run: pip install tox
+      - name: Build docs
+        run: tox -e docs
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - 3.6
+          - 3.7
+          - 3.8
+          - 3.9
+    needs: lint
+    steps:
+      - uses: actions/checkout@v2.3.4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+      - name: Install tox
+        run: pip install tox
+      - name: Run tests
+        run: tox -e py3
+      - name: Upload coverage report
+        if: ${{ matrix.python-version == 3.6 }} # Only upload coverage once
+        uses: codecov/codecov-action@v1
+
+  test-functional:
+    runs-on: ubuntu-latest
+    needs: test
+    strategy:
+      matrix:
+        python-version: [3.7]
+        test-program: [cromwell, snakemake, miniwdl]
+    steps:
+      - uses: actions/checkout@v2.3.4
+
+      # Setup python program requirements
+      - name: Set up Python ${{ matrix.python-version }}
+        if: ${{ matrix.test-program != 'cromwell' }}
+        uses: actions/setup-python@v2
+      - name: Install tox
+        if: ${{ matrix.test-program != 'cromwell' }}
+        run: pip install tox
+
+      # Setup cromwell requirements
+      - name: Install conda
+        if: ${{ matrix.test-program == 'cromwell' }}
+        uses: conda-incubator/setup-miniconda@v2.0.1 # https://github.com/conda-incubator/setup-miniconda.
+        with:
+          channels: conda-forge,defaults
+      - name: Install cromwell and tox
+        shell: bash -l {0}  # Needed for conda
+        if: ${{ matrix.test-program == 'cromwell' }}
+        run: conda install cromwell tox
+
+      # Test
+      - name: Test
+        shell: bash -l {0}  # Needed for conda
+        run: tox -e ${{ matrix.test-program }}

.travis.yml

@@ -7,6 +7,17 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.5.0
+---------------------------
++ Add support for python 3.9
++ Update the print statement for starting jobs to be more structured. This will
+  make the output easier to to read and use, since different fields (stdout,
+  stderr, command, etc) are all on their own line.
++ Do not crash when directories can not be removed due to permission errors.
+  Instead display a message to notify the users which directories could not be
+  removed. These issues occurred sometimes when tests involving docker were
+  run.
+
 version 1.4.0
 ---------------------------
 + Usage of the ``name`` keyword argument in workflow marks is now deprecated.

HISTORY.rst

@@ -41,8 +41,8 @@ For our complete documentation checkout our
 
 Installation
 ============
-Pytest-workflow requires Python 3.6 or higher. It is tested on Python 3.6, 3.7
-and 3.8. Python 2 is not supported.
+Pytest-workflow requires Python 3.6 or higher. It is tested on Python 3.6, 3.7,
+3.8 and 3.9. Python 2 is not supported.
 
 - Make sure your virtual environment is activated.
 - Install using pip ``pip install pytest-workflow``

README.rst

@@ -88,7 +88,7 @@ inside the test output directory. If you have a workflow with the output:
             Array[File] stats = moo_task.stats_files
         }
 
-Inside the ``output_links`` directory the directories ``moo_file`` and
+Inside the ``out`` directory the directories ``moo_file`` and
 ``stats`` will be created. Inside these directories will be the produced files.
 
 The following yaml file tests a WDL pipeline run with miniwdl.

docs/examples.rst

@@ -2,7 +2,7 @@
 Installation
 ============
 
-Pytest-workflow is tested on python 3.6, 3.7 and 3.8. Python 2 is not
+Pytest-workflow is tested on python 3.6, 3.7, 3.8 and 3.9. Python 2 is not
 supported.
 
 In a virtual environment

docs/installation.rst

@@ -14,19 +14,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with pytest-workflow.  If not, see <https://www.gnu.org/licenses/
 
-from setuptools import find_packages, setup
+from pathlib import Path
 
-with open("README.rst", "r") as readme_file:
-    LONG_DESCRIPTION = readme_file.read()
+from setuptools import find_packages, setup
 
 setup(
     name="pytest-workflow",
-    version="1.4.0",
+    version="1.5.0",
     description="A pytest plugin for configuring workflow/pipeline tests "
                 "using YAML files",
     author="Leiden University Medical Center",
     author_email="sasc@lumc.nl",  # A placeholder for now
-    long_description=LONG_DESCRIPTION,
+    long_description=Path("README.rst").read_text(),
     long_description_content_type="text/x-rst",
     license="AGPL-3.0-or-later",
     keywords="pytest workflow pipeline yaml yml wdl cromwell snakemake",
@@ -43,6 +42,7 @@
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: "
         "GNU Affero General Public License v3 or later (AGPLv3+)",

mypy.ini

@@ -15,13 +15,14 @@
 # along with pytest-workflow.  If not, see <https://www.gnu.org/licenses/
 
 """All tests for workflow files"""
-import hashlib
+
 from pathlib import Path
 
 import pytest
 
 from .content_tests import ContentTestCollector
 from .schema import FileTest
+from .util import file_md5sum
 from .workflow import Workflow
 
 
@@ -141,18 +142,3 @@ def repr_failure(self, excinfo, style=None):
             f"Observed md5sum '{self.observed_md5sum}' not equal to expected "
             f"md5sum '{self.expected_md5sum}' for file '{self.filepath}'"
         )
-
-
-# block_size 64k with python is a few percent faster than linux native md5sum.
-def file_md5sum(filepath: Path, block_size=64 * 1024) -> str:
-    """
-    Generates a md5sum for a file. Reads file in blocks to save memory.
-    :param filepath: a pathlib. Path to the file
-    :param block_size: Block size in bytes
-    :return: a md5sum as hexadecimal string.
-    """
-    hasher = hashlib.md5()  # nosec: only used for file integrity
-    with filepath.open('rb') as file_handler:  # Read the file in bytes
-        for block in iter(lambda: file_handler.read(block_size), b''):
-            hasher.update(block)
-    return hasher.hexdigest()