better handle multi-version upgrades

Author: mr-c

cwlupgrader/main.py

@@ -199,7 +199,12 @@ def write_cwl_document(document: Any, name: str, dirname: str) -> None:
     path = Path(dirname) / name
     with open(path, "w") as handle:
         if "cwlVersion" in document:
-            handle.write("#!/usr/bin/env cwl-runner\n")
+            if not (
+                document.ca
+                and document.ca.comment
+                and "cwl-runner" in document.ca.comment[1][0].value
+            ):
+                handle.write("#!/usr/bin/env cwl-runner\n")
         yaml.dump(document, stream=handle)
     if "cwlVersion" in document:
         path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
@@ -249,12 +254,13 @@ def v1_0_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
 def v1_0_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
     """CWL v1.0.x to v1.2 transformation."""
     document = v1_0_to_v1_1(document, outdir)
-    document["cwlVersion"] = "v1.2"
+    document = v1_1_to_v1_2(document, outdir)
     return document
 
 
 def v1_1_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
     """CWL v1.1 to v1.2 transformation."""
+    document = _v1_1_to_v1_2(document, outdir)
     document["cwlVersion"] = "v1.2"
     return document
 
@@ -350,6 +356,10 @@ def _v1_0_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
                             _v1_0_to_v1_1(process, outdir)
                             if "cwlVersion" in process:
                                 del process["cwlVersion"]
+                        elif isinstance(entry["run"], str) and "#" not in entry["run"]:
+                            path = Path(document.lc.filename).parent / entry["run"]
+                            process = v1_0_to_v1_1(load_cwl_document(str(path)), outdir)
+                            write_cwl_document(process, path.name, outdir)
             elif isinstance(steps, MutableMapping):
                 for step_name in steps:
                     with SourceLine(steps, step_name, Exception):
@@ -413,10 +423,62 @@ def _v1_0_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
 
 
 def _v1_0_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
-    return _v1_0_to_v1_1(document, outdir)  # nothing needs doing for v1.2
+    document = _v1_0_to_v1_1(document, outdir)
+    return _v1_1_to_v1_2(document, outdir)
 
 
 def _v1_1_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
+    if "class" in document:
+        if document["class"] == "Workflow":
+            steps = document["steps"]
+            if isinstance(steps, MutableSequence):
+                for index, entry in enumerate(steps):
+                    with SourceLine(steps, index, Exception):
+                        if "run" in entry and isinstance(entry["run"], CommentedMap):
+                            process = entry["run"]
+                            _v1_1_to_v1_2(process, outdir)
+                            if "cwlVersion" in process:
+                                del process["cwlVersion"]
+
+                        elif isinstance(entry["run"], str) and "#" not in entry["run"]:
+                            if hasattr(document.lc, "filename"):
+                                dirname = Path(document.lc.filename).parent
+                            else:
+                                dirname = Path(outdir)
+                            path = dirname / entry["run"]
+                            process = v1_1_to_v1_2(load_cwl_document(str(path)), outdir)
+                            write_cwl_document(process, path.name, outdir)
+            elif isinstance(steps, MutableMapping):
+                for step_name in steps:
+                    with SourceLine(steps, step_name, Exception):
+                        entry = steps[step_name]
+                        if "run" in entry:
+                            if isinstance(entry["run"], CommentedMap):
+                                process = entry["run"]
+                                _v1_1_to_v1_2(process, outdir)
+                                if "cwlVersion" in process:
+                                    del process["cwlVersion"]
+                            elif (
+                                isinstance(entry["run"], str)
+                                and "#" not in entry["run"]
+                            ):
+                                if hasattr(document.lc, "filename"):
+                                    dirname = Path(document.lc.filename).parent
+                                else:
+                                    dirname = Path(outdir)
+                                path = dirname / entry["run"]
+                                process = v1_1_to_v1_2(
+                                    load_cwl_document(str(path)), outdir
+                                )
+                                write_cwl_document(process, path.name, outdir)
+                            elif isinstance(entry["run"], str) and "#" in entry["run"]:
+                                pass  # reference to $graph entry
+                            else:
+                                raise Exception(
+                                    "'run' entry was neither a CWL Process nor "
+                                    "a path to one: %s.",
+                                    entry["run"],
+                                )
     return document
 
 
@@ -445,10 +507,10 @@ def cleanup(inp: Dict[str, Any]) -> None:
 
 
 def move_up_loadcontents(document: Dict[str, Any]) -> None:
-    """'loadContents' is promoted up a level in CWL v1.1."""
+    """Promote 'loadContents' up a level for CWL v1.1."""
 
     def cleanup(inp: Dict[str, Any]) -> None:
-        """Move loadContents to the preferred location"""
+        """Move loadContents to the preferred location."""
         if "inputBinding" in inp:
             bindings = inp["inputBinding"]
             for field in list(bindings.keys()):

testdata/v1.0/1st-workflow.cwl

@@ -0,0 +1,26 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: Workflow
+inputs:
+  tarball: File
+  name_of_file_to_extract: string
+
+outputs:
+  compiled_class:
+    type: File
+    outputSource: compile/classfile
+
+steps:
+  untar:
+    run: tar-param.cwl
+    in:
+      tarfile: tarball
+      extractfile: name_of_file_to_extract
+    out: [extracted_file]
+
+  compile:
+    run: arguments.cwl
+    in:
+      src: untar/extracted_file
+    out: [classfile]

testdata/v1.0/arguments.cwl

@@ -0,0 +1,21 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: CommandLineTool
+label: Example trivial wrapper for Java 9 compiler
+hints:
+  DockerRequirement:
+    dockerPull: openjdk:9.0.1-11-slim
+baseCommand: javac
+arguments: ["-d", $(runtime.outdir)]
+inputs:
+  src:
+    type: File
+    inputBinding:
+      position: 1
+outputs:
+  classfile:
+    type: File
+    outputBinding:
+      glob: "*.class"
+

testdata/v1.0/tar-param.cwl

@@ -0,0 +1,19 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: [tar, --extract]
+inputs:
+  tarfile:
+    type: File
+    inputBinding:
+      prefix: --file
+  extractfile:
+    type: string
+    inputBinding:
+      position: 1
+outputs:
+  extracted_file:
+    type: File
+    outputBinding:
+      glob: $(inputs.extractfile)

testdata/v1.2/1st-workflow.cwl

@@ -0,0 +1,25 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: Workflow
+inputs:
+  tarball: File
+  name_of_file_to_extract: string
+
+steps:
+  untar:
+    run: tar-param.cwl
+    in:
+      tarfile: tarball
+      extractfile: name_of_file_to_extract
+    out: [extracted_file]
+
+  compile:
+    run: arguments.cwl
+    in:
+      src: untar/extracted_file
+    out: [classfile]
+outputs:
+  compiled_class:
+    type: File
+    outputSource: compile/classfile
+

testdata/v1.2/arguments.cwl

@@ -0,0 +1,25 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+label: Example trivial wrapper for Java 9 compiler
+requirements:
+  NetworkAccess:
+    networkAccess: true
+  LoadListingRequirement:
+    loadListing: deep_listing
+hints:
+  DockerRequirement:
+    dockerPull: openjdk:9.0.1-11-slim
+inputs:
+  src:
+    type: File
+    inputBinding:
+      position: 1
+baseCommand: javac
+arguments: ["-d", $(runtime.outdir)]
+outputs:
+  classfile:
+    type: File
+    outputBinding:
+      glob: "*.class"
+

testdata/v1.2/tar-param.cwl

@@ -0,0 +1,23 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+requirements:
+  NetworkAccess:
+    networkAccess: true
+  LoadListingRequirement:
+    loadListing: deep_listing
+inputs:
+  tarfile:
+    type: File
+    inputBinding:
+      prefix: --file
+  extractfile:
+    type: string
+    inputBinding:
+      position: 1
+baseCommand: [tar, --extract]
+outputs:
+  extracted_file:
+    type: File
+    outputBinding:
+      glob: $(inputs.extractfile)

tests/test_complete.py

@@ -99,3 +99,19 @@ def test_packed_graph(tmp_path: Path) -> None:
         tmp_path / "conflict-wf.cwl",
         shallow=False,
     )
+
+def test_multi_version_upgrade_external_steps(tmp_path: Path) -> None:
+    """Test 1.0 to 1.2 upgrade of Workflow with external steps."""
+    main(
+        [f"--dir={tmp_path}", get_data("testdata/v1.0/1st-workflow.cwl")]
+    )
+    assert filecmp.cmp(
+        get_data("testdata/v1.2/arguments.cwl"),
+        tmp_path / "arguments.cwl",
+        shallow=False,
+    )
+    assert filecmp.cmp(
+        get_data("testdata/v1.2/tar-param.cwl"),
+        tmp_path / "tar-param.cwl",
+        shallow=False,
+    )