From 4f5b88517d51dea7fb93bc32d62e810d9d1f8336 Mon Sep 17 00:00:00 2001 From: GlassOfWhiskey Date: Sat, 24 Dec 2022 17:09:54 +0100 Subject: [PATCH 01/12] Added loop support --- Workflow.yml | 112 +++++++++++++++- conformance_tests.yaml | 2 + tests/loop/all-output-loop-no-iteration.cwl | 28 ++++ tests/loop/all-output-loop.cwl | 28 ++++ tests/loop/default-value-loop.cwl | 47 +++++++ tests/loop/invalid-loop-scatter.cwl | 34 +++++ ...valid-multi-source-loop-no-requirement.cwl | 64 +++++++++ tests/loop/invalid-no-when.cwl | 32 +++++ ...invalid-value-from-loop-no-requirement.cwl | 32 +++++ tests/loop/loop-inside-loop-all.cwl | 58 +++++++++ tests/loop/loop-inside-loop.cwl | 54 ++++++++ tests/loop/loop-inside-scatter-job.yml | 2 + tests/loop/loop-inside-scatter.cwl | 49 +++++++ tests/loop/multi-source-loop.cwl | 65 ++++++++++ tests/loop/single-var-loop-job.yml | 1 + tests/loop/single-var-loop-no-iteration.cwl | 28 ++++ tests/loop/single-var-loop.cwl | 28 ++++ tests/loop/test-index.yaml | 121 ++++++++++++++++++ tests/loop/two-vars-loop-2.cwl | 31 +++++ tests/loop/two-vars-loop-job.yml | 2 + tests/loop/two-vars-loop.cwl | 33 +++++ tests/loop/value-from-loop.cwl | 33 +++++ 22 files changed, 879 insertions(+), 5 deletions(-) create mode 100644 tests/loop/all-output-loop-no-iteration.cwl create mode 100644 tests/loop/all-output-loop.cwl create mode 100644 tests/loop/default-value-loop.cwl create mode 100644 tests/loop/invalid-loop-scatter.cwl create mode 100644 tests/loop/invalid-multi-source-loop-no-requirement.cwl create mode 100644 tests/loop/invalid-no-when.cwl create mode 100644 tests/loop/invalid-value-from-loop-no-requirement.cwl create mode 100644 tests/loop/loop-inside-loop-all.cwl create mode 100644 tests/loop/loop-inside-loop.cwl create mode 100644 tests/loop/loop-inside-scatter-job.yml create mode 100644 tests/loop/loop-inside-scatter.cwl create mode 100644 tests/loop/multi-source-loop.cwl create mode 100644 tests/loop/single-var-loop-job.yml create mode 100644 tests/loop/single-var-loop-no-iteration.cwl create mode 100644 tests/loop/single-var-loop.cwl create mode 100644 tests/loop/test-index.yaml create mode 100644 tests/loop/two-vars-loop-2.cwl create mode 100644 tests/loop/two-vars-loop-job.yml create mode 100644 tests/loop/two-vars-loop.cwl create mode 100644 tests/loop/value-from-loop.cwl diff --git a/Workflow.yml b/Workflow.yml index 176de9a..3b896b3 100644 --- a/Workflow.yml +++ b/Workflow.yml @@ -443,6 +443,56 @@ $graph: to connect the output value to downstream parameters. +- name: LoopInput + type: record + extends: [Identified, Sink] + fields: + - name: source + doc: | + Specifies one or more of the step output parameters that will + provide input to the loop iterations after the first one (inputs + of the first iteration are the step input parameters). + jsonldPredicate: + "_id": "cwl:source" + "_type": "@id" + refScope: 1 + type: + - string? + - string[]? + - name: default + type: ["null", File, Directory, Any] + doc: | + The default value for this parameter to use if either there is no + `source` field, or the value produced by the `source` is `null`. The + default must be applied prior to scattering or evaluating `valueFrom`. + jsonldPredicate: + _id: "sld:default" + noLinkCheck: true + - name: valueFrom + type: + - "null" + - string + - Expression + jsonldPredicate: "cwl:valueFrom" + doc: | + To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must + be specified in the workflow or workflow step requirements. + + If `valueFrom` is a constant string value, use this as the value for + this input parameter. + + If `valueFrom` is a parameter reference or expression, it must be + evaluated to yield the actual value to be assigned to the input field. + + The `self` value in the parameter reference or expression must be + `null` if there is no `source` field, or the value of the + parameter(s) specified in the `source` field. + + The value of `inputs` in the parameter reference or expression must be + the input object to the previous iteration of the workflow step (or the initial + inputs for the first iteration). + + - name: ScatterMethod type: enum docParent: "#WorkflowStep" @@ -453,6 +503,15 @@ $graph: - flat_crossproduct +- name: LoopOutputMethod + type: enum + docParent: "#WorkflowStep" + doc: The loop output method, as described in [workflow step loop](#WorkflowStep). + symbols: + - last + - all + + - name: WorkflowStep type: record extends: [Identified, Labeled, sld:Documented] @@ -505,7 +564,7 @@ $graph: output arrays must be flattened to a single level, but otherwise listed in the order that the input arrays are listed in the `scatter` field. - # Conditional execution (Optional) + # Conditional and iterative execution (Optional) Conditional execution makes execution of a step conditional on an expression. A step that is not executed is "skipped". A skipped @@ -522,12 +581,37 @@ $graph: input object (or individual scatter job), and returns a boolean value. It is an error if this expression returns a value other than `true` or `false`. - - Conditionals in CWL are an optional feature and are not required - to be implemented by all consumers of CWL documents. An - implementation that does not support conditionals must return a + + The `loop` field controls iterative execution. It defines the input + parameters of the loop iterations after the first one (inputs of the + first iteration are the step input parameters, as usual). If no + `loop` rule is specified for a given step `in` field, the initial + value is kept constant among all iterations. + + When a `loop` field is present, the `when` field is mandatory. It is + evaluated before each loop iteration and acts as a termination condition: + as soon as the `when` expression evaluates to `false`, the loop terminates + and the step outputs are propagated to the subsequent workflow steps. + + The `outputMethod` field describes how to deal with loop outputs after + termination: + + * **last** specifies that only the last computed element for each output + parameter should be propagated to the subsequenct steps. This is the + default value. + + * **all** specifies that a single ordered array with all output values + computed at the end of each loop iteration should be propagated to the + subsequent steps. + + Conditionals and iterative execution in CWL are an optional features + and are not required to be implemented by all consumers of CWL documents. + An implementation that does not support conditionals must return a fatal error when attempting to execute a workflow that uses conditional constructs the implementation does not support. + + At this time, the `loop` field is not compatible with the `scatter` field. + Combining the two in the same step will produce an error. # Subworkflows @@ -619,6 +703,24 @@ $graph: jsonldPredicate: "_id": "cwl:scatterMethod" "_type": "@vocab" + - name: loop + doc: | + Defines the input parameters of the loop iterations after the first one + (inputs of the first iteration are the step input parameters). If no + `loop` rule is specified for a given step `in` field, the initial value + is kept constant among all iterations. + type: LoopInput[]? + jsonldPredicate: + _id: "cwl:loop" + mapSubject: id + mapPredicate: source + - name: outputMethod + doc: | + Required if `loop` is defined. + type: LoopOutputMethod? + jsonldPredicate: + "_id": "cwl:outputMethod" + "_type": "@vocab" - name: Workflow diff --git a/conformance_tests.yaml b/conformance_tests.yaml index 8d397d7..ee8c0a6 100644 --- a/conformance_tests.yaml +++ b/conformance_tests.yaml @@ -3525,3 +3525,5 @@ an_array_of_mixed_booleans: [ false, true, false ] an_array_of_trues: [ true, true, true ] an_int: 42 + +- $import: tests/loop/test-index.yaml diff --git a/tests/loop/all-output-loop-no-iteration.cwl b/tests/loop/all-output-loop-no-iteration.cwl new file mode 100644 index 0000000..98e9715 --- /dev/null +++ b/tests/loop/all-output-loop-no-iteration.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 1) + loop: + i1: o1 + outputMethod: all + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] diff --git a/tests/loop/all-output-loop.cwl b/tests/loop/all-output-loop.cwl new file mode 100644 index 0000000..959da94 --- /dev/null +++ b/tests/loop/all-output-loop.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: all + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] diff --git a/tests/loop/default-value-loop.cwl b/tests/loop/default-value-loop.cwl new file mode 100644 index 0000000..04d3e56 --- /dev/null +++ b/tests/loop/default-value-loop.cwl @@ -0,0 +1,47 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: loop/o1 + pickValue: all_non_null +steps: + loop: + when: $(inputs.i1 < 20) + loop: + i1: + source: o1 + default: 5 + outputMethod: all + run: + class: Workflow + inputs: + i1: int + outputs: + o1: + type: int? + outputSource: big_values/o1 + steps: + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [ o1 ] diff --git a/tests/loop/invalid-loop-scatter.cwl b/tests/loop/invalid-loop-scatter.cwl new file mode 100644 index 0000000..5bc9135 --- /dev/null +++ b/tests/loop/invalid-loop-scatter.cwl @@ -0,0 +1,34 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int[] + i2: int +outputs: + o1: + type: int[] + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + scatter: i1 + out: [o1] diff --git a/tests/loop/invalid-multi-source-loop-no-requirement.cwl b/tests/loop/invalid-multi-source-loop-no-requirement.cwl new file mode 100644 index 0000000..c611b68 --- /dev/null +++ b/tests/loop/invalid-multi-source-loop-no-requirement.cwl @@ -0,0 +1,64 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: [loop/osmall, loop/obig] + linkMerge: merge_flattened + pickValue: all_non_null +steps: + loop: + when: $(inputs.i1 < 20) + loop: + i1: + source: [ osmall, obig ] + pickValue: the_only_non_null + outputMethod: all + run: + class: Workflow + inputs: + i1: int + outputs: + osmall: + type: int? + outputSource: small_values/o1 + obig: + type: int? + outputSource: big_values/o1 + steps: + small_values: + when: $(inputs.i1 < 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [osmall, obig] diff --git a/tests/loop/invalid-no-when.cwl b/tests/loop/invalid-no-when.cwl new file mode 100644 index 0000000..12d03e4 --- /dev/null +++ b/tests/loop/invalid-no-when.cwl @@ -0,0 +1,32 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + loop: + i1: o1 + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/invalid-value-from-loop-no-requirement.cwl b/tests/loop/invalid-value-from-loop-no-requirement.cwl new file mode 100644 index 0000000..187894a --- /dev/null +++ b/tests/loop/invalid-value-from-loop-no-requirement.cwl @@ -0,0 +1,32 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: + valueFrom: $(inputs.i1 + 1) + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/loop-inside-loop-all.cwl b/tests/loop/loop-inside-loop-all.cwl new file mode 100644 index 0000000..a1ccf1c --- /dev/null +++ b/tests/loop/loop-inside-loop-all.cwl @@ -0,0 +1,58 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + StepInputExpressionRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: + type: array + items: + type: array + items: int + outputSource: loop1/o1 +steps: + loop1: + when: $(inputs.i2 < 4) + loop: + i2: + valueFrom: $(inputs.i2 + 1) + outputMethod: all + run: + class: Workflow + inputs: + i1: int + i2: int + outputs: + o1: + type: int[] + outputSource: loop2/o1 + steps: + loop2: + when: $(inputs.i1 <= inputs.i2) + loop: + i1: o1 + outputMethod: all + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + i2: i2 + out: [o1] + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/loop-inside-loop.cwl b/tests/loop/loop-inside-loop.cwl new file mode 100644 index 0000000..0929dc2 --- /dev/null +++ b/tests/loop/loop-inside-loop.cwl @@ -0,0 +1,54 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + StepInputExpressionRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int[] + outputSource: loop1/o1 +steps: + loop1: + when: $(inputs.i2 < 4) + loop: + i2: + valueFrom: $(inputs.i2 + 1) + outputMethod: all + run: + class: Workflow + inputs: + i1: int + i2: int + outputs: + o1: + type: int + outputSource: loop2/o1 + steps: + loop2: + when: $(inputs.i1 <= inputs.i2) + loop: + i1: o1 + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + i2: i2 + out: [o1] + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/loop-inside-scatter-job.yml b/tests/loop/loop-inside-scatter-job.yml new file mode 100644 index 0000000..0d95d06 --- /dev/null +++ b/tests/loop/loop-inside-scatter-job.yml @@ -0,0 +1,2 @@ +i1: [1, 2, 3, 4, 5] +i2: 1 \ No newline at end of file diff --git a/tests/loop/loop-inside-scatter.cwl b/tests/loop/loop-inside-scatter.cwl new file mode 100644 index 0000000..6211a63 --- /dev/null +++ b/tests/loop/loop-inside-scatter.cwl @@ -0,0 +1,49 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int[] + i2: int +outputs: + o1: + type: int[] + outputSource: scatter/o1 +steps: + scatter: + run: + class: Workflow + inputs: + i1: int + i2: int + outputs: + o1: + type: int + outputSource: subworkflow/o1 + steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] + in: + i1: i1 + i2: i2 + scatter: i1 + out: [o1] diff --git a/tests/loop/multi-source-loop.cwl b/tests/loop/multi-source-loop.cwl new file mode 100644 index 0000000..d36ecb5 --- /dev/null +++ b/tests/loop/multi-source-loop.cwl @@ -0,0 +1,65 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + MultipleInputFeatureRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: [loop/osmall, loop/obig] + linkMerge: merge_flattened + pickValue: all_non_null +steps: + loop: + when: $(inputs.i1 < 20) + loop: + i1: + source: [ osmall, obig ] + pickValue: the_only_non_null + outputMethod: all + run: + class: Workflow + inputs: + i1: int + outputs: + osmall: + type: int? + outputSource: small_values/o1 + obig: + type: int? + outputSource: big_values/o1 + steps: + small_values: + when: $(inputs.i1 < 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [osmall, obig] diff --git a/tests/loop/single-var-loop-job.yml b/tests/loop/single-var-loop-job.yml new file mode 100644 index 0000000..d491f14 --- /dev/null +++ b/tests/loop/single-var-loop-job.yml @@ -0,0 +1 @@ +i1: 1 \ No newline at end of file diff --git a/tests/loop/single-var-loop-no-iteration.cwl b/tests/loop/single-var-loop-no-iteration.cwl new file mode 100644 index 0000000..7565e62 --- /dev/null +++ b/tests/loop/single-var-loop-no-iteration.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 1) + loop: + i1: o1 + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] diff --git a/tests/loop/single-var-loop.cwl b/tests/loop/single-var-loop.cwl new file mode 100644 index 0000000..bbccadf --- /dev/null +++ b/tests/loop/single-var-loop.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] diff --git a/tests/loop/test-index.yaml b/tests/loop/test-index.yaml new file mode 100644 index 0000000..8fc3263 --- /dev/null +++ b/tests/loop/test-index.yaml @@ -0,0 +1,121 @@ +- tool: invalid-loop-scatter.cwl + id: loop_fail_scatter + doc: "Affirm that a loop workflow does not validate if scatter and loop directives are on the same step" + should_fail: true + tags: [ conditional, loop, inline_javascript, scatter, workflow ] + +- tool: invalid-no-when.cwl + id: loop_fail_no_when + doc: "Affirm that a loop workflow does not validate if no when directive is specified" + should_fail: true + tags: [ conditional, loop, inline_javascript, scatter, workflow ] + +- job: single-var-loop-job.yml + tool: single-var-loop.cwl + id: loop_single_variable + doc: "Test a simple loop case with a single variable" + output: + o1: 10 + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: single-var-loop-job.yml + tool: single-var-loop-no-iteration.cwl + id: loop_single_variable_no_iteration + doc: "Test a simple loop case with a single variable and a false condition" + output: + o1: null + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: two-vars-loop-job.yml + tool: two-vars-loop.cwl + id: loop_two_variables + doc: "Test a loop case with two variables, which are both back-propagated between iterations" + output: + o1: 10 + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: two-vars-loop-job.yml + tool: two-vars-loop-2.cwl + id: loop_two_variables_single_backpropagation + doc: "Test a loop case with two variables, but when only one of them is back-propagated between iterations" + output: + o1: 10 + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: single-var-loop-job.yml + tool: all-output-loop.cwl + id: loop_with_all_output_method + doc: "Test a loop case with outputMethod set to all" + output: + o1: [2, 3, 4, 5, 6, 7, 8, 9, 10] + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: single-var-loop-job.yml + tool: all-output-loop-no-iteration.cwl + id: loop_with_all_output_method_no_iteration + doc: "Test a loop case with outputMethod set to all and a false condition" + output: + o1: [] + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: two-vars-loop-job.yml + tool: value-from-loop.cwl + id: loop_value_from + doc: "Test a loop case with a variable generated by a valueFrom directive" + output: + o1: 10 + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: two-vars-loop-job.yml + tool: invalid-value-from-loop-no-requirement.cwl + id: loop_value_from_fail_no_requirement + doc: "Test that a workflow loop fails if a valueFrom directive is specified without StepInputExpressionRequirement" + should_fail: true + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: loop-inside-scatter-job.yml + tool: loop-inside-scatter.cwl + id: loop_inside_scatter + doc: "Test a loop subworkflow inside a scatter step" + output: + o1: [10, 10, 10, 10, 10] + tags: [ conditional, loop, inline_javascript, scatter, workflow ] + +- job: two-vars-loop-job.yml + tool: loop-inside-loop.cwl + id: loop_nested + doc: "Test a workflow with two nested loops" + output: + o1: [2, 3, 4] + tags: [ conditional, loop, inline_javascript, scatter, workflow ] + +- job: two-vars-loop-job.yml + tool: loop-inside-loop-all.cwl + id: loop_nested_all + doc: "Test a workflow with two nested loops, both with outputMethod set to all" + output: + o1: [[2], [2, 3], [2, 3, 4]] + tags: [ conditional, loop, inline_javascript, scatter, workflow ] + +- job: single-var-loop-job.yml + tool: multi-source-loop.cwl + id: loop_multi_source_input + doc: "Test a loop with two sources, which are selected through a pickValue directive" + output: + o1: [2, 3, 4, 5, 8, 11, 14, 17, 20] + tags: [ conditional, loop, inline_javascript, multiple_input, scatter, workflow ] + +- job: single-var-loop-job.yml + tool: invalid-multi-source-loop-no-requirement.cwl + id: loop_multi_source_input_fail_no_requirement + doc: "Test that a loop with two sources fails without MultipleInputFeatureRequirement" + should_fail: true + tags: [ conditional, loop, inline_javascript, multiple_input, scatter, workflow ] + +- job: single-var-loop-job.yml + tool: default-value-loop.cwl + id: loop_defaultvalue + output: + o1: [8, 11, 14, 17, 20] + doc: "Test a loop whose source has a default value" + tags: [ conditional, loop, inline_javascript, scatter, workflow ] diff --git a/tests/loop/two-vars-loop-2.cwl b/tests/loop/two-vars-loop-2.cwl new file mode 100644 index 0000000..bf01c7f --- /dev/null +++ b/tests/loop/two-vars-loop-2.cwl @@ -0,0 +1,31 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/two-vars-loop-job.yml b/tests/loop/two-vars-loop-job.yml new file mode 100644 index 0000000..012ed44 --- /dev/null +++ b/tests/loop/two-vars-loop-job.yml @@ -0,0 +1,2 @@ +i1: 1 +i2: 1 \ No newline at end of file diff --git a/tests/loop/two-vars-loop.cwl b/tests/loop/two-vars-loop.cwl new file mode 100644 index 0000000..74a22b3 --- /dev/null +++ b/tests/loop/two-vars-loop.cwl @@ -0,0 +1,33 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + i2: o2 + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + o2: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2, 'o2': inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1, o2] diff --git a/tests/loop/value-from-loop.cwl b/tests/loop/value-from-loop.cwl new file mode 100644 index 0000000..877ff87 --- /dev/null +++ b/tests/loop/value-from-loop.cwl @@ -0,0 +1,33 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + StepInputExpressionRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: + valueFrom: $(inputs.i1 + 1) + outputMethod: last + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] From 6679cdb932cfed1a65395917db263107c7bffc86 Mon Sep 17 00:00:00 2001 From: GlassOfWhiskey Date: Tue, 3 Jan 2023 17:04:32 +0100 Subject: [PATCH 02/12] Make `LoopInput` extend `OutputSink` --- Workflow.yml | 16 ++-------------- tests/loop/default-value-loop.cwl | 2 +- .../invalid-multi-source-loop-no-requirement.cwl | 2 +- tests/loop/multi-source-loop.cwl | 2 +- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/Workflow.yml b/Workflow.yml index 3b896b3..459f8d5 100644 --- a/Workflow.yml +++ b/Workflow.yml @@ -445,20 +445,8 @@ $graph: - name: LoopInput type: record - extends: [Identified, Sink] + extends: [Identified, OutputSink] fields: - - name: source - doc: | - Specifies one or more of the step output parameters that will - provide input to the loop iterations after the first one (inputs - of the first iteration are the step input parameters). - jsonldPredicate: - "_id": "cwl:source" - "_type": "@id" - refScope: 1 - type: - - string? - - string[]? - name: default type: ["null", File, Directory, Any] doc: | @@ -713,7 +701,7 @@ $graph: jsonldPredicate: _id: "cwl:loop" mapSubject: id - mapPredicate: source + mapPredicate: outputSource - name: outputMethod doc: | Required if `loop` is defined. diff --git a/tests/loop/default-value-loop.cwl b/tests/loop/default-value-loop.cwl index 04d3e56..2748cab 100644 --- a/tests/loop/default-value-loop.cwl +++ b/tests/loop/default-value-loop.cwl @@ -17,7 +17,7 @@ steps: when: $(inputs.i1 < 20) loop: i1: - source: o1 + outputSource: o1 default: 5 outputMethod: all run: diff --git a/tests/loop/invalid-multi-source-loop-no-requirement.cwl b/tests/loop/invalid-multi-source-loop-no-requirement.cwl index c611b68..5c055b4 100644 --- a/tests/loop/invalid-multi-source-loop-no-requirement.cwl +++ b/tests/loop/invalid-multi-source-loop-no-requirement.cwl @@ -18,7 +18,7 @@ steps: when: $(inputs.i1 < 20) loop: i1: - source: [ osmall, obig ] + outputSource: [ osmall, obig ] pickValue: the_only_non_null outputMethod: all run: diff --git a/tests/loop/multi-source-loop.cwl b/tests/loop/multi-source-loop.cwl index d36ecb5..719125b 100644 --- a/tests/loop/multi-source-loop.cwl +++ b/tests/loop/multi-source-loop.cwl @@ -19,7 +19,7 @@ steps: when: $(inputs.i1 < 20) loop: i1: - source: [ osmall, obig ] + outputSource: [ osmall, obig ] pickValue: the_only_non_null outputMethod: all run: From fbbb9fd4c9590ae8d1099ee12a3f7dca9d0a2e26 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 3 Jan 2023 18:36:04 +0100 Subject: [PATCH 03/12] spelling fixes --- CODE_OF_CONDUCT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 4f04189..1e8ce6d 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -57,7 +57,7 @@ Reporting If you are being harassed by a member of the CWL Project, notice that someone else is being harassed, or have any other concerns, please contact the CWL Leadership Team at leadership@commonwl.org. If person who is harassing -you is on the team, they will recuse themselves from handling your incident. We +you is on the team, they will recurse themselves from handling your incident. We will respond as promptly as we can. This code of conduct applies to CWL Project spaces, but if you are being From 523e9ee6d8b92f3bc33ff4314e5d75e90069cb69 Mon Sep 17 00:00:00 2001 From: GlassOfWhiskey Date: Wed, 4 Jan 2023 17:39:35 +0100 Subject: [PATCH 04/12] (multiple -> multiple_input) in test labels --- .github/config/wordlist.txt | 6 + CODE_OF_CONDUCT.md | 2 +- CONFORMANCE_TESTS.md | 4 +- Workflow.yml | 225 ++++++++++-------- design-documents/conditionals-2019.md | 4 +- tests/conditionals/test-index.yaml | 12 +- .../loop/multi-source-loop-no-pick-value.cwl | 64 +++++ 7 files changed, 213 insertions(+), 104 deletions(-) create mode 100644 tests/loop/multi-source-loop-no-pick-value.cwl diff --git a/.github/config/wordlist.txt b/.github/config/wordlist.txt index ec05e6b..30d75d8 100644 --- a/.github/config/wordlist.txt +++ b/.github/config/wordlist.txt @@ -1,3 +1,4 @@ +abstractworkflowstep acyclic amstutz arrayschema @@ -160,6 +161,9 @@ loadlistingrequirement localhost lookahead loopback +loopinput +loopoutputmethod +loopworkflowstep maccallum macos mappredicate @@ -194,6 +198,7 @@ outputbinding outputenumschema outputeval outputformat +outputmethod outputschema outputsink outputsource @@ -231,6 +236,7 @@ saladversion sbgenomics scatterfeaturerequirement scattermethod +scatterworkflowstep schemadefrequirement schemaorg schemas diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 1e8ce6d..4f04189 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -57,7 +57,7 @@ Reporting If you are being harassed by a member of the CWL Project, notice that someone else is being harassed, or have any other concerns, please contact the CWL Leadership Team at leadership@commonwl.org. If person who is harassing -you is on the team, they will recurse themselves from handling your incident. We +you is on the team, they will recuse themselves from handling your incident. We will respond as promptly as we can. This code of conduct applies to CWL Project spaces, but if you are being diff --git a/CONFORMANCE_TESTS.md b/CONFORMANCE_TESTS.md index 2262870..8e401ac 100644 --- a/CONFORMANCE_TESTS.md +++ b/CONFORMANCE_TESTS.md @@ -144,7 +144,7 @@ We will use this single entry to explain the format Must include one or more of the following tags: `command_line_tool`, `expression_tool` or `workflow`. If the test does not test any optional features, the tag `required` is required. -Because `conformance_tests.yaml` is a `schema-salad` processed document, [`$import`](https://www.commonwl.org/v1.2/SchemaSalad.html#Import) +Because `conformance_tests.yaml` is a `schema-salad` processed document, [`$import`](https://www.commonwl.org/v1.3/SchemaSalad.html#Import) can be used to organize the tests into separate files. Currently, the main file is too big (over 3400 lines); we are slowly re-organizing it. @@ -166,7 +166,7 @@ At any level, if there is an extra field, then that will be considered an error. An exception to this is `class: File` and `class: Directory` objects, the `cwl-runner` under test can add additional fields here without causing a test to fail. Likewise, if you don't want to test some aspect of a `class: File` or `class: Directory` object (like `nameext`) you can just omit it. -[According to the CWL standards](https://www.commonwl.org/v1.2/CommandLineTool.html#File), the format of the `location` field in +[According to the CWL standards](https://www.commonwl.org/v1.3/CommandLineTool.html#File), the format of the `location` field in `class: File` and `class: Directory` is implementation specific and we should not be testing them. Please remember to use `location: Any` for them. diff --git a/Workflow.yml b/Workflow.yml index 459f8d5..d745c7a 100644 --- a/Workflow.yml +++ b/Workflow.yml @@ -275,7 +275,7 @@ $graph: - type: record name: WorkflowStepInput extends: [Identified, InputSink, LoadContents, Labeled] - docParent: "#WorkflowStep" + docParent: "#AbstractWorkflowStep" doc: | The input of a workflow step connects an upstream parameter (from the workflow inputs, or the outputs of other workflows steps) with the input @@ -301,7 +301,7 @@ $graph: in the workflow or workflow step requirements. If the sink parameter is an array, or named in a [workflow - scatter](#WorkflowStep) operation, there may be multiple inbound + scatter](#ScatterWorkflowStep) operation, there may be multiple inbound data links listed in the `source` field. The values from the input links are merged depending on the method specified in the `linkMerge` field. If both `linkMerge` and `pickValue` are null @@ -338,7 +338,7 @@ $graph: 3. Before `scatter` or `valueFrom`. This is specifically intended to be useful in combination with - [conditional execution](#WorkflowStep), where several upstream + [conditional execution](#AbstractWorkflowStep), where several upstream steps may be connected to a single input (`source` is a list), and skipped steps produce null values. @@ -430,7 +430,7 @@ $graph: - type: record name: WorkflowStepOutput - docParent: "#WorkflowStep" + docParent: "#AbstractWorkflowStep" extends: Identified doc: | Associate an output parameter of the underlying process with a workflow @@ -451,10 +451,11 @@ $graph: type: ["null", File, Directory, Any] doc: | The default value for this parameter to use if either there is no - `source` field, or the value produced by the `source` is `null`. The + `outputSource` field, or the value produced by the `source` is `null`. The default must be applied prior to scattering or evaluating `valueFrom`. jsonldPredicate: _id: "sld:default" + _container: "@list" noLinkCheck: true - name: valueFrom type: @@ -483,8 +484,8 @@ $graph: - name: ScatterMethod type: enum - docParent: "#WorkflowStep" - doc: The scatter method, as described in [workflow step scatter](#WorkflowStep). + docParent: "#ScatterWorkflowStep" + doc: The scatter method, as described in [workflow step scatter](#ScatterWorkflowStep). symbols: - dotproduct - nested_crossproduct @@ -493,14 +494,14 @@ $graph: - name: LoopOutputMethod type: enum - docParent: "#WorkflowStep" - doc: The loop output method, as described in [workflow step loop](#WorkflowStep). + docParent: "#LoopWorkflowStep" + doc: The loop output method, as described in [workflow step loop](#LoopWorkflowStep). symbols: - last - all -- name: WorkflowStep +- name: AbstractWorkflowStep type: record extends: [Identified, Labeled, sld:Documented] docParent: "#Workflow" @@ -510,96 +511,23 @@ $graph: `Workflow`) in the `run` field and connects the input and output parameters of the underlying process to workflow parameters. - # Scatter/gather - - To use scatter/gather, - [ScatterFeatureRequirement](#ScatterFeatureRequirement) must be specified - in the workflow or workflow step requirements. - - A "scatter" operation specifies that the associated workflow step or - subworkflow should execute separately over a list of input elements. Each - job making up a scatter operation is independent and may be executed - concurrently. - - The `scatter` field specifies one or more input parameters which will be - scattered. An input parameter may be listed more than once. The declared - type of each input parameter implicitly becomes an array of items of the - input parameter type. If a parameter is listed more than once, it becomes - a nested array. As a result, upstream parameters which are connected to - scattered parameters must be arrays. - - All output parameter types are also implicitly wrapped in arrays. Each job - in the scatter results in an entry in the output array. - - If any scattered parameter runtime value is an empty array, all outputs are - set to empty arrays and no work is done for the step, according to - applicable scattering rules. - - If `scatter` declares more than one input parameter, `scatterMethod` - describes how to decompose the input into a discrete set of jobs. - - * **dotproduct** specifies that each of the input arrays are aligned and one - element taken from each array to construct each job. It is an error - if all input arrays are not the same length. - - * **nested_crossproduct** specifies the Cartesian product of the inputs, - producing a job for every combination of the scattered inputs. The - output must be nested arrays for each level of scattering, in the - order that the input arrays are listed in the `scatter` field. - - * **flat_crossproduct** specifies the Cartesian product of the inputs, - producing a job for every combination of the scattered inputs. The - output arrays must be flattened to a single level, but otherwise listed in the - order that the input arrays are listed in the `scatter` field. - - # Conditional and iterative execution (Optional) + # Conditional execution (Optional) Conditional execution makes execution of a step conditional on an expression. A step that is not executed is "skipped". A skipped step produces `null` for all output parameters. - The condition is evaluated after `scatter`, using the input object - of each individual scatter job. This means over a set of scatter - jobs, some may be executed and some may be skipped. When the - results are gathered, skipped steps must be `null` in the output - arrays. - The `when` field controls conditional execution. This is an expression that must be evaluated with `inputs` bound to the step input object (or individual scatter job), and returns a boolean value. It is an error if this expression returns a value other than `true` or `false`. - - The `loop` field controls iterative execution. It defines the input - parameters of the loop iterations after the first one (inputs of the - first iteration are the step input parameters, as usual). If no - `loop` rule is specified for a given step `in` field, the initial - value is kept constant among all iterations. - - When a `loop` field is present, the `when` field is mandatory. It is - evaluated before each loop iteration and acts as a termination condition: - as soon as the `when` expression evaluates to `false`, the loop terminates - and the step outputs are propagated to the subsequent workflow steps. - - The `outputMethod` field describes how to deal with loop outputs after - termination: - * **last** specifies that only the last computed element for each output - parameter should be propagated to the subsequenct steps. This is the - default value. - - * **all** specifies that a single ordered array with all output values - computed at the end of each loop iteration should be propagated to the - subsequent steps. - - Conditionals and iterative execution in CWL are an optional features - and are not required to be implemented by all consumers of CWL documents. - An implementation that does not support conditionals must return a - fatal error when attempting to execute a workflow that uses - conditional constructs the implementation does not support. - - At this time, the `loop` field is not compatible with the `scatter` field. - Combining the two in the same step will produce an error. + Conditional execution in CWL is an optional feature and is not required + to be implemented by all consumers of CWL documents. An implementation that + does not support conditional executions must return a fatal error when + attempting to execute a workflow that uses conditional constructs the + implementation does not support. # Subworkflows @@ -675,6 +603,68 @@ $graph: If defined, only run the step when the expression evaluates to `true`. If `false` the step is skipped. A skipped step produces a `null` on each output. + + +- name: WorkflowStep + type: record + extends: AbstractWorkflowStep + docParent: "#Workflow" + + +- name: ScatterWorkflowStep + type: record + extends: AbstractWorkflowStep + docParent: "#Workflow" + doc: | + To use scatter/gather, + [ScatterFeatureRequirement](#ScatterFeatureRequirement) must be specified + in the workflow or workflow step requirements. + + A "scatter" operation specifies that the associated workflow step or + subworkflow should execute separately over a list of input elements. Each + job making up a scatter operation is independent and may be executed + concurrently. + + The `scatter` field specifies one or more input parameters which will be + scattered. An input parameter may be listed more than once. The declared + type of each input parameter implicitly becomes an array of items of the + input parameter type. If a parameter is listed more than once, it becomes + a nested array. As a result, upstream parameters which are connected to + scattered parameters must be arrays. + + All output parameter types are also implicitly wrapped in arrays. Each job + in the scatter results in an entry in the output array. + + If any scattered parameter runtime value is an empty array, all outputs are + set to empty arrays and no work is done for the step, according to + applicable scattering rules. + + If `scatter` declares more than one input parameter, `scatterMethod` + describes how to decompose the input into a discrete set of jobs. + + * **dotproduct** specifies that each of the input arrays are aligned and one + element taken from each array to construct each job. It is an error + if all input arrays are not the same length. + + * **nested_crossproduct** specifies the Cartesian product of the inputs, + producing a job for every combination of the scattered inputs. The + output must be nested arrays for each level of scattering, in the + order that the input arrays are listed in the `scatter` field. + + * **flat_crossproduct** specifies the Cartesian product of the inputs, + producing a job for every combination of the scattered inputs. The + output arrays must be flattened to a single level, but otherwise listed in the + order that the input arrays are listed in the `scatter` field. + + # Conditional execution (Optional) + + The condition is evaluated after `scatter`, using the input object + of each individual scatter job. This means over a set of scatter + jobs, some may be executed and some may be skipped. When the + results are gathered, skipped steps must be `null` in the output + arrays. + + fields: - name: scatter type: - string? @@ -691,6 +681,45 @@ $graph: jsonldPredicate: "_id": "cwl:scatterMethod" "_type": "@vocab" + + +- name: LoopWorkflowStep + type: record + extends: AbstractWorkflowStep + docParent: "#Workflow" + doc: | + # Iterative execution (Optional) + + The `loop` field controls iterative execution. It defines the input + parameters of the loop iterations after the first one (inputs of the + first iteration are the step input parameters, as usual). If no + `loop` rule is specified for a given step `in` field, the initial + value is kept constant among all iterations. + + When a `loop` field is present, the `when` field is mandatory. It is + evaluated before each loop iteration and acts as a termination condition: + as soon as the `when` expression evaluates to `false`, the loop terminates + and the step outputs are propagated to the subsequent workflow steps. + + The `outputMethod` field describes how to deal with loop outputs after + termination: + + * **last** specifies that only the last computed element for each output + parameter should be propagated to the subsequent steps. This is the + default value. + + * **all** specifies that an array with all output values computed at the + end of each loop iteration should be propagated to the subsequent steps. + Elements in the array must be ordered according to the loop iterations + that produced them. + + Iterative execution in CWL is an optional feature and is not required + to be implemented by all consumers of CWL documents. An implementation that + does not support iterative executions must return a fatal error when + attempting to execute a workflow that uses iterative constructs the + implementation does not support. + + fields: - name: loop doc: | Defines the input parameters of the loop iterations after the first one @@ -704,11 +733,21 @@ $graph: mapPredicate: outputSource - name: outputMethod doc: | - Required if `loop` is defined. + If not specified, the default method is "last". type: LoopOutputMethod? + default: last jsonldPredicate: "_id": "cwl:outputMethod" "_type": "@vocab" + - name: when + type: + - Expression + jsonldPredicate: "cwl:when" + doc: | + Only run the next iteration when the expression evaluates to `true`. + If the first iteration evaluates to `false` the step is skipped. + A skipped step produces a `null` on each output if the `outputMethod` + is set to `last`, and an empty array if the `outputMethod` is set to `all`. - name: Workflow @@ -791,7 +830,7 @@ $graph: concurrently, provided that dependencies between steps are met. type: - type: array - items: "#WorkflowStep" + items: AbstractWorkflowStep jsonldPredicate: mapSubject: id @@ -801,7 +840,7 @@ $graph: extends: ProcessRequirement doc: | Indicates that the workflow platform must support nested workflows in - the `run` field of [WorkflowStep](#WorkflowStep). + the `run` field of [AbstractWorkflowStep](#AbstractWorkflowStep). fields: - name: "class" type: @@ -819,7 +858,7 @@ $graph: extends: ProcessRequirement doc: | Indicates that the workflow platform must support the `scatter` and - `scatterMethod` fields of [WorkflowStep](#WorkflowStep). + `scatterMethod` fields of [ScatterWorkflowStep](#ScatterWorkflowStep). fields: - name: "class" type: diff --git a/design-documents/conditionals-2019.md b/design-documents/conditionals-2019.md index 34f7d69..ca581c3 100644 --- a/design-documents/conditionals-2019.md +++ b/design-documents/conditionals-2019.md @@ -10,7 +10,7 @@ This is a documentation of the design and design decisions for conditionals as o ![dual scatter nested](conditionals/conditional-patterns-3.png) ![dual scatter flattened](conditionals/conditional-patterns-4.png) -The design adds a new field `when` to a `WorkflowStep`. This field is an expression that +The design adds a new field `when` to a `AbstractWorkflowStep`. This field is an expression that evaluates to `True` or `False`. The executor runs the step if the value is `True`, skips it if `False`. A skipped step produces `null` values on all its outputs. @@ -114,7 +114,7 @@ outputs: pickValue: first_non_null ``` -The new syntax adds a single field to `WorkflowStep` (`when`) and a new +The new syntax adds a single field to `AbstractWorkflowStep` (`when`) and a new operator called `pickValue` to the `WorkflowStepInput` and `WorkflowOutputParameter`. This is a fairly non-intrusive modification, fully backwards compatible (it's an addition, not a modification) and allows diff --git a/tests/conditionals/test-index.yaml b/tests/conditionals/test-index.yaml index cee57e9..f646a56 100644 --- a/tests/conditionals/test-index.yaml +++ b/tests/conditionals/test-index.yaml @@ -20,7 +20,7 @@ tags: [ conditional, inline_javascript, workflow ] - id: direct_required - doc: Conditional using intermediate WorkflowStep.in input + doc: Conditional using intermediate AbstractWorkflowStep.in input tool: cond-wf-002.cwl job: val.1.job.yaml output: @@ -186,7 +186,7 @@ out1: [ "foo 2", "foo 4", "foo 6", "bar 1", "bar 3", "bar 5"] - tags: [ conditional, inline_javascript, scatter, multiple, workflow ] + tags: [ conditional, inline_javascript, scatter, multiple_input, workflow ] - id: direct_optional_null_result_nojs doc: simplest conditional pattern (true), no javascript @@ -205,7 +205,7 @@ tags: [ conditional, workflow ] - id: direct_required_nojs - doc: Conditional using intermediate WorkflowStep.in input; no javascript + doc: Conditional using intermediate AbstractWorkflowStep.in input; no javascript tool: cond-wf-002_nojs.cwl job: val.1.job.yaml output: @@ -371,7 +371,7 @@ out1: [ "foo 2", "foo 4", "foo 6", "bar 1", "bar 3", "bar 5"] - tags: [ conditional, scatter, multiple, workflow ] + tags: [ conditional, scatter, multiple_input, workflow ] - id: cond-with-defaults-1 @@ -395,7 +395,7 @@ "size": 34 } ] - tags: [ conditional, scatter, multiple, workflow ] + tags: [ conditional, scatter, multiple_input, workflow ] - id: cond-with-defaults-2 doc: "Default inputs, choose step to run based on what was provided, second case" @@ -411,4 +411,4 @@ "size": 12 } ] - tags: [ conditional, scatter, multiple, workflow ] + tags: [ conditional, scatter, multiple_input, workflow ] diff --git a/tests/loop/multi-source-loop-no-pick-value.cwl b/tests/loop/multi-source-loop-no-pick-value.cwl new file mode 100644 index 0000000..39a1474 --- /dev/null +++ b/tests/loop/multi-source-loop-no-pick-value.cwl @@ -0,0 +1,64 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + MultipleInputFeatureRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: [loop/osmall, loop/obig] + linkMerge: merge_flattened +steps: + loop: + when: $(inputs.i1 < 5) + loop: + i1: + outputSource: [ osmall, obig ] + valueFrom: $(self[0]) + outputMethod: all + run: + class: Workflow + inputs: + i1: int + outputs: + osmall: + type: int? + outputSource: small_values/o1 + obig: + type: int? + outputSource: big_values/o1 + steps: + small_values: + when: $(inputs.i1 < 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [osmall, obig] From 9557eeb7d9849e63a2fb1e8749d48e15ae0c6318 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 16 Jul 2024 15:24:40 -0400 Subject: [PATCH 05/12] AbstractWorkflowStep needs to be marked as 'abstract: true' --- Workflow.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Workflow.yml b/Workflow.yml index d745c7a..f83575b 100644 --- a/Workflow.yml +++ b/Workflow.yml @@ -480,7 +480,7 @@ $graph: The value of `inputs` in the parameter reference or expression must be the input object to the previous iteration of the workflow step (or the initial inputs for the first iteration). - + - name: ScatterMethod type: enum @@ -504,6 +504,7 @@ $graph: - name: AbstractWorkflowStep type: record extends: [Identified, Labeled, sld:Documented] + abstract: true docParent: "#Workflow" doc: | A workflow step is an executable element of a workflow. It specifies the @@ -655,9 +656,9 @@ $graph: producing a job for every combination of the scattered inputs. The output arrays must be flattened to a single level, but otherwise listed in the order that the input arrays are listed in the `scatter` field. - + # Conditional execution (Optional) - + The condition is evaluated after `scatter`, using the input object of each individual scatter job. This means over a set of scatter jobs, some may be executed and some may be skipped. When the @@ -689,30 +690,30 @@ $graph: docParent: "#Workflow" doc: | # Iterative execution (Optional) - + The `loop` field controls iterative execution. It defines the input parameters of the loop iterations after the first one (inputs of the first iteration are the step input parameters, as usual). If no `loop` rule is specified for a given step `in` field, the initial value is kept constant among all iterations. - + When a `loop` field is present, the `when` field is mandatory. It is evaluated before each loop iteration and acts as a termination condition: as soon as the `when` expression evaluates to `false`, the loop terminates and the step outputs are propagated to the subsequent workflow steps. - + The `outputMethod` field describes how to deal with loop outputs after termination: * **last** specifies that only the last computed element for each output - parameter should be propagated to the subsequent steps. This is the + parameter should be propagated to the subsequent steps. This is the default value. - + * **all** specifies that an array with all output values computed at the end of each loop iteration should be propagated to the subsequent steps. Elements in the array must be ordered according to the loop iterations that produced them. - + Iterative execution in CWL is an optional feature and is not required to be implemented by all consumers of CWL documents. An implementation that does not support iterative executions must return a fatal error when From b85595df67e0cb4db3f8fd95a57e07c7be035119 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 17 Jul 2024 16:21:21 -0400 Subject: [PATCH 06/12] conformance tests: report the names of the failures --- .github/workflows/cwltool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cwltool.yml b/.github/workflows/cwltool.yml index 9bb190a..0579183 100644 --- a/.github/workflows/cwltool.yml +++ b/.github/workflows/cwltool.yml @@ -33,4 +33,4 @@ jobs: run: cp conformance_tests.yaml conformance_tests.cwltest.yaml - name: Run tests against the reference runner - run: python -m pytest conformance_tests.cwltest.yaml -n auto -rs + run: python -m pytest conformance_tests.cwltest.yaml -n auto -rsfE From adba98ff185148732927374186d0643859edcea1 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 18 Jul 2024 10:14:11 -0400 Subject: [PATCH 07/12] Move initialWorkDir, envDef and networkAccess to CommandLineTool These are core features and using them via 'Requirements' just makes the syntax awkward. --- CommandLineTool.yml | 93 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/CommandLineTool.yml b/CommandLineTool.yml index 604c668..6ba7ae9 100644 --- a/CommandLineTool.yml +++ b/CommandLineTool.yml @@ -711,6 +711,91 @@ $graph: If not specified, all exit codes except 0 are considered permanent failure. + - name: networkAccess + type: [boolean, Expression] + doc: | + Indicate whether a process requires outgoing IPv4/IPv6 network + access. Choice of IPv4 or IPv6 is implementation and site + specific, correct tools must support both. + + If `networkAccess` is false or not specified, tools must not + assume network access, except for localhost (the loopback device). + + If `networkAccess` is true, the tool must be able to make outgoing + connections to network resources. Resources may be on a private + subnet or the public Internet. However, implementations and sites + may apply their own security policies to restrict what is + accessible by the tool. + + Enabling network access does not imply a publicly routable IP + address or the ability to accept inbound connections. + + - name: initialWorkDir + type: + - Expression + - type: array + items: + - "null" + - Dirent + - Expression + - File + - Directory + - type: array + items: + - File + - Directory + doc: | + Define a list of files and subdirectories that must be staged + by the workflow platform prior to executing the command line + tool. Normally this is the designated output directory, but + when containers are used, it is possible to stage to other + locations, see discussion below. + + Return type of each expression must validate as `["null", + File, Directory, Dirent, {type: array, items: [File, + Directory]}]`. + + Each `File` or `Directory` that is returned by an Expression + must be added to the designated output directory prior to + executing the tool. + + Each `Dirent` record that is listed or returned by an + expression specifies a file to be created or staged in the + designated output directory prior to executing the tool. + + Expressions may return null, in which case they have no effect. + + Files or Directories which are listed in the input parameters + and appear in `initialWorkDir` must have their `path` set to + their staged location. If the same File or Directory appears + more than once in the `initialWorkDir`, the implementation + must choose exactly one value for `path`; how this value is + chosen is undefined. + + Normally files are staged within the designated output + directory. However, when running inside containers, files may + be staged at arbitrary locations, see discussion for + [`Dirent.entryname`](#Dirent). Together with + `DockerRequirement.dockerOutputDirectory` it is possible to + control the locations of both input and output files when + running in containers. + + Files listed in `inputs` that are not listed in + `initialWorkDir` shall be staged to an arbitrary location on + the file system. This location must be outside the designated + output directory. + + - name: envDef + type: EnvironmentDef[] + doc: | + Define a list of environment variables which will be set in + the execution environment of the tool. See + [EnvironmentDef](#EnvironmentDef) for details. + jsonldPredicate: + mapSubject: envName + mapPredicate: envValue + + - type: record name: DockerRequirement @@ -991,7 +1076,9 @@ $graph: - name: InitialWorkDirRequirement type: record extends: ProcessRequirement - doc: + doc: | + **Deprecated as of v1.3** Prefer [CommandLineTool.initialWorkDir](#CommandLineTool) + Define a list of files and subdirectories that must be staged by the workflow platform prior to executing the command line tool. @@ -1059,6 +1146,8 @@ $graph: type: record extends: ProcessRequirement doc: | + **Deprecated as of v1.3** Prefer [CommandLineTool.envDef](#CommandLineTool) + Define a list of environment variables which will be set in the execution environment of the tool. See `EnvironmentDef` for details. fields: @@ -1291,6 +1380,8 @@ $graph: name: NetworkAccess extends: ProcessRequirement doc: | + **Deprecated as of v1.3** Prefer [CommandLineTool.networkAccess](#CommandLineTool) + Indicate whether a process requires outgoing IPv4/IPv6 network access. Choice of IPv4 or IPv6 is implementation and site specific, correct tools must support both. From 186667874554b703f8a013fa7c71d16abbbf2b4c Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 18 Jul 2024 10:29:25 -0400 Subject: [PATCH 08/12] Deprecate boilerplate feature requirements and update changelog Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- CommandLineTool.yml | 14 ++++++++++++ Workflow.yml | 54 ++++++++++++++++++++++++++------------------- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/CommandLineTool.yml b/CommandLineTool.yml index 6ba7ae9..4913159 100644 --- a/CommandLineTool.yml +++ b/CommandLineTool.yml @@ -52,6 +52,20 @@ $graph: ## Changelog for v1.3.0-dev1 See also the [CWL Workflow Description, v1.3.0-dev1 changelog](Workflow.html#Changelog). + + * New field `networkAccess` on + [CommandLineTool](#CommandLineTool). This supercedes the + [NetworkAccess](#NetworkAccess) requirement, which is now + deprecated. + * New field `envDef` on [CommandLineTool](#CommandLineTool). + This supercedes the [EnvVarRequirement](#EnvVarRequirement) + requirement, which is now deprecated. + * New field `initialWorkDir` on + [CommandLineTool](#CommandLineTool). This supercedes the + [InitialWorkDirRequirement](#InitialWorkDirRequirement) + requirement, which is now deprecated. + + For other changes since CWL v1.0, see the [CWL Command Line Tool Description, v1.1 changelog](https://www.commonwl.org/v1.1/CommandLineTool.html#Changelog) and diff --git a/Workflow.yml b/Workflow.yml index f83575b..0bf76b8 100644 --- a/Workflow.yml +++ b/Workflow.yml @@ -41,6 +41,17 @@ $graph: ## Changelog See also the [CWL Command Line Tool Description, v1.3.0-dev1 changelog](CommandLineTool.html#Changelog). + + * Workflow steps may now specify iterative execution. See + [LoopWorkflowStep](#LoopWorkflowStep) + * Including + [StepInputExpressionRequirement](#StepInputExpressionRequirement), + [MultipleInputFeatureRequirement](#MultipleInputFeatureRequirement), + [ScatterFeatureRequirement](#ScatterFeatureRequirement) and + [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) + in `requirements` is no longer necessary to use the associated + feature. These core features are now always available. + For other changes since CWL v1.0, see the [CWL Workflow Description, v1.1 changelog](https://www.commonwl.org/v1.1/Workflow.html#Changelog) and @@ -296,10 +307,6 @@ $graph: # Merging multiple inbound data links - To merge multiple inbound data links, - [MultipleInputFeatureRequirement](#MultipleInputFeatureRequirement) must be specified - in the workflow or workflow step requirements. - If the sink parameter is an array, or named in a [workflow scatter](#ScatterWorkflowStep) operation, there may be multiple inbound data links listed in the `source` field. The values from the @@ -404,9 +411,6 @@ $graph: - Expression jsonldPredicate: "cwl:valueFrom" doc: | - To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must - be specified in the workflow or workflow step requirements. - If `valueFrom` is a constant string value, use this as the value for this input parameter. @@ -464,9 +468,6 @@ $graph: - Expression jsonldPredicate: "cwl:valueFrom" doc: | - To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must - be specified in the workflow or workflow step requirements. - If `valueFrom` is a constant string value, use this as the value for this input parameter. @@ -532,10 +533,6 @@ $graph: # Subworkflows - To specify a nested workflow as part of a workflow step, - [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) must be - specified in the workflow or workflow step requirements. - It is a fatal error if a workflow directly or indirectly invokes itself as a subworkflow (recursive workflows are not allowed). @@ -617,10 +614,6 @@ $graph: extends: AbstractWorkflowStep docParent: "#Workflow" doc: | - To use scatter/gather, - [ScatterFeatureRequirement](#ScatterFeatureRequirement) must be specified - in the workflow or workflow step requirements. - A "scatter" operation specifies that the associated workflow step or subworkflow should execute separately over a list of input elements. Each job making up a scatter operation is independent and may be executed @@ -806,12 +799,11 @@ $graph: * If all workflow steps are executed and complete with `success`, then the workflow status is `success`. - # Extensions + # Complex steps - [ScatterFeatureRequirement](#ScatterFeatureRequirement) and - [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) are - available as standard [extensions](#Extensions_and_Metadata) to core - workflow semantics. + Individual workflow steps may specify complex behavior. Currently + this consists scatter/gather, conditional execution, iterative + execution (loops), or executing other workflows. fields: - name: "class" @@ -840,6 +832,10 @@ $graph: name: SubworkflowFeatureRequirement extends: ProcessRequirement doc: | + **Deprecated as of v1.3** This feature is available by default, + workflows that use it do not need to declare it. Including this + requirement is a no-op. + Indicates that the workflow platform must support nested workflows in the `run` field of [AbstractWorkflowStep](#AbstractWorkflowStep). fields: @@ -858,6 +854,10 @@ $graph: type: record extends: ProcessRequirement doc: | + **Deprecated as of v1.3** This feature is available by default, + workflows that use it do not need to declare it. Including this + requirement is a no-op. + Indicates that the workflow platform must support the `scatter` and `scatterMethod` fields of [ScatterWorkflowStep](#ScatterWorkflowStep). fields: @@ -876,6 +876,10 @@ $graph: type: record extends: ProcessRequirement doc: | + **Deprecated as of v1.3** This feature is available by default, + workflows that use it do not need to declare it. Including this + requirement is a no-op. + Indicates that the workflow platform must support multiple inbound data links listed in the `source` field of [WorkflowStepInput](#WorkflowStepInput). fields: @@ -894,6 +898,10 @@ $graph: name: StepInputExpressionRequirement extends: ProcessRequirement doc: | + **Deprecated as of v1.3** This feature is available by default, + workflows that use it do not need to declare it. Including this + requirement is a no-op. + Indicate that the workflow platform must support the `valueFrom` field of [WorkflowStepInput](#WorkflowStepInput). fields: From 8cac90e6bab4ff0d0ac25746682a5ab50232bc27 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 18 Jul 2024 10:48:56 -0400 Subject: [PATCH 09/12] Set predicates for networkAccess and envDef --- CommandLineTool.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CommandLineTool.yml b/CommandLineTool.yml index 4913159..9ac851a 100644 --- a/CommandLineTool.yml +++ b/CommandLineTool.yml @@ -727,6 +727,7 @@ $graph: - name: networkAccess type: [boolean, Expression] + jsonldPredicate: "cwl:NetworkAccess/networkAccess" doc: | Indicate whether a process requires outgoing IPv4/IPv6 network access. Choice of IPv4 or IPv6 is implementation and site @@ -806,6 +807,7 @@ $graph: the execution environment of the tool. See [EnvironmentDef](#EnvironmentDef) for details. jsonldPredicate: + "_id": "cwl:EnvVarRequirement/envDef" mapSubject: envName mapPredicate: envValue @@ -1179,6 +1181,7 @@ $graph: type: EnvironmentDef[] doc: The list of environment variables. jsonldPredicate: + "_id": "cwl:EnvVarRequirement/envDef" mapSubject: envName mapPredicate: envValue @@ -1425,6 +1428,7 @@ $graph: "_type": "@vocab" - name: networkAccess type: [boolean, Expression] + jsonldPredicate: "cwl:NetworkAccess/networkAccess" - name: InplaceUpdateRequirement type: record From 4e791893b658ce9e1d93cb747f62bd8b29d8abbb Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 18 Jul 2024 11:14:04 -0400 Subject: [PATCH 10/12] Make networkAccess, envDef and initialWorkDir optional --- CommandLineTool.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CommandLineTool.yml b/CommandLineTool.yml index 9ac851a..09cd1d6 100644 --- a/CommandLineTool.yml +++ b/CommandLineTool.yml @@ -726,7 +726,7 @@ $graph: If not specified, all exit codes except 0 are considered permanent failure. - name: networkAccess - type: [boolean, Expression] + type: ['null', boolean, Expression] jsonldPredicate: "cwl:NetworkAccess/networkAccess" doc: | Indicate whether a process requires outgoing IPv4/IPv6 network @@ -747,6 +747,7 @@ $graph: - name: initialWorkDir type: + - 'null' - Expression - type: array items: @@ -801,7 +802,7 @@ $graph: output directory. - name: envDef - type: EnvironmentDef[] + type: EnvironmentDef[]? doc: | Define a list of environment variables which will be set in the execution environment of the tool. See From 258966e00801df204fd945a875f9191e18529c03 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 18 Jul 2024 13:49:26 -0400 Subject: [PATCH 11/12] `networkAcess`, `envDef`, and `initialWorkDir` tests --- conformance_tests.yaml | 510 +++++++++++++++++- tests/dir5_noiwdr.cwl | 18 + tests/env-tool1_noreq.cwl | 16 + tests/env-tool4_noreq.cwl | 16 + tests/env-wf1_noreq.cwl | 23 + tests/initialwork-path_noiwdr.cwl | 14 + tests/initialworkdir-glob-fullpath_noiwdr.cwl | 26 + ...alworkdirrequirement-docker-out_noiwdr.cwl | 30 ++ tests/inp_update_wf_noiwdr.cwl | 33 ++ tests/inpdir_update_wf_noiwdr.cwl | 42 ++ tests/iwd/iwd-container-entryname1_noiwdr.cwl | 27 + tests/iwd/iwd-container-entryname2_noiwdr.cwl | 21 + tests/iwd/iwd-container-entryname3_noiwdr.cwl | 24 + tests/iwd/iwd-container-entryname4_noiwdr.cwl | 21 + tests/iwd/iwd-fileobjs1_noiwdr.cwl | 20 + tests/iwd/iwd-fileobjs2_noiwdr.cwl | 18 + tests/iwd/iwd-jsondump1-nl_noiwdr.cwl | 22 + tests/iwd/iwd-jsondump1_noiwdr.cwl | 23 + tests/iwd/iwd-jsondump2-nl_noiwdr.cwl | 23 + tests/iwd/iwd-jsondump2_noiwdr.cwl | 23 + tests/iwd/iwd-jsondump3-nl_noiwdr.cwl | 19 + tests/iwd/iwd-jsondump3_noiwdr.cwl | 19 + tests/iwd/iwd-nolimit_noiwdr.cwl | 24 + tests/iwd/iwd-passthrough1_noreq.cwl | 22 + tests/iwd/iwd-passthrough2_noiwdr.cwl | 29 + tests/iwd/iwd-passthrough3_noreq.cwl | 15 + tests/iwd/iwd-passthrough4_noreq.cwl | 14 + tests/iwd/iwd-passthrough5_noreq.cwl | 15 + tests/iwd/iwd-subdir-tool_noreq.cwl | 15 + tests/iwd/iwd-subdir-wf_noreq.cwl | 19 + tests/iwd/iwd_dir_literal_real_file.cwl | 25 + tests/iwd/test-index.yaml | 357 +++++++++++- tests/iwdr-entry_noreq.cwl | 18 + tests/iwdr_with_nested_dirs_noreq.cwl | 39 ++ tests/linkfile_noreq.cwl | 20 + tests/networkaccess_noreq.cwl | 14 + tests/recursive-input-directory_noiwdr.cwl | 33 ++ tests/rename_noreq.cwl | 14 + tests/search_noiwdr.cwl | 109 ++++ .../secondaryfiles/rename-outputs_noiwdr.cwl | 49 ++ tests/secondaryfiles/test-index.yaml | 20 +- tests/stage-array-dirs_noreq.cwl | 20 + tests/stage-array_noiwdr.cwl | 32 ++ ...le_array_basename_and_entryname_noiwdr.cwl | 23 + tests/stage_file_array_noiwdr.cwl | 23 + tests/stagefile_noreq.cwl | 25 + .../bash-dollar-quote_noiwdr.cwl | 56 ++ ...ine-continuation-with-expression_noreq.cwl | 20 + .../bash-line-continuation_noreq.cwl | 17 + .../string-interpolation/js-quote_noiwdr.cwl | 36 ++ tests/string-interpolation/test-index.yaml | 62 ++- tests/template-tool_noiwdr.cwl | 25 + tests/updatedir_inplace_noiwdr.cwl | 20 + tests/updateval_inplace_noiwdr.cwl | 27 + tests/writable-dir-docker_noiwdr.cwl | 23 + tests/writable-dir_noiwdr.cwl | 17 + 56 files changed, 2220 insertions(+), 45 deletions(-) create mode 100644 tests/dir5_noiwdr.cwl create mode 100644 tests/env-tool1_noreq.cwl create mode 100644 tests/env-tool4_noreq.cwl create mode 100644 tests/env-wf1_noreq.cwl create mode 100644 tests/initialwork-path_noiwdr.cwl create mode 100644 tests/initialworkdir-glob-fullpath_noiwdr.cwl create mode 100644 tests/initialworkdirrequirement-docker-out_noiwdr.cwl create mode 100644 tests/inp_update_wf_noiwdr.cwl create mode 100644 tests/inpdir_update_wf_noiwdr.cwl create mode 100644 tests/iwd/iwd-container-entryname1_noiwdr.cwl create mode 100644 tests/iwd/iwd-container-entryname2_noiwdr.cwl create mode 100644 tests/iwd/iwd-container-entryname3_noiwdr.cwl create mode 100644 tests/iwd/iwd-container-entryname4_noiwdr.cwl create mode 100644 tests/iwd/iwd-fileobjs1_noiwdr.cwl create mode 100644 tests/iwd/iwd-fileobjs2_noiwdr.cwl create mode 100644 tests/iwd/iwd-jsondump1-nl_noiwdr.cwl create mode 100644 tests/iwd/iwd-jsondump1_noiwdr.cwl create mode 100644 tests/iwd/iwd-jsondump2-nl_noiwdr.cwl create mode 100644 tests/iwd/iwd-jsondump2_noiwdr.cwl create mode 100644 tests/iwd/iwd-jsondump3-nl_noiwdr.cwl create mode 100644 tests/iwd/iwd-jsondump3_noiwdr.cwl create mode 100644 tests/iwd/iwd-nolimit_noiwdr.cwl create mode 100644 tests/iwd/iwd-passthrough1_noreq.cwl create mode 100644 tests/iwd/iwd-passthrough2_noiwdr.cwl create mode 100644 tests/iwd/iwd-passthrough3_noreq.cwl create mode 100644 tests/iwd/iwd-passthrough4_noreq.cwl create mode 100644 tests/iwd/iwd-passthrough5_noreq.cwl create mode 100644 tests/iwd/iwd-subdir-tool_noreq.cwl create mode 100644 tests/iwd/iwd-subdir-wf_noreq.cwl create mode 100644 tests/iwd/iwd_dir_literal_real_file.cwl create mode 100644 tests/iwdr-entry_noreq.cwl create mode 100644 tests/iwdr_with_nested_dirs_noreq.cwl create mode 100644 tests/linkfile_noreq.cwl create mode 100644 tests/networkaccess_noreq.cwl create mode 100644 tests/recursive-input-directory_noiwdr.cwl create mode 100644 tests/rename_noreq.cwl create mode 100644 tests/search_noiwdr.cwl create mode 100644 tests/secondaryfiles/rename-outputs_noiwdr.cwl create mode 100644 tests/stage-array-dirs_noreq.cwl create mode 100644 tests/stage-array_noiwdr.cwl create mode 100644 tests/stage_file_array_basename_and_entryname_noiwdr.cwl create mode 100644 tests/stage_file_array_noiwdr.cwl create mode 100644 tests/stagefile_noreq.cwl create mode 100644 tests/string-interpolation/bash-dollar-quote_noiwdr.cwl create mode 100644 tests/string-interpolation/bash-line-continuation-with-expression_noreq.cwl create mode 100644 tests/string-interpolation/bash-line-continuation_noreq.cwl create mode 100644 tests/string-interpolation/js-quote_noiwdr.cwl create mode 100755 tests/template-tool_noiwdr.cwl create mode 100755 tests/updatedir_inplace_noiwdr.cwl create mode 100755 tests/updateval_inplace_noiwdr.cwl create mode 100644 tests/writable-dir-docker_noiwdr.cwl create mode 100644 tests/writable-dir_noiwdr.cwl diff --git a/conformance_tests.yaml b/conformance_tests.yaml index ee8c0a6..09f9b0e 100644 --- a/conformance_tests.yaml +++ b/conformance_tests.yaml @@ -431,7 +431,7 @@ size: 15 tool: tests/env-wf1.cwl id: requirement_priority - doc: Test requirement priority + doc: Test requirement priority using nested EnvVarRequirements tags: [ env_var, workflow ] - job: tests/env-job.json @@ -614,7 +614,7 @@ size: 16 tool: tests/iwdr-entry.cwl id: initial_workdir_trailingnl - doc: Test if trailing newline is present in file entry in InitialWorkDir + doc: Test if trailing newline is present in file entry in InitialWorkDirRequirement tags: [ initial_work_dir, command_line_tool ] - job: tests/wc-job.json @@ -1167,7 +1167,7 @@ } tool: tests/dir5.cwl id: dynamic_initial_workdir - doc: Test dynamic initial work dir + doc: Test dynamic InitialWorkDirRequirement tags: [ shell_command, initial_work_dir, command_line_tool ] - job: tests/stagefile-job.yml @@ -1181,7 +1181,7 @@ } tool: tests/stagefile.cwl id: writable_stagedfiles - doc: Test writable staged files. + doc: Test writable staged files using InitialWorkDirRequirement. tags: [ initial_work_dir, command_line_tool ] - job: tests/file-literal.yml @@ -1205,7 +1205,7 @@ size: 0 tool: tests/linkfile.cwl id: initial_workdir_expr - doc: Test expression in InitialWorkDir listing + doc: Test expression in InitialWorkDirRequirement listing tags: [ initial_work_dir, command_line_tool ] - job: tests/wc-job.json @@ -1448,7 +1448,7 @@ } tool: tests/recursive-input-directory.cwl id: input_dir_recurs_copy_writable - doc: Test if a writable input directory is recursively copied and writable + doc: Test if a writable IWDR input directory is recursively copied and writable tags: [ initial_work_dir, shell_command, command_line_tool ] - output: @@ -1502,7 +1502,7 @@ output: {} tool: tests/initialwork-path.cwl id: initialworkpath_output - doc: Test that file path in $(inputs) for initialworkdir is in $(outdir). + doc: Test that file path in $(inputs) for initialWorkDirRequirement is in $(outdir). tags: [ initial_work_dir, command_line_tool ] - job: tests/count-lines6-job.json @@ -2690,7 +2690,7 @@ - job: tests/empty.json output: {} tool: tests/networkaccess.cwl - doc: Test networkaccess enabled + doc: Test networkaccess enabled using the NetworkAccess requirement id: networkaccess tags: [ command_line_tool, networkaccess ] @@ -2747,7 +2747,7 @@ tool: tests/env-tool3.cwl id: cwl_requirements_addition doc: Test requirements in input document via EnvVarRequirement - tags: [ command_line_tool, input_object_requirements ] + tags: [ command_line_tool, input_object_requirements, env_var ] - job: tests/env-job3.yaml output: @@ -2759,7 +2759,7 @@ tool: tests/env-tool4.cwl id: cwl_requirements_override_expression doc: Test conflicting requirements in input document via EnvVarRequirement and expression - tags: [ command_line_tool, input_object_requirements ] + tags: [ command_line_tool, input_object_requirements, env_var ] - job: tests/env-job4.yaml output: @@ -2771,7 +2771,7 @@ id: cwl_requirements_override_static tool: tests/env-tool3.cwl doc: Test conflicting requirements in input document via EnvVarRequirement - tags: [ command_line_tool, input_object_requirements ] + tags: [ command_line_tool, input_object_requirements, env_var ] - job: tests/initialworkdirrequirement-docker-out-job.json output: @@ -2788,7 +2788,7 @@ "size": 12010 tool: tests/initialworkdir-glob-fullpath.cwl id: initial_workdir_output_glob - doc: Test full path glob output of InitialWorkDir + doc: Test full path glob output of InitialWorkDirRequirement tags: [ initial_work_dir, command_line_tool ] - job: tests/empty.json @@ -2825,8 +2825,8 @@ - job: tests/empty.json tool: tests/inp_update_wf.cwl - doc: inplace update has side effect on file content - tags: [ inplace_update, workflow ] + doc: inplace update has side effect on file content (using InitialWorkDirRequirement) + tags: [ inplace_update, workflow, initial_work_dir ] output: a: 4 b: 4 @@ -2834,8 +2834,8 @@ - job: tests/empty.json tool: tests/inpdir_update_wf.cwl - doc: inplace update has side effect on directory content - tags: [ inplace_update, workflow ] + doc: inplace update has side effect on directory content (using InitialWorkDirRequirement) + tags: [ inplace_update, workflow, initial_work_dir ] output: { a: [ { @@ -2871,7 +2871,7 @@ listing: [] tags: [ required, command_line_tool ] -- doc: Test that array of input files can be staged to directory with entryname +- doc: Test that array of input files can be staged to directory with InitialWorkDirRequirement entryname id: stage_file_array tool: tests/stage_file_array.cwl job: tests/stage_file_array.job.json @@ -2939,7 +2939,7 @@ ] tags: [ command_line_tool, initial_work_dir, inline_javascript ] -- doc: Test that if array of input files are staged to directory with basename and entryname, entryname overrides +- doc: Test that if array of input files are staged using InitialWorkDirRequirement to directory with basename and entryname, entryname overrides id: stage_file_array_entryname_overrides tool: tests/stage_file_array_basename_and_entryname.cwl job: tests/stage_file_array.job.json @@ -3527,3 +3527,477 @@ an_int: 42 - $import: tests/loop/test-index.yaml + +- job: tests/empty.json + output: {} + tool: tests/networkaccess_noreq.cwl + doc: Test networkAcccess enabled using the networkAccess field + id: networkaccess_noreq + tags: [ command_line_tool, networkaccess ] + +- job: tests/env-job.json + output: + out: + class: File + checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 + location: out + size: 15 + tool: tests/env-tool1_noreq.cwl + id: envvar_noreq + doc: Test CommandLineTool.envDef + tags: [ env_var, command_line_tool ] + +- job: tests/env-job.json + output: + out: + class: File + checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 + location: out + size: 15 + tool: tests/env-wf1_noreq.cwl + id: requirement_priority_noreq + doc: | + Test requirement priority: EnvVarRequirement + CommandLineTool.envDef + tags: [ env_var, workflow ] + +- job: tests/env-job3.yaml + output: + out: + class: File + checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 + location: out + size: 15 + tool: tests/env-tool4_noreq.cwl + id: cwl_envdef_override_expression + doc: Test conflicting requirements in input document via envDef and expression + tags: [ command_line_tool, input_object_requirements, env_var ] + +- job: tests/dir-job.yml + output: { + "outlist": { + "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592", + "size": 20, + "location": "output.txt", + "class": "File" + } + } + tool: tests/dir5_noiwdr.cwl + id: dynamic_initial_workdir_noreq + doc: Test dynamic initialWorkDir + tags: [ shell_command, initial_work_dir, command_line_tool ] + +- job: tests/initialworkdirrequirement-docker-out-job.json + output: + OUTPUT: + "checksum": "sha1$aeb3d11bdf536511649129f4077d5cda6a324118" + "location": "ref.fasta" + "secondaryFiles": [{ + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "ref.fasta.fai", + "class": "File", + "size": 0 + }] + "class": "File" + "size": 12010 + tool: tests/initialworkdir-glob-fullpath_noiwdr.cwl + id: initial_workdir_output_glob_noreq + doc: Test full path glob output of initialWorkDir + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/wc-job.json + output: {} + tool: tests/initialwork-path_noiwdr.cwl + id: initialworkpath_output_noreq + doc: Test that file path in $(inputs) for initialWorkDir is in $(outdir). + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/string-job.json + output: + out: + class: File + checksum: sha1$6a47aa22b2a9d13a66a24b3ee5eaed95ce4753cf + location: example.conf + size: 16 + tool: tests/iwdr-entry_noreq.cwl + id: initial_workdir_trailingnl_noreq + doc: Test if trailing newline is present in file entry in initialWorkDir + tags: [ initial_work_dir, command_line_tool ] + +- doc: Test initialWorkDir with a nested directory structure from another step + job: tests/empty.json + output: + ya_empty: + class: File + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: ya + size: 0 + tool: tests/iwdr_with_nested_dirs_noreq.cwl + id: initialworkdir_nesteddir_noreq + tags: [ initial_work_dir, workflow ] + +- job: tests/arguments-job.yml + output: + classfile: + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: Hello.class + class: File + size: 0 + tool: tests/linkfile_noreq.cwl + id: initial_workdir_expr_noreq + doc: Test expression in initialWorkDir + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/recursive-input-directory.yml + output: + output_dir: { + "basename": "work_dir", + "class": "Directory", + "listing": [ + { + "basename": "a", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "work_dir/a", + "size": 0 + }, + { + "basename": "b", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "work_dir/b", + "size": 0 + }, + { + "basename": "c", + "class": "Directory", + "listing": [ + { + "basename": "d", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "work_dir/c/d", + "size": 0 + } + ], + "location": "work_dir/c", + }, + { + "basename": "e", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "work_dir/e", + "size": 0 + }, + ], + "location": "work_dir", + } + test_result: { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "output.txt", + "size": 0 + } + tool: tests/recursive-input-directory_noiwdr.cwl + id: input_dir_recurs_copy_writable_noiwdr + doc: Test if a writable initialWorkDir input directory is recursively copied and writable + tags: [ initial_work_dir, shell_command, command_line_tool ] + +- job: tests/rename-job.json + output: + outfile: + class: File + checksum: sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376 + location: fish.txt + size: 1111 + tool: tests/rename_noreq.cwl + id: rename_noreq + doc: >- + Test initialWorkDir with expression in filename. + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/search-job.json + output: + outfile: + class: File + checksum: sha1$e2dc9daaef945ac15f01c238ed2f1660f60909a0 + location: result.txt + size: 142 + indexedfile: { + "location": "input.txt", + "class": "File", + "checksum": "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376", + "secondaryFiles": [ + { + "location": "input.txt.idx1", + "class": "File", + "checksum": "sha1$1f6fe811644355974cdd06d9eb695d6e859f3b44", + "size": 1500 + }, + { + "location": "input.idx2", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "location": "input.txt.idx3", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "location": "input.txt.idx4", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "location": "input.txt.idx5", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "location": "input.idx6.txt", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "input.txt.idx7", + "size": 0 + }, + { + "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b", + "class": "File", + "location": "hello.txt", + "size": 13 + }, + { + "class": "Directory", + "listing": [{ + "basename": "index", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "index", + "size": 0 + }], + "location": "input.txt_idx8", + } + ], + "size": 1111 + } + tool: "tests/search_noiwdr.cwl#main" + id: initial_workdir_secondary_files_expr_noiwdr + doc: >- + Test initialWorkDir linking input files and capturing secondaryFiles + on input and output. Also tests the use of a variety of parameter references + and expressions in the secondaryFiles field. + tags: [ initial_work_dir, inline_javascript, workflow ] + +- job: tests/stage-array-job.json + tool: tests/stage-array_noiwdr.cwl + doc: Test null and array input in initialWorkDir + output: + "output": { + "basename": "lsout", + "checksum": "sha1$e453d26efd859a9abc80ae1a1d9d63db72376053", + "class": "File", + "location": "lsout", + "size": 32 + } + tags: [ resource, command_line_tool, initial_work_dir ] + id: initial_work_dir_for_null_and_arrays_noreq + +- job: tests/stage-array-dirs-job.yml + tool: tests/stage-array-dirs_noreq.cwl + doc: Test array of directories initialWorkDir + output: { + "output": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "a", + "size": 0 + }, + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "B", + "size": 0 + } + ] + } + tags: [ resource, command_line_tool, initial_work_dir ] + id: initial_work_dir_for_array_dirs_noreq + +- doc: Test that array of input files can be staged to directory with initialWorkDir.entryname + id: stage_file_array_noiwdr + tool: tests/stage_file_array_noiwdr.cwl + job: tests/stage_file_array.job.json + output: + output: + [ + { + "basename": "sfa-1.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-1.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + }, + { + "basename": "sfa-2.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-2.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + } + ] + tags: [ command_line_tool, initial_work_dir, inline_javascript ] + +- doc: Test that if array of input files are staged using initialWorkDir to a directory with basename and entryname, entryname overrides + id: stage_file_array_entryname_overrides_noiwdr + tool: tests/stage_file_array_basename_and_entryname_noiwdr.cwl + job: tests/stage_file_array.job.json + output: + output: + [ + { + "basename": "sfa-1.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-1.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + }, + { + "basename": "sfa-2.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-2.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + } + ] + tags: [ command_line_tool, initial_work_dir, inline_javascript ] + +- job: tests/stagefile-job.yml + output: { + "outfile": { + "checksum": "sha1$b769c7b2e316edd4b5eb2d24799b2c1f9d8c86e6", + "size": 1111, + "location": "bob.txt", + "class": "File" + } + } + tool: tests/stagefile_noreq.cwl + id: writable_stagedfiles_noreq + doc: Test writable staged files using initialWorkDir. + tags: [ initial_work_dir, command_line_tool ] + +- output: + "foo": { + "checksum": "sha1$63da67422622fbf9251a046d7a34b7ea0fd4fead", + "class": "File", + "location": "foo.txt", + "size": 22 + } + job: tests/cat-job.json + tool: tests/template-tool_noiwdr.cwl + id: initworkdir_expreng_requirements_noiwdr + doc: Test initialWorkDir ExpressionEngineRequirement.engineConfig feature + tags: [ initial_work_dir, inline_javascript, command_line_tool ] + +- job: tests/empty.json + tool: tests/inpdir_update_wf_noiwdr.cwl + doc: inplace update has side effect on directory content (using initialWorkDir) + tags: [ inplace_update, workflow, initial_work_dir ] + output: { + a: [ + { + "basename": "blurb", + "class": "File", + "location": "blurb" + } + ], + b: [ + { + "basename": "blurb", + "class": "File", + "location": "blurb" + } + ] + } + id: modify_directory_content_noiwdr + +- job: tests/empty.json + tool: tests/inp_update_wf_noiwdr.cwl + doc: inplace update has side effect on file content (using initialWorkDir) + tags: [ inplace_update, workflow, initial_work_dir ] + output: + a: 4 + b: 4 + id: modify_file_content + +- job: tests/empty.json + output: + out: { + "basename": "emptyWritableDir", + "listing": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "basename": "blurg", + "location": "blurg", + "class": "File", + "size": 0 + } + ], + "location": "emptyWritableDir", + "class": "Directory" + } + tool: tests/writable-dir_noiwdr.cwl + id: initial_workdir_empty_writable_noiwdr + doc: Test empty writable dir with initialWorkDir + tags: [ inline_javascript, initial_work_dir, command_line_tool ] + + +- job: tests/empty.json + output: + out: { + "basename": "emptyWritableDir", + "listing": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "basename": "blurg", + "location": "blurg", + "class": "File", + "size": 0 + } + ], + "location": "emptyWritableDir", + "class": "Directory" + } + tool: tests/writable-dir-docker_noiwdr.cwl + id: initial_workdir_empty_writable_docker_noiwdr + doc: Test empty writable dir with initialWorkDir inside Docker + tags: [ inline_javascript, initial_work_dir, command_line_tool ] diff --git a/tests/dir5_noiwdr.cwl b/tests/dir5_noiwdr.cwl new file mode 100644 index 0000000..485b90e --- /dev/null +++ b/tests/dir5_noiwdr.cwl @@ -0,0 +1,18 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + - class: ShellCommandRequirement +initialWorkDir: $(inputs.indir.listing) +inputs: + indir: + type: Directory + loadListing: shallow_listing +outputs: + outlist: + type: File + outputBinding: + glob: output.txt +arguments: ["find", "-L", ".", "!", "-path", "*.txt", + {shellQuote: false, valueFrom: "|"}, + "sort"] +stdout: output.txt diff --git a/tests/env-tool1_noreq.cwl b/tests/env-tool1_noreq.cwl new file mode 100644 index 0000000..4551536 --- /dev/null +++ b/tests/env-tool1_noreq.cwl @@ -0,0 +1,16 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +inputs: + in: string +outputs: + out: + type: File + outputBinding: + glob: out + +envDef: + TEST_ENV: $(inputs.in) + +baseCommand: ["/bin/sh", "-c", "echo $TEST_ENV"] + +stdout: out diff --git a/tests/env-tool4_noreq.cwl b/tests/env-tool4_noreq.cwl new file mode 100644 index 0000000..8393cb0 --- /dev/null +++ b/tests/env-tool4_noreq.cwl @@ -0,0 +1,16 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +inputs: + in: string +outputs: + out: + type: File + outputBinding: + glob: out + +envDef: + TEST_ENV: conflict_original + +baseCommand: ["/bin/bash", "-c", "echo $TEST_ENV"] + +stdout: out diff --git a/tests/env-wf1_noreq.cwl b/tests/env-wf1_noreq.cwl new file mode 100644 index 0000000..da1c032 --- /dev/null +++ b/tests/env-wf1_noreq.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner +class: Workflow +cwlVersion: v1.3.0-dev1 + +inputs: + in: string + +outputs: + out: + type: File + outputSource: step1/out + +requirements: + EnvVarRequirement: + envDef: + TEST_ENV: override + +steps: + step1: + run: env-tool1_noreq.cwl + in: + in: in + out: [out] diff --git a/tests/initialwork-path_noiwdr.cwl b/tests/initialwork-path_noiwdr.cwl new file mode 100644 index 0000000..e89d86f --- /dev/null +++ b/tests/initialwork-path_noiwdr.cwl @@ -0,0 +1,14 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + ShellCommandRequirement: {} +initialWorkDir: + - entry: $(inputs.file1) + entryname: bob.txt +inputs: + file1: File +outputs: [] +arguments: + - shellQuote: false + valueFrom: | + test "$(inputs.file1.path)" = "$(runtime.outdir)/bob.txt" diff --git a/tests/initialworkdir-glob-fullpath_noiwdr.cwl b/tests/initialworkdir-glob-fullpath_noiwdr.cwl new file mode 100644 index 0000000..7b8c0a7 --- /dev/null +++ b/tests/initialworkdir-glob-fullpath_noiwdr.cwl @@ -0,0 +1,26 @@ +#!/usr/bin/env cwl-runner + +cwlVersion: v1.3.0-dev1 + +initialWorkDir: + - $(inputs.INPUT) + +class: CommandLineTool + +inputs: + - id: INPUT + type: File + +outputs: + - id: OUTPUT + type: File + outputBinding: + glob: $(runtime.outdir)/$(inputs.INPUT.basename) + secondaryFiles: + - .fai + +arguments: + - valueFrom: $(inputs.INPUT.basename).fai + position: 0 + +baseCommand: [touch] diff --git a/tests/initialworkdirrequirement-docker-out_noiwdr.cwl b/tests/initialworkdirrequirement-docker-out_noiwdr.cwl new file mode 100644 index 0000000..7898665 --- /dev/null +++ b/tests/initialworkdirrequirement-docker-out_noiwdr.cwl @@ -0,0 +1,30 @@ +#!/usr/bin/env cwl-runner + +cwlVersion: v1.3.0-dev1 + +requirements: + - class: DockerRequirement + dockerPull: docker.io/debian:stable-slim + +initialWorkDir: + - $(inputs.INPUT) + +class: CommandLineTool + +inputs: + - id: INPUT + type: File + +outputs: + - id: OUTPUT + type: File + outputBinding: + glob: $(inputs.INPUT.basename) + secondaryFiles: + - .fai + +arguments: + - valueFrom: $(inputs.INPUT.basename).fai + position: 0 + +baseCommand: [touch] diff --git a/tests/inp_update_wf_noiwdr.cwl b/tests/inp_update_wf_noiwdr.cwl new file mode 100644 index 0000000..fe86e5c --- /dev/null +++ b/tests/inp_update_wf_noiwdr.cwl @@ -0,0 +1,33 @@ +#!/usr/bin/env cwl-runner +class: Workflow +cwlVersion: v1.3.0-dev1 +inputs: [] +outputs: + a: + type: int + outputSource: step3/output + b: + type: int + outputSource: step4/output +steps: + step1: + in: + in: {default: "3"} + out: [out] + run: echo-file-tool.cwl + step2: + in: + r: step1/out + out: [out] + run: updateval_inplace_noiwdr.cwl + step3: + in: + file1: step1/out + wait: step2/out + out: [output] + run: parseInt-tool.cwl + step4: + in: + file1: step2/out + out: [output] + run: parseInt-tool.cwl diff --git a/tests/inpdir_update_wf_noiwdr.cwl b/tests/inpdir_update_wf_noiwdr.cwl new file mode 100644 index 0000000..2e539a4 --- /dev/null +++ b/tests/inpdir_update_wf_noiwdr.cwl @@ -0,0 +1,42 @@ +#!/usr/bin/env cwl-runner +class: Workflow +cwlVersion: v1.3.0-dev1 +inputs: [] +requirements: + InlineJavascriptRequirement: {} +outputs: + a: + type: File[] + outputSource: step3/d1out + b: + type: File[] + outputSource: step3/d2out +steps: + step1: + in: + dirname: {default: step1dir} + out: [out] + run: mkdir.cwl + step2: + in: + r: step1/out + out: [out] + run: updatedir_inplace_noiwdr.cwl + step3: + in: + d1: step1/out + d2: step2/out + out: [d1out, d2out] + run: + class: ExpressionTool + inputs: + d1: + type: Directory + loadListing: shallow_listing + d2: + type: Directory + loadListing: shallow_listing + outputs: + d1out: File[] + d2out: File[] + expression: "$({d1out: inputs.d1.listing, d2out: inputs.d2.listing})" diff --git a/tests/iwd/iwd-container-entryname1_noiwdr.cwl b/tests/iwd/iwd-container-entryname1_noiwdr.cwl new file mode 100644 index 0000000..3b861e6 --- /dev/null +++ b/tests/iwd/iwd-container-entryname1_noiwdr.cwl @@ -0,0 +1,27 @@ +cwlVersion: v1.3.0-dev1 +class: CommandLineTool +doc: | + When executing in a container, entryname can have an absolute path + to a mount location inside the container. + +inputs: + filelist: File + +outputs: + head: + type: File + outputBinding: + glob: head.txt + +requirements: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim + dockerOutputDirectory: /output + ShellCommandRequirement: {} + +initialWorkDir: + - entryname: /tmp2j3y7rpb/input/stuff.txt # Give it a weird prefix to minimize chance of conflict with a real file + entry: $(inputs.filelist) + +arguments: + - {shellQuote: false, valueFrom: "head -n10 /tmp2j3y7rpb/input/stuff.txt > /output/head.txt"} diff --git a/tests/iwd/iwd-container-entryname2_noiwdr.cwl b/tests/iwd/iwd-container-entryname2_noiwdr.cwl new file mode 100644 index 0000000..b06a72c --- /dev/null +++ b/tests/iwd/iwd-container-entryname2_noiwdr.cwl @@ -0,0 +1,21 @@ +cwlVersion: v1.3.0-dev1 +class: CommandLineTool +doc: | + Must fail if entryname is an absolute path and DockerRequirement is + not in the 'requirements' section. +inputs: + filelist: File +outputs: + head: + type: File + outputBinding: + glob: head.txt +requirements: + ShellCommandRequirement: {} + +initialWorkDir: + - entryname: /tmp2j3y7rpb/input/stuff.txt # Give it a weird prefix to minimize chance of conflict with a real file + entry: $(inputs.filelist) + +arguments: + - {shellQuote: false, valueFrom: "head -n10 /tmp2j3y7rpb/input/stuff.txt > /output/head.txt"} diff --git a/tests/iwd/iwd-container-entryname3_noiwdr.cwl b/tests/iwd/iwd-container-entryname3_noiwdr.cwl new file mode 100644 index 0000000..4e63e48 --- /dev/null +++ b/tests/iwd/iwd-container-entryname3_noiwdr.cwl @@ -0,0 +1,24 @@ +cwlVersion: v1.3.0-dev1 +class: CommandLineTool +doc: | + Must fail if entryname is an absolute path and DockerRequirement is + not in the 'requirements' section. +inputs: + filelist: File +outputs: + head: + type: File + outputBinding: + glob: head.txt +hints: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim + dockerOutputDirectory: /output +requirements: + ShellCommandRequirement: {} + +initialWorkDir: + - entryname: /tmp2j3y7rpb/input/stuff.txt # Give it a weird prefix to minimize chance of conflict with a real file + entry: $(inputs.filelist) +arguments: + - {shellQuote: false, valueFrom: "head -n10 /tmp2j3y7rpb/input/stuff.txt > /output/head.txt"} diff --git a/tests/iwd/iwd-container-entryname4_noiwdr.cwl b/tests/iwd/iwd-container-entryname4_noiwdr.cwl new file mode 100644 index 0000000..1f866d9 --- /dev/null +++ b/tests/iwd/iwd-container-entryname4_noiwdr.cwl @@ -0,0 +1,21 @@ +cwlVersion: v1.3.0-dev1 +class: CommandLineTool +doc: | + Must fail if entryname starts with ../ +inputs: + filelist: File +outputs: + head: + type: File + outputBinding: + glob: head.txt + +initialWorkDir: + - entryname: ../input/stuff.txt # Illegal + entry: $(inputs.filelist) + +requirements: + ShellCommandRequirement: {} + +arguments: + - {shellQuote: false, valueFrom: "head -n10 ../input/stuff.txt > head.txt"} diff --git a/tests/iwd/iwd-fileobjs1_noiwdr.cwl b/tests/iwd/iwd-fileobjs1_noiwdr.cwl new file mode 100644 index 0000000..3f87a30 --- /dev/null +++ b/tests/iwd/iwd-fileobjs1_noiwdr.cwl @@ -0,0 +1,20 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +doc: Can have a file declared directly in an initialWorkDir +initialWorkDir: + - class: File + location: ../loadContents/inp-filelist.txt + - class: Directory + location: ../testdir +inputs: [] +outputs: + filelist: + type: File + outputBinding: + glob: inp-filelist.txt + testdir: + type: Directory + outputBinding: + glob: testdir +baseCommand: "true" diff --git a/tests/iwd/iwd-fileobjs2_noiwdr.cwl b/tests/iwd/iwd-fileobjs2_noiwdr.cwl new file mode 100644 index 0000000..6aef62b --- /dev/null +++ b/tests/iwd/iwd-fileobjs2_noiwdr.cwl @@ -0,0 +1,18 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +doc: Can have a file declared directly in initialWorkDir +initialWorkDir: + - [{class: File, location: ../loadContents/inp-filelist.txt}, + {class: Directory, location: ../testdir}] +inputs: [] +outputs: + filelist: + type: File + outputBinding: + glob: inp-filelist.txt + testdir: + type: Directory + outputBinding: + glob: testdir +baseCommand: "true" diff --git a/tests/iwd/iwd-jsondump1-nl_noiwdr.cwl b/tests/iwd/iwd-jsondump1-nl_noiwdr.cwl new file mode 100644 index 0000000..68d1596 --- /dev/null +++ b/tests/iwd/iwd-jsondump1-nl_noiwdr.cwl @@ -0,0 +1,22 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +initialWorkDir: + - entryname: "out-filelist.json" + entry: | + ${ + var ls = []; + for (var i = 1; i < 10000; i++) { + ls.push("example_input_file"+i+".txt"); + } + return ls; + } +requirements: + InlineJavascriptRequirement: {} +inputs: [] +outputs: + filelist: + type: File + outputBinding: + glob: out-filelist.json +arguments: ["true"] diff --git a/tests/iwd/iwd-jsondump1_noiwdr.cwl b/tests/iwd/iwd-jsondump1_noiwdr.cwl new file mode 100644 index 0000000..c0e560b --- /dev/null +++ b/tests/iwd/iwd-jsondump1_noiwdr.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + InlineJavascriptRequirement: {} + +initialWorkDir: + - entryname: "out-filelist.json" + entry: |- + ${ + var ls = []; + for (var i = 1; i < 10000; i++) { + ls.push("example_input_file"+i+".txt"); + } + return ls; + } +inputs: [] +outputs: + filelist: + type: File + outputBinding: + glob: out-filelist.json +arguments: ["true"] diff --git a/tests/iwd/iwd-jsondump2-nl_noiwdr.cwl b/tests/iwd/iwd-jsondump2-nl_noiwdr.cwl new file mode 100644 index 0000000..b69800c --- /dev/null +++ b/tests/iwd/iwd-jsondump2-nl_noiwdr.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + InlineJavascriptRequirement: {} + +initialWorkDir: + - entryname: "out-filelist.json" + entry: | + ${ + var ls = []; + for (var i = 1; i < 10000; i++) { + ls.push("example_input_file"+i+".txt"); + } + return {"filelist": ls}; + } +inputs: [] +outputs: + filelist: + type: File + outputBinding: + glob: out-filelist.json +arguments: ["true"] diff --git a/tests/iwd/iwd-jsondump2_noiwdr.cwl b/tests/iwd/iwd-jsondump2_noiwdr.cwl new file mode 100644 index 0000000..84d7220 --- /dev/null +++ b/tests/iwd/iwd-jsondump2_noiwdr.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + InlineJavascriptRequirement: {} + +initialWorkDir: + - entryname: "out-filelist.json" + entry: |- + ${ + var ls = []; + for (var i = 1; i < 10000; i++) { + ls.push("example_input_file"+i+".txt"); + } + return {"filelist": ls}; + } +inputs: [] +outputs: + filelist: + type: File + outputBinding: + glob: out-filelist.json +arguments: ["true"] diff --git a/tests/iwd/iwd-jsondump3-nl_noiwdr.cwl b/tests/iwd/iwd-jsondump3-nl_noiwdr.cwl new file mode 100644 index 0000000..4d92a00 --- /dev/null +++ b/tests/iwd/iwd-jsondump3-nl_noiwdr.cwl @@ -0,0 +1,19 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + InlineJavascriptRequirement: {} + +initialWorkDir: + - entryname: "out-number.json" + entry: | + ${ + return 44.4; + } +inputs: [] +outputs: + filelist: + type: File + outputBinding: + glob: out-number.json +arguments: ["true"] diff --git a/tests/iwd/iwd-jsondump3_noiwdr.cwl b/tests/iwd/iwd-jsondump3_noiwdr.cwl new file mode 100644 index 0000000..24a679b --- /dev/null +++ b/tests/iwd/iwd-jsondump3_noiwdr.cwl @@ -0,0 +1,19 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + InlineJavascriptRequirement: {} + +initialWorkDir: + - entryname: "out-number.json" + entry: |- + ${ + return 44.4; + } +inputs: [] +outputs: + filelist: + type: File + outputBinding: + glob: out-number.json +arguments: ["true"] diff --git a/tests/iwd/iwd-nolimit_noiwdr.cwl b/tests/iwd/iwd-nolimit_noiwdr.cwl new file mode 100644 index 0000000..0394af1 --- /dev/null +++ b/tests/iwd/iwd-nolimit_noiwdr.cwl @@ -0,0 +1,24 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + InlineJavascriptRequirement: {} + + +initialWorkDir: + - entryname: "out-filelist.txt" + entry: |- + ${ + var ls = ""; + for (var i = 1; i < 10000; i++) { + ls += "example_input_file"+i+".txt\n"; + } + return ls; + } +inputs: [] +outputs: + filelist: + type: File + outputBinding: + glob: out-filelist.txt +baseCommand: "true" diff --git a/tests/iwd/iwd-passthrough1_noreq.cwl b/tests/iwd/iwd-passthrough1_noreq.cwl new file mode 100644 index 0000000..1a9c0b1 --- /dev/null +++ b/tests/iwd/iwd-passthrough1_noreq.cwl @@ -0,0 +1,22 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +doc: | + YAML |- syntax does not add trailing newline so in the listing entry + below there is no whitespace surrounding the value + $(inputs.filelist), so it is evaluated as a File object. Compare to + iwd-passthrough2.cwl + +initialWorkDir: + - entryname: "renamed-filelist.txt" + entry: |- + $(inputs.filelist) +inputs: + filelist: + type: File +outputs: + filelist: + type: File + outputBinding: + glob: renamed-filelist.txt +baseCommand: "true" diff --git a/tests/iwd/iwd-passthrough2_noiwdr.cwl b/tests/iwd/iwd-passthrough2_noiwdr.cwl new file mode 100644 index 0000000..4b9653c --- /dev/null +++ b/tests/iwd/iwd-passthrough2_noiwdr.cwl @@ -0,0 +1,29 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +doc: | + YAML | syntax adds a trailing newline, so in the listing entry + below, it becomes a string interpolation -- it evaluates to a string + consisting of $(inputs.filelist) serialized to JSON and then adding + a newline. Compare to iwd-passthrough1.cwl + +requirements: + DockerRequirement: + dockerPull: docker.io/python:3-slim + +initialWorkDir: + - entryname: "renamed-filelist.txt" + entry: | + $(inputs.filelist) + +inputs: + filelist: File + script: + type: File + default: + class: File + location: loadit.py +stdout: out.txt +outputs: + out: stdout +arguments: [python3, $(inputs.script), "renamed-filelist.txt"] diff --git a/tests/iwd/iwd-passthrough3_noreq.cwl b/tests/iwd/iwd-passthrough3_noreq.cwl new file mode 100644 index 0000000..9997859 --- /dev/null +++ b/tests/iwd/iwd-passthrough3_noreq.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +initialWorkDir: + - entryname: "renamed-filelist.txt" + entry: $(inputs.filelist) +inputs: + filelist: + type: File +outputs: + filelist: + type: File + outputBinding: + glob: renamed-filelist.txt +baseCommand: "true" diff --git a/tests/iwd/iwd-passthrough4_noreq.cwl b/tests/iwd/iwd-passthrough4_noreq.cwl new file mode 100644 index 0000000..7fff53f --- /dev/null +++ b/tests/iwd/iwd-passthrough4_noreq.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +initialWorkDir: + - $(inputs.filelist) +inputs: + filelist: + type: File +outputs: + filelist: + type: File + outputBinding: + glob: $(inputs.filelist.basename) +baseCommand: "true" diff --git a/tests/iwd/iwd-passthrough5_noreq.cwl b/tests/iwd/iwd-passthrough5_noreq.cwl new file mode 100644 index 0000000..2ac858f --- /dev/null +++ b/tests/iwd/iwd-passthrough5_noreq.cwl @@ -0,0 +1,15 @@ +class: CommandLineTool +cwlVersion: v1.2 +doc: Given a directory, emit it unchanged +initialWorkDir: + - entry: $(inputs.testdir) + writable: false +inputs: + testdir: + type: Directory +outputs: + testdir: + type: Directory + outputBinding: + glob: $(inputs.testdir.basename) +baseCommand: "true" diff --git a/tests/iwd/iwd-subdir-tool_noreq.cwl b/tests/iwd/iwd-subdir-tool_noreq.cwl new file mode 100644 index 0000000..819ce5f --- /dev/null +++ b/tests/iwd/iwd-subdir-tool_noreq.cwl @@ -0,0 +1,15 @@ +class: CommandLineTool +cwlVersion: v1.2 +doc: Given an input directory, emit a subdirectory as output +initialWorkDir: + - entry: $(inputs.testdir) + writable: false +inputs: + testdir: + type: Directory +outputs: + testdir: + type: Directory + outputBinding: + glob: testdir/c +baseCommand: "true" diff --git a/tests/iwd/iwd-subdir-wf_noreq.cwl b/tests/iwd/iwd-subdir-wf_noreq.cwl new file mode 100644 index 0000000..a169e51 --- /dev/null +++ b/tests/iwd/iwd-subdir-wf_noreq.cwl @@ -0,0 +1,19 @@ +class: Workflow +cwlVersion: v1.2 +inputs: + testdir: Directory +outputs: + out: + type: Directory + outputSource: step2/testdir +steps: + step1: + run: iwd-subdir-tool_noreq.cwl + in: + testdir: testdir + out: [testdir] + step2: + run: iwd-passthrough5_noreq.cwl + in: + testdir: step1/testdir + out: [testdir] diff --git a/tests/iwd/iwd_dir_literal_real_file.cwl b/tests/iwd/iwd_dir_literal_real_file.cwl new file mode 100644 index 0000000..27458e7 --- /dev/null +++ b/tests/iwd/iwd_dir_literal_real_file.cwl @@ -0,0 +1,25 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 + +requirements: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim + InlineJavascriptRequirement: {} + +initialWorkDir: | + ${ + return [{"class": "Directory", + "basename": "subdir", + "listing": [ inputs.filelist ] + }]} + +inputs: + filelist: File + +outputs: + same: + type: File + outputBinding: + glob: subdir/$(inputs.filelist.basename) + +baseCommand: echo diff --git a/tests/iwd/test-index.yaml b/tests/iwd/test-index.yaml index d39f474..34ee8df 100644 --- a/tests/iwd/test-index.yaml +++ b/tests/iwd/test-index.yaml @@ -9,7 +9,7 @@ "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", "size": 268866, } - doc: "Test that InitialWorkDir contents can be bigger than 64k" + doc: "Test that InitialWorkDirRequirement contents can be bigger than 64k" tags: [ initial_work_dir, command_line_tool ] - job: null @@ -23,7 +23,7 @@ "checksum": "sha1$5bbeb2a75327927cb97d7e9716c8299682001b36", "size": 298863 } - doc: "Test dump object to JSON in InitialWorkDir file contents, no trailing newline" + doc: "Test dump object to JSON in InitialWorkDirRequirement file contents, no trailing newline" tags: [ initial_work_dir, command_line_tool ] - job: null @@ -37,7 +37,7 @@ "checksum": "sha1$7307f027449371b3642c1f7c32124218af0e41b5", "size": 298864 } - doc: "Test dump object to JSON in InitialWorkDir file contents, with trailing newline" + doc: "Test dump object to JSON in InitialWorkDirRequirement file contents, with trailing newline" tags: [ initial_work_dir, command_line_tool ] - job: null @@ -51,7 +51,7 @@ "checksum": "sha1$a0e2225d47c9ed2f07e7633d00dd19d1cbf65c9f", "size": 298877, } - doc: "Test array to JSON in InitialWorkDir file contents, no trailing newline" + doc: "Test array to JSON in InitialWorkDirRequirement file contents, no trailing newline" tags: [ initial_work_dir, command_line_tool ] - job: null @@ -65,7 +65,7 @@ "checksum": "sha1$750c95cb45561cd1d863506f82a1a75fffd53a54", "size": 298878, } - doc: "Test array to JSON in InitialWorkDir file contents, with trailing newline" + doc: "Test array to JSON in InitialWorkDirRequirement file contents, with trailing newline" tags: [ initial_work_dir, command_line_tool ] - job: null @@ -79,7 +79,7 @@ "checksum": "sha1$356b190d3274c960b34c2c9538023dda438d67d4", "size": 4, } - doc: "Test number to JSON in InitialWorkDir file contents, no trailing newline" + doc: "Test number to JSON in InitialWorkDirRequirement file contents, no trailing newline" tags: [ initial_work_dir, command_line_tool ] - job: null @@ -93,7 +93,7 @@ "checksum": "sha1$0b02e6b07d199025bfdcfc6b9830d550a0a6bde9", "size": 5, } - doc: "Test number to JSON in InitialWorkDir file contents, with trailing newline" + doc: "Test number to JSON in InitialWorkDirRequirement file contents, with trailing newline" tags: [ initial_work_dir, command_line_tool ] - job: ../loadContents/input.yml @@ -107,7 +107,7 @@ "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", "size": 268866, } - doc: "Test InitialWorkDir file passthrough" + doc: "Test InitialWorkDirRequirement file passthrough" tags: [ initial_work_dir, command_line_tool ] - job: ../loadContents/input.yml @@ -121,7 +121,7 @@ "checksum": "sha1$406e83b1cd694780f1b3ea4fe8fbb754511fe3f7", "size": 49, } - doc: "Test InitialWorkDir file object is serialized to json" + doc: "Test InitialWorkDirRequirement file object is serialized to json" tags: [ initial_work_dir, command_line_tool ] - job: ../loadContents/input.yml @@ -135,7 +135,7 @@ "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", "size": 268866, } - doc: "Test InitialWorkDir file object is passed through" + doc: "Test InitialWorkDirRequirement file object is passed through" tags: [ initial_work_dir, command_line_tool ] - job: ../loadContents/input.yml @@ -149,7 +149,7 @@ "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", "size": 268866, } - doc: "Test InitialWorkDir file object is passed through" + doc: "Test InitialWorkDirRequirement file object is passed through" tags: [ initial_work_dir, command_line_tool ] - job: null @@ -198,7 +198,7 @@ } ], } - doc: "Test File and Directory object in listing" + doc: "Test File and Directory object in InitialWorkDirRequirement.listing" tags: [ initial_work_dir, command_line_tool ] - job: null @@ -247,7 +247,7 @@ } ], } - doc: "Test File and Directory object in listing" + doc: "Test File and Directory object in InitialWorkDirRequirement.listing" tags: [ initial_work_dir, command_line_tool ] - job: ../loadContents/input.yml @@ -268,27 +268,27 @@ id: iwd-container-entryname2 tool: iwd-container-entryname2.cwl should_fail: true - doc: "Test input mount locations when no container (should fail)" + doc: "Test InitialWorkDirRequirement input mount locations when no container (should fail)" tags: [ initial_work_dir, command_line_tool ] - job: ../loadContents/input.yml id: iwd-container-entryname3 tool: iwd-container-entryname3.cwl should_fail: true - doc: "Test input mount locations when container is a hint (should fail)" + doc: "Test InitialWorkDirRequirement input mount locations when container is a hint (should fail)" tags: [ initial_work_dir, command_line_tool ] - job: ../loadContents/input.yml id: iwd-container-entryname4 tool: iwd-container-entryname4.cwl should_fail: true - doc: "Must fail if entryname starts with ../" + doc: "Must fail if InitialWorkDirRequirement.entryname starts with ../" tags: [ initial_work_dir, command_line_tool ] - job: ../loadContents/input.yml id: iwdr_dir_literal_real_file tool: iwdr_dir_literal_real_file.cwl - doc: "Test directory literal containing a real file" + doc: "Test InitialWorkDirRequirement directory literal containing a real file" output: same: "location": "inp-filelist.txt" @@ -301,7 +301,7 @@ - job: iwd-subdir-job.yml id: iwd-subdir tool: iwd-subdir-wf.cwl - doc: "Test emitting a subdirectory from initial workdir" + doc: "Test emitting a subdirectory from InitialWorkDirRequirement" output: "out": { "location": "c", @@ -318,3 +318,324 @@ ] } tags: [ initial_work_dir, command_line_tool, workflow ] + +- job: ../loadContents/input.yml + id: iwd-container-entryname1_noreq + tool: iwd-container-entryname1_noiwdr.cwl + output: + "head": { + "location": "head.txt", + "basename": "head.txt", + "class": "File", + "checksum": "sha1$8b5071fa49953fcdb8729b16345b7c894b493f83", + "size": 241 + } + doc: "Test input mount locations when container required" + tags: [ initial_work_dir, command_line_tool ] + +- job: ../loadContents/input.yml + id: iwd-container-entryname2_noreq + tool: iwd-container-entryname2_noiwdr.cwl + should_fail: true + doc: "Test initialWorkDir input mount locations when no container (should fail)" + tags: [ initial_work_dir, command_line_tool ] + +- job: ../loadContents/input.yml + id: iwd-container-entryname3_noreq + tool: iwd-container-entryname3_noiwdr.cwl + should_fail: true + doc: "Test initialWorkDir input mount locations when container is a hint (should fail)" + tags: [ initial_work_dir, command_line_tool ] + +- job: ../loadContents/input.yml + id: iwd-container-entryname4_noreq + tool: iwd-container-entryname4_noiwdr.cwl + should_fail: true + doc: "Must fail if initialWorkDir.entryname starts with ../" + tags: [ initial_work_dir, command_line_tool ] + +- job: null + id: iwd-fileobjs1_noreq + tool: iwd-fileobjs1_noiwdr.cwl + output: + "filelist": { + "location": "inp-filelist.txt", + "basename": "inp-filelist.txt", + "class": "File", + "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", + "size": 268866, + } + "testdir": { + "location": "testdir", + "basename": "testdir", + "class": "Directory", + "listing": [ + { + "class": "File", + "location": "b", + "basename": "b", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0, + }, + { + "class": "Directory", + "location": "c", + "basename": "c", + "listing": [ + { + "class": "File", + "location": "d", + "basename": "d", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0, + } + ], + }, + { + "class": "File", + "location": "a", + "basename": "a", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0, + } + ], + } + doc: "Test File and Directory object in initialWorkDir" + tags: [ initial_work_dir, command_line_tool ] + +- job: null + id: iwd-fileobjs2_noreq + tool: iwd-fileobjs2_noiwdr.cwl + output: + "filelist": { + "location": "inp-filelist.txt", + "basename": "inp-filelist.txt", + "class": "File", + "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", + "size": 268866, + } + "testdir": { + "location": "testdir", + "basename": "testdir", + "class": "Directory", + "listing": [ + { + "class": "File", + "location": "b", + "basename": "b", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0, + }, + { + "class": "Directory", + "location": "c", + "basename": "c", + "listing": [ + { + "class": "File", + "location": "d", + "basename": "d", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0, + } + ], + }, + { + "class": "File", + "location": "a", + "basename": "a", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0, + } + ], + } + doc: "Test File and Directory object in initialWorkDir" + tags: [ initial_work_dir, command_line_tool ] + +- job: null + id: iwd-jsondump1_noreq + tool: iwd-jsondump1_noiwdr.cwl + output: + "filelist": { + "location": "out-filelist.json", + "basename": "out-filelist.json", + "class": "File", + "checksum": "sha1$5bbeb2a75327927cb97d7e9716c8299682001b36", + "size": 298863 + } + doc: "Test dump object to JSON in initialWorkDir file contents, no trailing newline" + tags: [ initial_work_dir, command_line_tool ] + +- job: null + id: iwd-jsondump1-nl_noreq + tool: iwd-jsondump1-nl_noiwdr.cwl + output: + "filelist": { + "location": "out-filelist.json", + "basename": "out-filelist.json", + "class": "File", + "checksum": "sha1$7307f027449371b3642c1f7c32124218af0e41b5", + "size": 298864 + } + doc: "Test dump object to JSON in initialWorkDir file contents, with trailing newline" + tags: [ initial_work_dir, command_line_tool ] + +- job: null + id: iwd-jsondump2-noreq + tool: iwd-jsondump2_noiwdr.cwl + output: + "filelist": { + "location": "out-filelist.json", + "basename": "out-filelist.json", + "class": "File", + "checksum": "sha1$a0e2225d47c9ed2f07e7633d00dd19d1cbf65c9f", + "size": 298877, + } + doc: "Test array to JSON in initialWorkDir file contents, no trailing newline" + tags: [ initial_work_dir, command_line_tool ] + +- job: null + id: iwd-jsondump2-nl-noreq + tool: iwd-jsondump2-nl_noiwdr.cwl + output: + "filelist": { + "location": "out-filelist.json", + "basename": "out-filelist.json", + "class": "File", + "checksum": "sha1$750c95cb45561cd1d863506f82a1a75fffd53a54", + "size": 298878, + } + doc: "Test array to JSON in initialWorkDir file contents, with trailing newline" + tags: [ initial_work_dir, command_line_tool ] + +- job: null + id: iwd-jsondump3-noreq + tool: iwd-jsondump3_noiwdr.cwl + output: + "filelist": { + "location": "out-number.json", + "basename": "out-number.json", + "class": "File", + "checksum": "sha1$356b190d3274c960b34c2c9538023dda438d67d4", + "size": 4, + } + doc: "Test number to JSON in initialWorkDir file contents, no trailing newline" + tags: [ initial_work_dir, command_line_tool ] + +- job: null + id: iwd-jsondump3-nl-noreq + tool: iwd-jsondump3-nl_noiwdr.cwl + output: + "filelist": { + "location": "out-number.json", + "basename": "out-number.json", + "class": "File", + "checksum": "sha1$0b02e6b07d199025bfdcfc6b9830d550a0a6bde9", + "size": 5, + } + doc: "Test number to JSON in initialWorkDir file contents, with trailing newline" + tags: [ initial_work_dir, command_line_tool ] + +- job: null + id: iwd-nolimit-noreq + tool: iwd-nolimit_noiwdr.cwl + output: + "filelist": { + "location": "out-filelist.txt", + "basename": "out-filelist.txt", + "class": "File", + "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", + "size": 268866, + } + doc: "Test that initialWorkDir contents can be bigger than 64k" + tags: [ initial_work_dir, command_line_tool ] + +- job: ../loadContents/input.yml + id: iwd-passthrough1-noreq + tool: iwd-passthrough1_noreq.cwl + output: + "filelist": { + "location": "renamed-filelist.txt", + "basename": "renamed-filelist.txt", + "class": "File", + "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", + "size": 268866, + } + doc: "Test initialWorkDir file passthrough" + tags: [ initial_work_dir, command_line_tool ] + +- job: ../loadContents/input.yml + id: iwd-passthrough2-noreq + tool: iwd-passthrough2_noiwdr.cwl + output: + "out": { + "location": "out.txt", + "basename": "out.txt", + "class": "File", + "checksum": "sha1$406e83b1cd694780f1b3ea4fe8fbb754511fe3f7", + "size": 49, + } + doc: "Test initialWorkDir file object is serialized to json" + tags: [ initial_work_dir, command_line_tool ] + +- job: ../loadContents/input.yml + id: iwd-passthrough3-noreq + tool: iwd-passthrough3_noreq.cwl + output: + "filelist": { + "location": "renamed-filelist.txt", + "basename": "renamed-filelist.txt", + "class": "File", + "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", + "size": 268866, + } + doc: "Test initialWorkDir file object is passed through" + tags: [ initial_work_dir, command_line_tool ] + +- job: ../loadContents/input.yml + id: iwd-passthrough4-noreq + tool: iwd-passthrough4_noreq.cwl + output: + "filelist": { + "location": "inp-filelist.txt", + "basename": "inp-filelist.txt", + "class": "File", + "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c", + "size": 268866, + } + doc: "Test initialWorkDir file object is passed through" + tags: [ initial_work_dir, command_line_tool ] + +- job: iwd-subdir-job.yml + id: iwd-subdir-noreq + tool: iwd-subdir-wf_noreq.cwl + doc: "Test emitting a subdirectory from initialWorkDir" + output: + "out": { + "location": "c", + "basename": "c", + "class": "Directory", + "listing": [ + { + "location": "d", + "basename": "d", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + } + ] + } + tags: [ initial_work_dir, command_line_tool, workflow ] + +- job: ../loadContents/input.yml + id: iwd_dir_literal_real_file + tool: iwd_dir_literal_real_file.cwl + doc: "Test initialWorkDir directory literal containing a real file" + output: + same: + "location": "inp-filelist.txt" + "basename": "inp-filelist.txt" + "class": "File" + "checksum": "sha1$57f77b36009332d236b52b4beca77301b503b27c" + "size": 268866 + tags: [ initial_work_dir, command_line_tool ] diff --git a/tests/iwdr-entry_noreq.cwl b/tests/iwdr-entry_noreq.cwl new file mode 100644 index 0000000..6895789 --- /dev/null +++ b/tests/iwdr-entry_noreq.cwl @@ -0,0 +1,18 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +baseCommand: ["cat", "example.conf"] + +initialWorkDir: + - entryname: example.conf + entry: | + CONFIGVAR=$(inputs.message) + +inputs: + message: string +outputs: + out: + type: File + outputBinding: + glob: example.conf diff --git a/tests/iwdr_with_nested_dirs_noreq.cwl b/tests/iwdr_with_nested_dirs_noreq.cwl new file mode 100644 index 0000000..24cd769 --- /dev/null +++ b/tests/iwdr_with_nested_dirs_noreq.cwl @@ -0,0 +1,39 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow + +inputs: [] +outputs: + ya_empty: + type: File + outputSource: second/ya + +steps: + first: + run: + class: CommandLineTool + baseCommand: [ mkdir, -p, deeply/nested/dir/structure ] + inputs: [] + outputs: + deep_dir: + type: Directory + outputBinding: { glob: deeply } + in: {} + out: [ deep_dir ] + + second: + run: + class: CommandLineTool + baseCommand: [ touch, deeply/nested/dir/structure/ya ] + initialWorkDir: + - entry: $(inputs.dir) + writable: true + inputs: + dir: Directory + outputs: + ya: + type: File + outputBinding: { glob: deeply/nested/dir/structure/ya } + + in: { dir: first/deep_dir } + out: [ ya ] diff --git a/tests/linkfile_noreq.cwl b/tests/linkfile_noreq.cwl new file mode 100644 index 0000000..78e868b --- /dev/null +++ b/tests/linkfile_noreq.cwl @@ -0,0 +1,20 @@ +cwlVersion: v1.3.0-dev1 +class: CommandLineTool + +initialWorkDir: + - $(inputs.src) + +inputs: + src: + type: File + inputBinding: + position: 1 + valueFrom: $(self.nameroot).class + +baseCommand: touch + +outputs: + classfile: + type: File + outputBinding: + glob: "*.class" diff --git a/tests/networkaccess_noreq.cwl b/tests/networkaccess_noreq.cwl new file mode 100644 index 0000000..0cc14b5 --- /dev/null +++ b/tests/networkaccess_noreq.cwl @@ -0,0 +1,14 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + DockerRequirement: + dockerPull: docker.io/python:3-slim +networkAccess: true +inputs: [] +outputs: [] +baseCommand: python +arguments: + - "-c" + - valueFrom: | + import urllib.request + assert(urllib.request.urlopen("http://commonwl.org").code == 200) diff --git a/tests/recursive-input-directory_noiwdr.cwl b/tests/recursive-input-directory_noiwdr.cwl new file mode 100644 index 0000000..db74ddb --- /dev/null +++ b/tests/recursive-input-directory_noiwdr.cwl @@ -0,0 +1,33 @@ +cwlVersion: v1.3.0-dev1 +class: CommandLineTool +requirements: + ShellCommandRequirement: {} + +initialWorkDir: + - entry: $(inputs.input_dir) + entryname: work_dir + writable: true +stdout: output.txt +arguments: + - shellQuote: false + valueFrom: | + touch work_dir/e; + if [ ! -w work_dir ]; then echo work_dir not writable; fi; + if [ -L work_dir ]; then echo work_dir is a symlink; fi; + if [ ! -w work_dir/a ]; then echo work_dir/a not writable; fi; + if [ -L work_dir/a ]; then echo work_dir/a is a symlink; fi; + if [ ! -w work_dir/c ]; then echo work_dir/c not writable; fi; + if [ -L work_dir/c ]; then echo work_dir/c is a symlink; fi; + if [ ! -w work_dir/c/d ]; then echo work_dir/c/d not writable; fi; + if [ -L work_dir/c/d ]; then echo work_dir/c/d is a symlink; fi; + if [ ! -w work_dir/e ]; then echo work_dir/e not writable; fi; + if [ -L work_dir/e ]; then echo work_dir/e is a symlink ; fi; +inputs: + input_dir: Directory +outputs: + output_dir: + type: Directory + outputBinding: + glob: work_dir + test_result: + type: stdout diff --git a/tests/rename_noreq.cwl b/tests/rename_noreq.cwl new file mode 100644 index 0000000..99deb2d --- /dev/null +++ b/tests/rename_noreq.cwl @@ -0,0 +1,14 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +baseCommand: "true" +initialWorkDir: + - entryname: $(inputs.newname) + entry: $(inputs.srcfile) +inputs: + srcfile: File + newname: string +outputs: + outfile: + type: File + outputBinding: + glob: $(inputs.newname) diff --git a/tests/search_noiwdr.cwl b/tests/search_noiwdr.cwl new file mode 100644 index 0000000..910081f --- /dev/null +++ b/tests/search_noiwdr.cwl @@ -0,0 +1,109 @@ +cwlVersion: v1.3.0-dev1 +$graph: +- id: index + class: CommandLineTool + baseCommand: python + arguments: + - valueFrom: input.txt + position: 1 + initialWorkDir: + - entryname: input.txt + entry: $(inputs.file) + requirements: + - class: InlineJavascriptRequirement + hints: + - class: DockerRequirement + dockerPull: docker.io/python:3-slim + + inputs: + file: File + secondfile: File + index.py: + type: File + default: + class: File + location: index.py + inputBinding: + position: 0 + outputs: + result: + type: File + outputBinding: + glob: input.txt + secondaryFiles: + - ".idx1" + - "^.idx2" + - '$(self.basename).idx3' + - '${ return self.basename+".idx4"; }' + - '$({"path": self.path+".idx5", "class": "File"})' + - '$(self.nameroot).idx6$(self.nameext)' + - '${ return [self.basename+".idx7", inputs.secondfile]; }' + - "_idx8" + +- id: search + class: CommandLineTool + baseCommand: python + requirements: + - class: InlineJavascriptRequirement + hints: + - class: DockerRequirement + dockerPull: docker.io/python:3-slim + inputs: + file: + type: File + inputBinding: + position: 1 + secondaryFiles: + - ".idx1" + - "^.idx2" + - '$(self.basename).idx3' + - '${ return self.basename+".idx4"; }' + - '$(self.nameroot).idx6$(self.nameext)' + - '${ return [self.basename+".idx7"]; }' + - "_idx8" + search.py: + type: File + default: + class: File + location: search.py + inputBinding: + position: 0 + term: + type: string + inputBinding: + position: 2 + outputs: + result: + type: File + outputBinding: + glob: result.txt + stdout: result.txt + +- id: main + class: Workflow + inputs: + infile: File + secondfile: File + term: string + outputs: + outfile: + type: File + outputSource: search/result + indexedfile: + type: File + outputSource: index/result + + steps: + index: + run: "#index" + in: + file: infile + secondfile: secondfile + out: [result] + + search: + run: "#search" + in: + file: index/result + term: term + out: [result] diff --git a/tests/secondaryfiles/rename-outputs_noiwdr.cwl b/tests/secondaryfiles/rename-outputs_noiwdr.cwl new file mode 100644 index 0000000..c11191e --- /dev/null +++ b/tests/secondaryfiles/rename-outputs_noiwdr.cwl @@ -0,0 +1,49 @@ +#!/usr/bin/env cwl-runner +id: OutputSecondaryFileConformanceTest +baseCommand: +- ls +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +doc: | + Simple test to confirm the implementation of expressions returning a File within a CommandOutputParameter.secondaryFile field. + + Related links: + - Issue: https://github.com/common-workflow-language/cwltool/issues/1232 + - PR: https://github.com/common-workflow-language/cwltool/pull/1233 + - Discourse: https://cwl.discourse.group/t/ask-cwl-to-rename-a-secondary-file/72 + +inputs: [] +outputs: +- id: output_file + label: out + outputBinding: + glob: "*.txt" + secondaryFiles: | + ${ + function resolveSecondary(base, secPattern) { + if (secPattern[0] == "^") { + var spl = base.split("."); + var endIndex = spl.length > 1 ? spl.length - 1 : 1; + return resolveSecondary(spl.slice(undefined, endIndex).join("."), secPattern.slice(1)); + } + return base + secPattern; + } + return [ + { + "class": "File", + "path": resolveSecondary(self.path, "^.accessory"), + "basename": resolveSecondary(self.basename, ".accessory") + } + ]; + } + type: File +requirements: + InlineJavascriptRequirement: {} + +initialWorkDir: + - entry: "" + entryname: secondary_file_test.txt + writable: true + - entry: "" + entryname: secondary_file_test.accessory + writable: true diff --git a/tests/secondaryfiles/test-index.yaml b/tests/secondaryfiles/test-index.yaml index 54fb40e..f540dfe 100644 --- a/tests/secondaryfiles/test-index.yaml +++ b/tests/secondaryfiles/test-index.yaml @@ -11,7 +11,7 @@ tags: [ inline_javascript, secondary_files, command_line_tool, shell_command ] - tool: rename-outputs.cwl - doc: Confirm CommandOutputParameter expression can receive a File object + doc: Confirm CommandOutputParameter expression can receive a File object (using InitialWorkDirRequirement) id: command_output_file_expression output: output_file: @@ -24,4 +24,20 @@ checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 location: secondary_file_test.txt.accessory size: 0 - tags: [ inline_javascript, secondary_files, command_line_tool ] + tags: [ inline_javascript, secondary_files, command_line_tool, initial_work_dir] + +- tool: rename-outputs_noiwdr.cwl + doc: Confirm CommandOutputParameter expression can receive a File object (using initialWorkDir) + id: command_output_file_expression_noiwdr + output: + output_file: + class: File + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + size: 0 + location: secondary_file_test.txt + secondaryFiles: + - class: File + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: secondary_file_test.txt.accessory + size: 0 + tags: [ inline_javascript, secondary_files, command_line_tool, initial_work_dir ] diff --git a/tests/stage-array-dirs_noreq.cwl b/tests/stage-array-dirs_noreq.cwl new file mode 100644 index 0000000..9c59459 --- /dev/null +++ b/tests/stage-array-dirs_noreq.cwl @@ -0,0 +1,20 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +id: stage_array_dirs +baseCommand: + - ls +inputs: + - id: input_list + type: Directory[] +outputs: + - id: output + type: File[] + outputBinding: + glob: + - testdir/a + - rec/B +label: stage-array-dirs.cwl +initialWorkDir: + - $(inputs.input_list) diff --git a/tests/stage-array_noiwdr.cwl b/tests/stage-array_noiwdr.cwl new file mode 100644 index 0000000..544d147 --- /dev/null +++ b/tests/stage-array_noiwdr.cwl @@ -0,0 +1,32 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +id: stage_array +arguments: + - {shellQuote: false, valueFrom: "ls | grep -v lsout"} +inputs: + - id: input_file + type: File + - id: optional_file + type: File? + - id: input_list + type: 'File[]' + secondaryFiles: + - ^.tar +stdout: lsout +outputs: + - id: output + type: File? + outputBinding: + glob: lsout +label: stage-array.cwl +initialWorkDir: + - $(inputs.input_file) + - $(inputs.optional_file) + - entry: $(inputs.input_list) + - entry: $(null) + - entryname: a + entry: b +requirements: + - class: ShellCommandRequirement diff --git a/tests/stage_file_array_basename_and_entryname_noiwdr.cwl b/tests/stage_file_array_basename_and_entryname_noiwdr.cwl new file mode 100644 index 0000000..8882b3b --- /dev/null +++ b/tests/stage_file_array_basename_and_entryname_noiwdr.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +id: stage_file_array_basename_and_entryname +label: Stage File Array (with Directory Basename AND entryname) +arguments: [ls] +inputs: + - id: input_list + type: 'File[]' + secondaryFiles: + - .sec +outputs: + - id: output + type: File[] + outputBinding: + glob: input_dir/* +requirements: + - class: InlineJavascriptRequirement + +initialWorkDir: + - entryname: input_dir + entry: "${ return {class: 'Directory', basename: 'not_input_dir', listing: inputs.input_list} }" diff --git a/tests/stage_file_array_noiwdr.cwl b/tests/stage_file_array_noiwdr.cwl new file mode 100644 index 0000000..6aaadfa --- /dev/null +++ b/tests/stage_file_array_noiwdr.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +id: stage_file_array +label: Stage File Array +arguments: [ls] +inputs: + - id: input_list + type: 'File[]' + secondaryFiles: + - .sec +outputs: + - id: output + type: File[] + outputBinding: + glob: input_dir/* +requirements: + - class: InlineJavascriptRequirement + +initialWorkDir: + - entryname: input_dir + entry: "${ return {class: 'Directory', listing: inputs.input_list} }" diff --git a/tests/stagefile_noreq.cwl b/tests/stagefile_noreq.cwl new file mode 100644 index 0000000..0256d29 --- /dev/null +++ b/tests/stagefile_noreq.cwl @@ -0,0 +1,25 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +hints: + - class: DockerRequirement + dockerPull: docker.io/python:3-slim + +initialWorkDir: + - entry: $(inputs.infile) + entryname: bob.txt + writable: true +inputs: + infile: File +outputs: + outfile: + type: File + outputBinding: + glob: bob.txt +baseCommand: "python" +arguments: + - "-c" + - | + f = open("bob.txt", "r+") + f.seek(8) + f.write("Bob. ") + f.close() diff --git a/tests/string-interpolation/bash-dollar-quote_noiwdr.cwl b/tests/string-interpolation/bash-dollar-quote_noiwdr.cwl new file mode 100644 index 0000000..aaeedf9 --- /dev/null +++ b/tests/string-interpolation/bash-dollar-quote_noiwdr.cwl @@ -0,0 +1,56 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + InlineJavascriptRequirement: {} + + +initialWorkDir: + - entryname: script.sh + entry: | + echo \$(echo \ + "My country, 'tis of thee," \ + "Sweet land of liberty" | rev) + echo '$(inputs.val)' # produce val + echo '\$(inputs.val)' # produce $(inputs.val) + echo + echo '\\$(inputs.val)' # produce \val + echo '\\\$(inputs.val)' # produce \$(inputs.val) + echo + echo '\\\\$(inputs.val)' # produce \\val + echo '\\\\\$(inputs.val)' # produce \\$(inputs.val) + echo + echo '\\\\\\$(inputs.val)' # produce \\\val + echo '\\\\\\\$(inputs.val)' # produce \\\$(inputs.val) + echo + echo '\\\\\\\\$(inputs.val)' # produce \\\\val + echo '\\\\\\\\\$(inputs.val)' # produce \\\\$(inputs.val) + echo + echo '\\\\\\\\\\$(inputs.val)' # produce \\\\\val + echo '\\\\\\\\\\\$(inputs.val)' # produce \\\\\$(inputs.val) + echo '\' # produce \ + echo '\\' # produce \ + echo '\\\' # produce \\ + echo '\\\\' # produce \\ + echo '\\\\\' # produce \\\ + echo '\\\\\\' # produce \\\ + echo '\\\\\\\' # produce \\\\ + echo '\\\\\\\\' # produce \\\\ + echo '\\\\\\\\\' # produce \\\\\ + echo '\\\\\\\\\\' # produce \\\\\ + echo '$' # produce $ + echo '\$' # produce \$ + echo '\\$' # produce \$ + echo '$$' # produce $$ + echo '$$$' # produce $$$ + +baseCommand: ["bash", "script.sh"] + +inputs: + val: + type: string + default: "val" + +outputs: + out: stdout + +stdout: out.txt diff --git a/tests/string-interpolation/bash-line-continuation-with-expression_noreq.cwl b/tests/string-interpolation/bash-line-continuation-with-expression_noreq.cwl new file mode 100644 index 0000000..a9aac1f --- /dev/null +++ b/tests/string-interpolation/bash-line-continuation-with-expression_noreq.cwl @@ -0,0 +1,20 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +initialWorkDirReq: + - entryname: script.sh + entry: | + echo \ + "My country, 'tis of thee," \ + "Sweet land of $(inputs.liberty)" + +baseCommand: ["bash", "script.sh"] + +inputs: + liberty: + type: string? + default: liberty + +outputs: + out: stdout + +stdout: out.txt diff --git a/tests/string-interpolation/bash-line-continuation_noreq.cwl b/tests/string-interpolation/bash-line-continuation_noreq.cwl new file mode 100644 index 0000000..13a3bbb --- /dev/null +++ b/tests/string-interpolation/bash-line-continuation_noreq.cwl @@ -0,0 +1,17 @@ +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +initialWorkDir: + - entryname: script.sh + entry: | + echo \ + "My country, 'tis of thee," \ + "Sweet land of liberty" + +baseCommand: ["bash", "script.sh"] + +inputs: [] + +outputs: + out: stdout + +stdout: out.txt diff --git a/tests/string-interpolation/js-quote_noiwdr.cwl b/tests/string-interpolation/js-quote_noiwdr.cwl new file mode 100644 index 0000000..40f75d9 --- /dev/null +++ b/tests/string-interpolation/js-quote_noiwdr.cwl @@ -0,0 +1,36 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +label: Check for a JS quoting bug + +requirements: + InlineJavascriptRequirement: {} + ResourceRequirement: + coresMin: 2 + ramMin: 1000 + + +initialWorkDir: + - entryname: file.txt + entry: | + ${return 'quote "' + inputs.quote + '"'} + - entryname: script.sh + entry: | + set -xe + cat file.txt + +inputs: + quote: + type: string + default: Hello + +outputs: + out: + type: File + outputBinding: + glob: file.txt + +baseCommand: +- echo +arguments: [] diff --git a/tests/string-interpolation/test-index.yaml b/tests/string-interpolation/test-index.yaml index cc6e2b3..8607888 100644 --- a/tests/string-interpolation/test-index.yaml +++ b/tests/string-interpolation/test-index.yaml @@ -1,7 +1,7 @@ # Run with cwltest --test test-index.yaml --tool toil-cwl-runner -- --enable-dev - id: continuation - doc: Line continuations in bash scripts should behave correctly + doc: Line continuations in bash scripts should behave correctly (using InitialWorkDirRequirement) tool: bash-line-continuation.cwl output: out: { @@ -10,10 +10,10 @@ "location": "out.txt", "size": 48 } - tags: [ inline_javascript, command_line_tool ] + tags: [ inline_javascript, command_line_tool, initial_work_dir ] - id: continuation_expression - doc: Line continuations in bash scripts should always behave correctly + doc: Line continuations in bash scripts should always behave correctly (using InitialWorkDirRequirement) tool: bash-line-continuation-with-expression.cwl output: out: { @@ -22,10 +22,10 @@ "location": "out.txt", "size": 48 } - tags: [ inline_javascript, command_line_tool ] + tags: [ inline_javascript, command_line_tool, initial_work_dir ] - id: quoting_multiple_backslashes - doc: Test quoting multiple backslashes + doc: Test quoting multiple backslashes using InitialWorkDirRequirement tool: bash-dollar-quote.cwl output: out: { @@ -34,7 +34,7 @@ "location": "out.txt", "size": 246 } - tags: [ inline_javascript, command_line_tool ] + tags: [ inline_javascript, command_line_tool, initial_work_dir ] - id: escaping_expression_no_extra_quotes doc: Strings returned from JS expressions should not have extra quotes around them @@ -46,4 +46,52 @@ "location": "file.txt", "size": 14 } - tags: [ inline_javascript, command_line_tool ] + tags: [ inline_javascript, command_line_tool, initial_work_dir ] + +- id: continuation_noiwdr + doc: Line continuations in bash scripts should behave correctly (using initialWorkDir) + tool: bash-line-continuation_noreq.cwl + output: + out: { + "checksum": "sha1$47d8510dce768c907f4dea6bcaf90f8d59cb265c", + "class": "File", + "location": "out.txt", + "size": 48 + } + tags: [ inline_javascript, command_line_tool, initial_work_dir ] + +- id: continuation_expression_noiwdr + doc: Line continuations in bash scripts should always behave correctly (using initialWorkDir) + tool: bash-line-continuation-with-expression_noreq.cwl + output: + out: { + "checksum": "sha1$47d8510dce768c907f4dea6bcaf90f8d59cb265c", + "class": "File", + "location": "out.txt", + "size": 48 + } + tags: [ inline_javascript, command_line_tool, initial_work_dir ] + +- id: quoting_multiple_backslashes_noiwdr + doc: Test quoting multiple backslashes using initialWorkDir + tool: bash-dollar-quote_noiwdr.cwl + output: + out: { + "checksum": "sha1$acfdc38aef5354c03b976cbb6d9f7d08a179951d", + "class": "File", + "location": "out.txt", + "size": 246 + } + tags: [ inline_javascript, command_line_tool, initial_work_dir ] + +- id: escaping_expression_no_extra_quotes_noiwdr + doc: Strings returned from JS expressions should not have extra quotes around them (using initialWorkDir) + tool: js-quote_noiwdr.cwl + output: + out: { + "checksum": "sha1$726e9e616f278d9028b4a870653b01c125c2fc89", + "class": "File", + "location": "file.txt", + "size": 14 + } + tags: [ inline_javascript, command_line_tool, initial_work_dir ] diff --git a/tests/template-tool_noiwdr.cwl b/tests/template-tool_noiwdr.cwl new file mode 100755 index 0000000..9fb94a7 --- /dev/null +++ b/tests/template-tool_noiwdr.cwl @@ -0,0 +1,25 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: CommandLineTool +requirements: + - class: InlineJavascriptRequirement + expressionLib: + - { $include: underscore.js } + - "var t = function(s) { return _.template(s, {variable: 'data'})({'inputs': inputs}); };" + +initialWorkDir: + - entryname: foo.txt + entry: $(t("The file is <%= data.inputs.file1.path.split('/').slice(-1)[0] %>\n")) + +hints: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim +inputs: + - id: file1 + type: File +outputs: + - id: foo + type: File + outputBinding: + glob: foo.txt +baseCommand: [cat, foo.txt] diff --git a/tests/updatedir_inplace_noiwdr.cwl b/tests/updatedir_inplace_noiwdr.cwl new file mode 100755 index 0000000..7270c15 --- /dev/null +++ b/tests/updatedir_inplace_noiwdr.cwl @@ -0,0 +1,20 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + InplaceUpdateRequirement: + inplaceUpdate: true + +initialWorkDir: + - entry: $(inputs.r) + entryname: inp + writable: true + +inputs: + r: Directory +outputs: + out: + type: Directory + outputBinding: + glob: inp +arguments: [touch, inp/blurb] diff --git a/tests/updateval_inplace_noiwdr.cwl b/tests/updateval_inplace_noiwdr.cwl new file mode 100755 index 0000000..d0c5cfc --- /dev/null +++ b/tests/updateval_inplace_noiwdr.cwl @@ -0,0 +1,27 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.3.0-dev1 +requirements: + InplaceUpdateRequirement: + inplaceUpdate: true + +initialWorkDir: + - entry: $(inputs.r) + writable: true + +hints: + DockerRequirement: + dockerPull: docker.io/python:3-slim +inputs: + r: File + script: + type: File + default: + class: File + location: updateval.py +outputs: + out: + type: File + outputBinding: + glob: $(inputs.r.basename) +arguments: [python, $(inputs.script), $(inputs.r.basename)] diff --git a/tests/writable-dir-docker_noiwdr.cwl b/tests/writable-dir-docker_noiwdr.cwl new file mode 100644 index 0000000..4af55cf --- /dev/null +++ b/tests/writable-dir-docker_noiwdr.cwl @@ -0,0 +1,23 @@ +cwlVersion: v1.3.0-dev1 + +class: CommandLineTool + +requirements: + - class: InlineJavascriptRequirement + +initialWorkDir: + - entryname: emptyWritableDir + entry: "$({class: 'Directory', listing: []})" + writable: true + +hints: + - class: DockerRequirement + dockerPull: docker.io/alpine:latest + +inputs: [] +outputs: + out: + type: Directory + outputBinding: + glob: emptyWritableDir +arguments: [touch, emptyWritableDir/blurg] diff --git a/tests/writable-dir_noiwdr.cwl b/tests/writable-dir_noiwdr.cwl new file mode 100644 index 0000000..ae6ce7f --- /dev/null +++ b/tests/writable-dir_noiwdr.cwl @@ -0,0 +1,17 @@ +cwlVersion: v1.3.0-dev1 +class: CommandLineTool +requirements: + InlineJavascriptRequirement: {} + +initialWorkDir: + - entryname: emptyWritableDir + writable: true + entry: "$({class: 'Directory', listing: []})" + +inputs: [] +outputs: + out: + type: Directory + outputBinding: + glob: emptyWritableDir +arguments: [touch, emptyWritableDir/blurg] From 99de554d49fb9ee244329f9750b6d49e0c4c5ea6 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 19 Jul 2024 08:41:25 -0400 Subject: [PATCH 12/12] fix typo --- CommandLineTool.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CommandLineTool.yml b/CommandLineTool.yml index 09cd1d6..31e72af 100644 --- a/CommandLineTool.yml +++ b/CommandLineTool.yml @@ -54,14 +54,14 @@ $graph: See also the [CWL Workflow Description, v1.3.0-dev1 changelog](Workflow.html#Changelog). * New field `networkAccess` on - [CommandLineTool](#CommandLineTool). This supercedes the + [CommandLineTool](#CommandLineTool). This supersedes the [NetworkAccess](#NetworkAccess) requirement, which is now deprecated. * New field `envDef` on [CommandLineTool](#CommandLineTool). - This supercedes the [EnvVarRequirement](#EnvVarRequirement) + This supersedes the [EnvVarRequirement](#EnvVarRequirement) requirement, which is now deprecated. * New field `initialWorkDir` on - [CommandLineTool](#CommandLineTool). This supercedes the + [CommandLineTool](#CommandLineTool). This supersedes the [InitialWorkDirRequirement](#InitialWorkDirRequirement) requirement, which is now deprecated.