diff --git a/.gitignore b/.gitignore index 68652fd..6122aee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ __pycache__ +.DS_Store -# cromwell jar files +# cromwell jar files, etc. +cromwell-executions +cromwell-workflow-logs *.jar +pytest-wdl-tests/tests/literals/run.sh +pytest-wdl-tests/tests/literals/validate.sh +pytest-wdl-tests/tests/literals/json.sh # python virtual environment -env/ \ No newline at end of file +env/ diff --git a/pytest-wdl-tests/tests/literals/data/array_literals_out b/pytest-wdl-tests/tests/literals/data/array_literals_out new file mode 100644 index 0000000..62b4fc4 --- /dev/null +++ b/pytest-wdl-tests/tests/literals/data/array_literals_out @@ -0,0 +1,7 @@ +INTS +0 +1 +2 +STRINGS +array +literal diff --git a/pytest-wdl-tests/tests/literals/data/map_literals_out b/pytest-wdl-tests/tests/literals/data/map_literals_out new file mode 100644 index 0000000..6168376 --- /dev/null +++ b/pytest-wdl-tests/tests/literals/data/map_literals_out @@ -0,0 +1,6 @@ +INTS +1 10 +2 11 +MIXED +a 1 +b 2 \ No newline at end of file diff --git a/pytest-wdl-tests/tests/literals/data/object_literals_out b/pytest-wdl-tests/tests/literals/data/object_literals_out new file mode 100644 index 0000000..293fbdd --- /dev/null +++ b/pytest-wdl-tests/tests/literals/data/object_literals_out @@ -0,0 +1,2 @@ +a b +10 11 diff --git a/pytest-wdl-tests/tests/literals/data/object_map_coercion_out b/pytest-wdl-tests/tests/literals/data/object_map_coercion_out new file mode 100644 index 0000000..4b2ed1d --- /dev/null +++ b/pytest-wdl-tests/tests/literals/data/object_map_coercion_out @@ -0,0 +1,6 @@ +OBJECT_SYNTAX +key1 key2 key3 +7 8 9 +MAP_COERCION +key1name key2name key3name +10 11 12 diff --git a/pytest-wdl-tests/tests/literals/data/pair_literals_out b/pytest-wdl-tests/tests/literals/data/pair_literals_out new file mode 100644 index 0000000..f636470 --- /dev/null +++ b/pytest-wdl-tests/tests/literals/data/pair_literals_out @@ -0,0 +1 @@ +23 twenty-three diff --git a/pytest-wdl-tests/tests/literals/literals.wdl b/pytest-wdl-tests/tests/literals/literals.wdl new file mode 100644 index 0000000..88decaa --- /dev/null +++ b/pytest-wdl-tests/tests/literals/literals.wdl @@ -0,0 +1,172 @@ +# This workflow tests the the literals sections of the WDL 1.0 Spec +# It also requires the use of some functions such as write_lines(), write_object(), write_map(), and stdout() +# write_map() is slightly broken as it does not print a \n after writing like the related functions do. + +version 1.0 + +workflow literals { + input { + ### Array Literals + Array[String] array_literal_strings = ["array", "literal"] + Array[Int] array_literal_ints = [0, 1, 2] + + ### Map Literals + Map[Int, Int] map_literal_ints = {1: 10, 2: 11} + Map[String, Int] map_literal_mixed = {"a": 1, "b": 2} + + ### Object Literals + Object object_literal = object { + a: 10, + b: 11 + } + + ### Object Coercion from Map + # "object" keyword before {} indicates Object, absence of indicates Map. + String key1 = "key1name" + String key2 = "key2name" + String key3 = "key3name" + + Object object_syntax = object { + key1: 7, + key2: 8, + key3: 9 + } + + Object map_coercion = { + key1: 10, + key2: 11, + key3: 12 + } + + ### Pair Literals + Pair[Int, String] pair_literal_mixed = (23, "twenty-three") + } + call array_literals { + input: + array_literal_strings=write_lines(array_literal_strings), + array_literal_ints=write_lines(array_literal_ints) + } + call map_literals { + input: + map_literal_ints=write_map(map_literal_ints), + map_literal_mixed=write_map(map_literal_mixed) + } + call object_literals { + input: + object_literal = write_object(object_literal) + } + call object_map_coercion { + input: + object_syntax=write_object(object_syntax), + map_coercion=write_object(map_coercion) + } + call pair_literals { + input: + pair_literal_mixed_left=pair_literal_mixed.left, + pair_literal_mixed_right=pair_literal_mixed.right + } + output { + File array_literals_out = array_literals.array_literals_out + File map_literals_out = map_literals.map_literals_out + File object_literals_out = object_literals.object_literals_out + File object_map_coercion_out = object_map_coercion.object_map_coercion_out + File pair_literals_out = pair_literals.pair_literals_out + } +} + +task array_literals { + input { + File array_literal_ints + File array_literal_strings + } + command { + set -e + echo INTS + cat ~{array_literal_ints} + printf "STRINGS\n" + cat ~{array_literal_strings} + } + output { + File array_literals_out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task map_literals { + input { + File map_literal_ints + File map_literal_mixed + } + command { + set -e + + echo INTS + cat ~{map_literal_ints} + printf "\nMIXED\n" # Need a newline following the first written map + cat ~{map_literal_mixed} + } + output { + File map_literals_out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task object_literals { + input { + File object_literal + } + command { + set -e + + cat ~{object_literal} + } + output { + File object_literals_out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task object_map_coercion { + input { + File object_syntax + File map_coercion + } + command { + set -e + + echo OBJECT_SYNTAX + cat ~{object_syntax} + printf "MAP_COERCION\n" + cat ~{map_coercion} + } + output { + File object_map_coercion_out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task pair_literals { + input { + Int pair_literal_mixed_left + String pair_literal_mixed_right + } + command { + set -e + + echo ~{pair_literal_mixed_left} ~{pair_literal_mixed_right} + } + output { + File pair_literals_out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} \ No newline at end of file diff --git a/pytest-wdl-tests/tests/literals/test_data.json b/pytest-wdl-tests/tests/literals/test_data.json new file mode 100644 index 0000000..7a73a41 --- /dev/null +++ b/pytest-wdl-tests/tests/literals/test_data.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/pytest-wdl-tests/tests/literals/test_literals.py b/pytest-wdl-tests/tests/literals/test_literals.py new file mode 100644 index 0000000..c6922da --- /dev/null +++ b/pytest-wdl-tests/tests/literals/test_literals.py @@ -0,0 +1,4 @@ + +def test_literals(workflow_data, workflow_runner): + expected = workflow_data.get_dict("array_literals_out", "map_literals_out", "object_literals_out", "object_map_coercion_out", "pair_literals_out") + workflow_runner("literals.wdl", {}, expected, workflow_name="literals") \ No newline at end of file diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/basic_array.wdl b/pytest-wdl-tests/tests/stdlib/basic_array/basic_array.wdl new file mode 100644 index 0000000..2ff4378 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/basic_array.wdl @@ -0,0 +1,164 @@ +version 1.0 + +workflow basic_array_functions { + input { + File tsv + Int num = 3 # range(num) --> [0, 1, 2] + Array[Int] array_1 = [0, 1, 2] + Array[Int] array_2 = [3, 4, 5] + Array[String] array_3 = ['a', 'b', 'c'] + Array[String] array_4 = ['d', 'e'] + } + # tsv is read into a 2D array before it is fed to transpose and flatten + Array[Array[String]] array_2d = read_tsv(tsv) + + call range { + input: array_range=range(num) + } + call transpose { + input: array_transpose=transpose(array_2d) + } + scatter (pair in zip(array_1, array_3)) { + call zip { + input: + pair_left=pair.left, + pair_right=pair.right + } + } + scatter (pair in cross(array_1, array_4)) { + call cross { + input: + pair_left=pair.left, + pair_right=pair.right + } + } + call length { + input: + array_1_length=length(array_1), + array_4_length=length(array_4) + } + call flatten { + input: array_flatten=flatten(array_2d) + } + call prefix { + input: array_prefix=prefix("prefix_", array_3) + } + output { + File range_out = range.out + File transpose_out = transpose.out + Array[File] zip_out = zip.out + Array[File] cross_out = cross.out + File length_out = length.out + File flatten_out = flatten.out + File prefix_out = prefix.out + } +} + +task range { + input { + Array[Int] array_range + } + command { + cat ~{write_lines(array_range)} + } + output { + File out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task transpose { + input { + Array[Array[String]] array_transpose + } + command { + cat ~{write_tsv(array_transpose)} + } + output { + File out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task zip { + input { + Int pair_left + String pair_right + } + command { + echo ~{pair_left} ~{pair_right} + } + output { + File out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task cross { + input { + Int pair_left + String pair_right + } + command { + echo ~{pair_left} ~{pair_right} + } + output { + File out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task length { + input { + Int array_1_length + Int array_4_length + } + command { + echo ~{array_1_length} # 3 + echo ~{array_4_length} # 2 + } + output { + File out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task flatten { + input { + Array[String] array_flatten + } + command { + cat ~{write_lines(array_flatten)} + } + output { + File out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} + +task prefix { + input { + Array[String] array_prefix + } + command { + cat ~{write_lines(array_prefix)} + } + output { + File out = stdout() + } + runtime { + docker: "ubuntu:latest" + } +} \ No newline at end of file diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/colors.tsv b/pytest-wdl-tests/tests/stdlib/basic_array/data/colors.tsv new file mode 100644 index 0000000..b9d747d --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/colors.tsv @@ -0,0 +1,2 @@ +red blue yellow +purple green orange \ No newline at end of file diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-0_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-0_out new file mode 100644 index 0000000..2c40181 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-0_out @@ -0,0 +1 @@ +0 d diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-1_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-1_out new file mode 100644 index 0000000..6bc62f5 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-1_out @@ -0,0 +1 @@ +0 e diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-2_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-2_out new file mode 100644 index 0000000..40623fc --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-2_out @@ -0,0 +1 @@ +1 d diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-3_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-3_out new file mode 100644 index 0000000..1cd0813 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-3_out @@ -0,0 +1 @@ +1 e diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-4_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-4_out new file mode 100644 index 0000000..14e32a8 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-4_out @@ -0,0 +1 @@ +2 d diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-5_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-5_out new file mode 100644 index 0000000..33f5d5e --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/cross_shard-5_out @@ -0,0 +1 @@ +2 e diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/flatten_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/flatten_out new file mode 100644 index 0000000..1afa570 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/flatten_out @@ -0,0 +1,6 @@ +red +blue +yellow +purple +green +orange diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/length_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/length_out new file mode 100644 index 0000000..f1e5eee --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/length_out @@ -0,0 +1,2 @@ +3 +2 diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/prefix_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/prefix_out new file mode 100644 index 0000000..4d8fa43 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/prefix_out @@ -0,0 +1,3 @@ +prefix_a +prefix_b +prefix_c diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/range_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/range_out new file mode 100644 index 0000000..4539bbf --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/range_out @@ -0,0 +1,3 @@ +0 +1 +2 diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/transpose_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/transpose_out new file mode 100644 index 0000000..5b6ecfd --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/transpose_out @@ -0,0 +1,3 @@ +red purple +blue green +yellow orange diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/zip_shard-1_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/zip_shard-1_out new file mode 100644 index 0000000..bacfc40 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/zip_shard-1_out @@ -0,0 +1 @@ +1 b diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/zip_shard-2_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/zip_shard-2_out new file mode 100644 index 0000000..c422c8a --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/zip_shard-2_out @@ -0,0 +1 @@ +2 c diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/data/zip_shard_0_out b/pytest-wdl-tests/tests/stdlib/basic_array/data/zip_shard_0_out new file mode 100644 index 0000000..9adb254 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/data/zip_shard_0_out @@ -0,0 +1 @@ +0 a diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/test_basic_array_functions.py b/pytest-wdl-tests/tests/stdlib/basic_array/test_basic_array_functions.py new file mode 100644 index 0000000..4471cf9 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/test_basic_array_functions.py @@ -0,0 +1,23 @@ +def test_basic_array_functions(workflow_data, workflow_runner): + inputs = {"tsv":workflow_data["colors.tsv"]} + expected = { + "range_out":workflow_data["range_out"], + "transpose_out":workflow_data["transpose_out"], + "zip_out": [ + workflow_data["zip_shard_0_out"], + workflow_data["zip_shard-1_out"], + workflow_data["zip_shard-2_out"] + ], + "cross_out": [ + workflow_data["cross_shard-0_out"], + workflow_data["cross_shard-1_out"], + workflow_data["cross_shard-2_out"], + workflow_data["cross_shard-3_out"], + workflow_data["cross_shard-4_out"], + workflow_data["cross_shard-5_out"] + ], + "length_out":workflow_data["length_out"], + "flatten_out":workflow_data["flatten_out"], + "prefix_out":workflow_data["prefix_out"] + } + workflow_runner("basic_array.wdl", inputs, expected, workflow_name="basic_array_functions") \ No newline at end of file diff --git a/pytest-wdl-tests/tests/stdlib/basic_array/test_data.json b/pytest-wdl-tests/tests/stdlib/basic_array/test_data.json new file mode 100644 index 0000000..eaf2444 --- /dev/null +++ b/pytest-wdl-tests/tests/stdlib/basic_array/test_data.json @@ -0,0 +1,3 @@ +{ + "basic_array_functions.tsv": "data/colors.tsv" +} \ No newline at end of file diff --git a/pytest-wdl-tests/tests/whitespace/data/spaces_out b/pytest-wdl-tests/tests/whitespace/data/spaces_out new file mode 100644 index 0000000..ab07670 --- /dev/null +++ b/pytest-wdl-tests/tests/whitespace/data/spaces_out @@ -0,0 +1,2 @@ +I am a document. I am all spaces. Let me pass! +Look now I'm in this python heredoc! I am a document. I am all spaces. Let me pass! diff --git a/pytest-wdl-tests/tests/whitespace/data/tabs_out b/pytest-wdl-tests/tests/whitespace/data/tabs_out new file mode 100644 index 0000000..ff866e8 --- /dev/null +++ b/pytest-wdl-tests/tests/whitespace/data/tabs_out @@ -0,0 +1,2 @@ +I am a document. I am all tabs. Let me pass! +Look now I'm in this python heredoc! I am a document. I am all tabs. Let me pass! diff --git a/pytest-wdl-tests/tests/whitespace/mixed.wdl b/pytest-wdl-tests/tests/whitespace/mixed.wdl new file mode 100644 index 0000000..bd90925 --- /dev/null +++ b/pytest-wdl-tests/tests/whitespace/mixed.wdl @@ -0,0 +1,34 @@ +# This workflow has 2-space indents but also few sneaky tabs! It is intended to fail miserably. + +version 1.0 + +workflow mixed { + input { + String message = "I am a document. I am mixed. Let me fail!" + } + call t { + input: message=message + } + output { + File mixed_out = t.out + } +} + +task t { + input { + String message + } + command { + echo ~{message} + python <