From 207cf6d9933f177c7c2b44b66dd69931b09d24b3 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 11 Feb 2019 10:12:58 -0500 Subject: [PATCH 1/5] Handle form parameters in POST that are not attachments --- wes_service/arvados_wes.py | 7 ++++--- wes_service/util.py | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/wes_service/arvados_wes.py b/wes_service/arvados_wes.py index 59444e6..293d7e8 100644 --- a/wes_service/arvados_wes.py +++ b/wes_service/arvados_wes.py @@ -129,7 +129,7 @@ def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params, }) try: - with tempfile.NamedTemporaryFile(dir=tempdir, suffix=".json") as inputtemp: + with tempfile.NamedTemporaryFile("wt", dir=tempdir, suffix=".json") as inputtemp: json.dump(workflow_params, inputtemp) inputtemp.flush() @@ -163,7 +163,7 @@ def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params, if proc.returncode != 0: api.container_requests().update(uuid=cr_uuid, body={"priority": 0}).execute() - self.log_for_run(cr_uuid, stderrdata, env['ARVADOS_API_TOKEN']) + self.log_for_run(cr_uuid, stderrdata.decode("utf-8"), env['ARVADOS_API_TOKEN']) if tempdir: shutil.rmtree(tempdir) @@ -212,7 +212,8 @@ def RunWorkflow(self, **args): tempdir)).start() except Exception as e: - self.log_for_run(cr["uuid"], str(e)) + logging.exception("Error") + self.log_for_run(cr["uuid"], "An exception ocurred while handling your request: " + str(e)) cr = api.container_requests().update(uuid=cr["uuid"], body={"container_request": {"priority": 0}}).execute() diff --git a/wes_service/util.py b/wes_service/util.py index dd23687..baaa6ea 100644 --- a/wes_service/util.py +++ b/wes_service/util.py @@ -68,10 +68,18 @@ def collect_attachments(self, run_id=None): body[k] = json.loads(content.decode("utf-8")) else: body[k] = v.read().decode() + for k, ls in iterlists(connexion.request.form): + for v in ls: + if k in ("workflow_params", "tags", "workflow_engine_parameters"): + body[k] = json.loads(v) + else: + body[k] = v - if ":" not in body["workflow_url"]: - body["workflow_url"] = "file://%s" % os.path.join(tempdir, secure_filename(body["workflow_url"])) - - self.log_for_run(run_id, "Using workflow_url '%s'" % body.get("workflow_url")) + if "workflow_url" in body: + if ":" not in body["workflow_url"]: + body["workflow_url"] = "file://%s" % os.path.join(tempdir, secure_filename(body["workflow_url"])) + self.log_for_run(run_id, "Using workflow_url '%s'" % body.get("workflow_url")) + else: + raise Exception("Missing 'workflow_url' in submission") return tempdir, body From b687441b0fcf4837298dd9fe3891f5e36c77c821 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 11 Feb 2019 11:07:34 -0500 Subject: [PATCH 2/5] Move cwltool so it isn't a core dependency --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 660115d..1b41ae1 100644 --- a/setup.py +++ b/setup.py @@ -26,8 +26,7 @@ 'future', 'connexion==1.4.2', 'ruamel.yaml >= 0.12.4, <= 0.15.77', - 'cwlref-runner==1.0', - 'schema-salad >= 3.0, < 3.1', + 'schema-salad', 'subprocess32==3.5.2' ], entry_points={ @@ -35,6 +34,7 @@ "wes-client=wes_client.wes_client_main:main"] }, extras_require={ + "cwltool": ['cwlref-runner'], "arvados": ["arvados-cwl-runner" ], "toil": ["toil[all]==3.18.0" From 5c306e1c2e5c59cab7dbb750241992bdc10d80b1 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 11 Feb 2019 11:13:01 -0500 Subject: [PATCH 3/5] cwltool test uses runner=cwltool explicitly --- test/test_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_integration.py b/test/test_integration.py index e40485a..a143c35 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -180,6 +180,7 @@ def setUp(self): self.wes_server_process = subprocess.Popen( ['python', os.path.abspath('wes_service/wes_service_main.py'), '--backend=wes_service.cwl_runner', + '--opt', 'runner=cwltool', '--port=8080', '--debug']) time.sleep(5) From a44bb87f128d8cbe265488a8b158d7da8c19c833 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 11 Feb 2019 12:37:51 -0500 Subject: [PATCH 4/5] Skip arvados test if not configured --- test/test_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_integration.py b/test/test_integration.py index a143c35..ecbf5ca 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -9,6 +9,7 @@ import logging import sys import requests +import pytest pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # noqa sys.path.insert(0, pkg_root) # noqa @@ -210,6 +211,7 @@ def test_local_wdl(self): self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path)) +@pytest.mark.skipif(not os.environ.get("ARVADOS_API_TOKEN"), "Arvados not configured") class ArvadosTest(IntegrationTest): """Test using arvados-cwl-runner.""" From b236857a78973f67baa8878c9b3dc285061801c8 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 11 Feb 2019 13:15:23 -0500 Subject: [PATCH 5/5] Fix pytest.mark.skipif --- test/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_integration.py b/test/test_integration.py index ecbf5ca..dde9a64 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -211,7 +211,7 @@ def test_local_wdl(self): self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path)) -@pytest.mark.skipif(not os.environ.get("ARVADOS_API_TOKEN"), "Arvados not configured") +@pytest.mark.skipif(not os.environ.get("ARVADOS_API_TOKEN"), reason="Arvados not configured") class ArvadosTest(IntegrationTest): """Test using arvados-cwl-runner."""