diff --git a/requirements_dev.txt b/requirements_dev.txt index a4079d2a..fe8e5a6d 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -2,6 +2,7 @@ bidict==0.21.4 pytest==7.1.1 pytest-cov==3.0.0 pytest-timeout==2.1.0 +ruamel.yaml setuptools==60.10.0 stomp.py==8.0.0 pika==1.2.0 diff --git a/setup.cfg b/setup.cfg index cb1bc074..371c1a9d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ project_urls = install_requires = bidict pika + ruamel.yaml setuptools stomp.py>=7 packages = find: diff --git a/src/workflows/recipe/recipe.py b/src/workflows/recipe/recipe.py index d4744489..dbb66391 100644 --- a/src/workflows/recipe/recipe.py +++ b/src/workflows/recipe/recipe.py @@ -1,15 +1,33 @@ from __future__ import annotations import copy +import io import json import string from typing import Any, Dict +from ruamel.yaml import YAML +from ruamel.yaml.constructor import SafeConstructor + import workflows basestring = (str, bytes) +def construct_yaml_map(self, node): + # Test if there are duplicate node keys + # In the case of duplicate keys, last wins + data = {} + yield data + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=True) + val = self.construct_object(value_node, deep=True) + data[key] = val + + +SafeConstructor.add_constructor("tag:yaml.org,2002:map", construct_yaml_map) + + class Recipe: """Object containing a processing recipe that can be passed to services. A recipe describes how all involved services are connected together, how @@ -29,7 +47,9 @@ def __init__(self, recipe=None): def deserialize(self, string): """Convert a recipe that has been stored as serialized json string to a data structure.""" - return self._sanitize(json.loads(string)) + with YAML(typ="safe") as yaml: + yaml.allow_duplicate_keys = True + return self._sanitize(yaml.load(string)) @staticmethod def _sanitize(recipe): @@ -50,9 +70,20 @@ def _sanitize(recipe): recipe["start"] = [tuple(x) for x in recipe["start"]] return recipe - def serialize(self): - """Write out the current recipe as serialized json string.""" - return json.dumps(self.recipe) + def serialize(self, format: str = "json"): + """Write out the current recipe as serialized string. + + Supported serialization formats are "json" and "yaml". + """ + if format == "json": + return json.dumps(self.recipe) + elif format == "yaml": + buf = io.StringIO() + with YAML(output=buf, typ="safe") as yaml: + yaml.dump(self.recipe) + return buf.getvalue() + else: + raise ValueError(f"Unsupported serialization format {format}") def pretty(self): """Write out the current recipe as serialized json string with pretty formatting.""" diff --git a/tests/recipe/test_recipe.py b/tests/recipe/test_recipe.py index c27eb8f2..bf2690fa 100644 --- a/tests/recipe/test_recipe.py +++ b/tests/recipe/test_recipe.py @@ -1,8 +1,10 @@ from __future__ import annotations +import json from unittest import mock import pytest +from ruamel.yaml import YAML import workflows import workflows.recipe @@ -100,6 +102,9 @@ def test_serializing_and_deserializing_recipes(): assert A.deserialize(A.serialize()) == A.recipe assert B.deserialize(B.serialize()) == B.recipe + assert A.deserialize(A.serialize(format="yaml")) == A.recipe + assert B.deserialize(B.serialize(format="yaml")) == B.recipe + def test_validate_tests_for_empty_recipe(): """Validating a recipe that has not been defined must throw an error.""" @@ -368,3 +373,120 @@ def test_merging_recipes(): C.recipe.values(), ) ) + + +def test_deserialize_json(): + healthy_recipe = """ + { + "1": { + "queue": "my.queue", + "parameters": { + "foo": "bar", + "ingredients": ["ham", "spam"], + "workingdir": "/path/to/workingdir", + "output_file": "out.txt" + } + }, + "start": [ + [1, []] + ] + } + """ + A = workflows.recipe.Recipe(healthy_recipe) + assert A.recipe == { + "start": [(1, [])], + 1: { + "queue": "my.queue", + "parameters": { + "foo": "bar", + "ingredients": ["ham", "spam"], + "workingdir": "/path/to/workingdir", + "output_file": "out.txt", + }, + }, + } + + +def test_deserialize_yaml(): + healthy_recipe = """ +1: + queue: my.queue + parameters: + foo: bar + ingredients: + - ham + - spam + workingdir: /path/to/workingdir + output_file: out.txt +start: +- [1, []] +""" + A = workflows.recipe.Recipe(healthy_recipe) + assert A.recipe == { + "start": [(1, [])], + 1: { + "queue": "my.queue", + "parameters": { + "foo": "bar", + "ingredients": ["ham", "spam"], + "workingdir": "/path/to/workingdir", + "output_file": "out.txt", + }, + }, + } + + +def test_serialize_json(): + A, B = generate_recipes() + + assert ( + A.recipe + == workflows.recipe.Recipe(json.loads(A.serialize(format="json"))).recipe + ) + assert ( + B.recipe + == workflows.recipe.Recipe(json.loads(B.serialize(format="json"))).recipe + ) + + +def test_serialize_yaml(): + A, B = generate_recipes() + + # Verify that this definitely isn't json + with pytest.raises(json.JSONDecodeError): + json.loads(A.serialize(format="yaml")) + + with YAML(typ="safe") as yaml: + assert ( + A.recipe + == workflows.recipe.Recipe(yaml.load(A.serialize(format="yaml"))).recipe + ) + assert ( + B.recipe + == workflows.recipe.Recipe(yaml.load(B.serialize(format="yaml"))).recipe + ) + + +def test_unsupported_serialization_format_raises_error(): + A, _ = generate_recipes() + + with pytest.raises(ValueError): + A.serialize(format="xml") + + +def test_json_recipe_with_pseudo_comment(): + recipe = """ + { + "1": ["This is a comment"], + "1": { + "parameters": { + "foo": "bar" + } + }, + "start": [ + [1, []] + ] + } + """ + A = workflows.recipe.Recipe(recipe) + assert A.recipe == {"start": [(1, [])], 1: {"parameters": {"foo": "bar"}}} diff --git a/tests/recipe/test_validate.py b/tests/recipe/test_validate.py index 46bed6fc..67cc5c2a 100644 --- a/tests/recipe/test_validate.py +++ b/tests/recipe/test_validate.py @@ -4,11 +4,11 @@ from __future__ import annotations -import json import sys from unittest import mock import pytest +import ruamel.yaml import workflows from workflows.recipe.validate import main, validate_recipe @@ -64,8 +64,8 @@ def test_value_error_when_validating_bad_json(tmpdir): recipe_file = tmpdir.join("recipe.json") recipe_file.write(bad_json) - # Run validate with mock open, expect JSON error - with pytest.raises(json.JSONDecodeError): + # Run validate with mock open, expect parsing error + with pytest.raises(ruamel.yaml.parser.ParserError): validate_recipe(recipe_file.strpath) diff --git a/tests/recipe/test_wrapped_recipe.py b/tests/recipe/test_wrapped_recipe.py index bf4704f4..5219d46d 100644 --- a/tests/recipe/test_wrapped_recipe.py +++ b/tests/recipe/test_wrapped_recipe.py @@ -4,7 +4,7 @@ import pytest -import workflows +import workflows.transport.common_transport from workflows.recipe import Recipe from workflows.recipe.wrapper import RecipeWrapper