diff --git a/docs/source/pages/protocols/uws.rst b/docs/source/pages/protocols/uws.rst index 27c30f8..d9d346c 100644 --- a/docs/source/pages/protocols/uws.rst +++ b/docs/source/pages/protocols/uws.rst @@ -59,6 +59,9 @@ Represents a single parameter of a UWS job. :start-after: parameter-xml-start :end-before: parameter-xml-end +For multi-valued attributes, use the type `vo_models.uws.models.MultiValuedParameter` instead of ``list[Parameter]``. +This is equivalent to ``list[Parameter]`` but adds some special validation support required for multi-valued UWS job parameters. + Parameters ********** @@ -225,4 +228,4 @@ The following simple types are provided in the ``vo_models.uws`` package for use - :py:class:`vo_models.uws.types.ErrorType` - :py:class:`vo_models.uws.types.ExecutionPhase` -- :py:class:`vo_models.uws.types.UWSVersion` \ No newline at end of file +- :py:class:`vo_models.uws.types.UWSVersion` diff --git a/pyproject.toml b/pyproject.toml index ceee6a9..b9bbae1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ + "pydantic>2", "pydantic-xml[lxml]>=2.6.0", ] diff --git a/tests/uws/uws_models_test.py b/tests/uws/uws_models_test.py index d007cca..52803e8 100644 --- a/tests/uws/uws_models_test.py +++ b/tests/uws/uws_models_test.py @@ -12,6 +12,7 @@ ErrorSummary, Jobs, JobSummary, + MultiValuedParameter, Parameter, Parameters, ResultReference, @@ -257,13 +258,28 @@ class TestParameters(Parameters): param1: Optional[Parameter] = element(tag="parameter", default=None) param2: Optional[Parameter] = element(tag="parameter", default=None) - param3: Optional[Parameter] = element(tag="parameter", default=None) + param3: Optional[MultiValuedParameter] = element(tag="parameter", default=None) + param4: Optional[MultiValuedParameter] = element(tag="parameter", default=None) + param5: MultiValuedParameter = element(tag="parameter") test_parameters_xml = ( f"" 'value1' 'value2' - 'value3' + 'value4' + 'second value4' + 'value5' + "" + ) + + # Test that the same parameters are parsed correctly when given out of order. + test_parameters_xml_reordered = ( + f"" + 'value5' + 'value4' + 'value1' + 'second value4' + 'value2' "" ) @@ -271,15 +287,32 @@ def test_read_from_xml(self): """Test reading from XML""" parameters = self.TestParameters.from_xml(self.test_parameters_xml) - self.assertEqual(len(parameters.model_dump()), 3) + + self.assertEqual(len(parameters.dict()), 5) + + self.assertEqual(parameters.param3, None) + self.assertEqual(len(parameters.param4), 2) + self.assertEqual(len(parameters.param5), 1) self.assertEqual(parameters.param1.id, "param1") self.assertEqual(parameters.param2.id, "param2") - self.assertEqual(parameters.param3.id, "param3") + self.assertEqual(parameters.param4[0].id, "param4") + self.assertEqual(parameters.param4[1].id, "param4") + self.assertEqual(parameters.param5[0].id, "param5") self.assertEqual(parameters.param1.value, "value1") self.assertEqual(parameters.param2.value, "value2") - self.assertEqual(parameters.param3.value, "value3") + self.assertEqual(parameters.param4[0].value, "value4") + self.assertEqual(parameters.param4[1].value, "second value4") + self.assertEqual(parameters.param5[0].value, "value5") + + def test_read_from_xml_reordered(self): + """Test reading from XML out of order""" + + parameters = self.TestParameters.from_xml(self.test_parameters_xml) + parameters_reordered = self.TestParameters.from_xml(self.test_parameters_xml_reordered) + + assert parameters == parameters_reordered def test_write_to_xml(self): """Test writing to XML""" @@ -287,7 +320,12 @@ def test_write_to_xml(self): parameters_element = self.TestParameters( param1=Parameter(id="param1", value="value1"), param2=Parameter(id="param2", value="value2"), - param3=Parameter(id="param3", value="value3"), + param3=None, + param4=[ + Parameter(id="param4", value="value4"), + Parameter(id="param4", value="second value4"), + ], + param5=[Parameter(id="param5", value="value5")], ) parameters_xml = parameters_element.to_xml(skip_empty=True, encoding=str) @@ -302,7 +340,11 @@ def test_validate(self): parameters = self.TestParameters( param1=Parameter(id="param1", value="value1"), param2=Parameter(id="param2", value="value2"), - param3=Parameter(id="param3", value="value3"), + param4=[ + Parameter(id="param4", value="value4"), + Parameter(id="param4", value="second value4"), + ], + param5=[Parameter(id="param5", value="value5")], ) parameters_xml = etree.fromstring( parameters.to_xml(skip_empty=True, encoding=str) diff --git a/vo_models/uws/__init__.py b/vo_models/uws/__init__.py index 9f19708..7c5f411 100644 --- a/vo_models/uws/__init__.py +++ b/vo_models/uws/__init__.py @@ -9,6 +9,7 @@ Job, Jobs, JobSummary, + MultiValuedParameter, Parameter, Parameters, ParametersType, diff --git a/vo_models/uws/models.py b/vo_models/uws/models.py index 4a8b992..21e2368 100644 --- a/vo_models/uws/models.py +++ b/vo_models/uws/models.py @@ -1,7 +1,10 @@ """UWS Job Schema using Pydantic-XML models""" -from typing import Dict, Generic, Optional, TypeVar +from typing import Annotated, Dict, Generic, Optional, TypeAlias, TypeVar + +from pydantic import BeforeValidator from pydantic import ConfigDict + from pydantic_xml import BaseXmlModel, attr, element from vo_models.uws.types import ErrorType, ExecutionPhase, UWSVersion @@ -42,23 +45,61 @@ class Parameter(BaseXmlModel, tag="parameter", ns="uws", nsmap=NSMAP): is_post: Optional[bool] = attr(name="isPost", default=False) +MultiValuedParameter: TypeAlias = Annotated[ + list[Parameter], + BeforeValidator(lambda v: v if isinstance(v, list) else [v]) +] +"""Type for a multi-valued parameter. + +This type must be used instead of ``list[Parameter]`` for parameters that may +take multiple values. The resulting model attribute will be a list of +`Parameter` objects with the same ``id``. +""" + + class Parameters(BaseXmlModel, tag="parameters", ns="uws", nsmap=NSMAP): - """ - An abstract holder of UWS parameters. + """An abstract holder of UWS parameters. - The list of input parameters to the job - if the job description language does not naturally have - parameters, then this list should contain one element which is the content of the original POST that created the - job. + The input parameters to the job. For simple key/value pair parameters, there must be one model field per key, with + a type of either `Parameter` or `MultiValuedParameter` depending on whether it can be repeated. If the job + description language does not naturally have parameters, then this model should contain one element, which is the + content of the original POST that created the job. """ def __init__(__pydantic_self__, **data) -> None: # pylint: disable=no-self-argument - # during init -- especially if reading from xml -- we may not get the parameters in the order - # pydantic-xml expects. This will remap the dict with keys based on the parameter id. - parameter_vals = [val for val in data.values() if val is not None] + # pydantic-xml's from_xml method only knows how to gather parameters by tag, not by attribute, but UWS job + # parameters all use the same tag and distinguish between different parameters by the id attribute. Therefore, + # the input data that we get from pydantic-xml will have assigned the Parameter objects to the model attributes + # in model declaration order. This may be wildly incorrect if the input parameters were specified in a + # different order than the model. + # + # This processing collapses all of the parameters down to a simple list, turns that into a dict to Parameter + # objects or lists of Parameter objects by id attribute values, and then calls the parent constructor with that + # as the input data instead. This should cause Pydantic to associate the job parameters with the correct model + # attributes. + parameter_vals = [] + for val in data.values(): + if val is None: + continue + elif isinstance(val, list): + parameter_vals.extend(v for v in val if v is not None) + else: + parameter_vals.append(val) + + # Reconstruct the proper input parameters to the model based on the id attribute of the parameters. First + # convert each parameter to a Parameter object, and then turn the parameters into a dict mapping the id to a + # value or list of values. If we see multiple parameters with the same id, assume the parameter may be + # multivalued and build a list. If this assumption is incorrect, Pydantic will reject the list during input + # validation. remapped_vals = {} for param in parameter_vals: if isinstance(param, dict): - remapped_vals[param["id"]] = Parameter(**param) + param = Parameter(**param) + if param.id in remapped_vals: + if isinstance(remapped_vals[param.id], list): + remapped_vals[param.id].append(param) + else: + remapped_vals[param.id] = [remapped_vals[param.id], param] else: remapped_vals[param.id] = param data = remapped_vals