From 37c7649f10cec778954b8df2008e67f8a69fffa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 13:55:24 +0200 Subject: [PATCH 1/2] FIX DateTime with min and max operators --- CHANGES_NEXT_RELEASE | 3 +- doc/manuals/orion-api.md | 3 +- src/lib/jsonParseV2/parseContextAttribute.cpp | 40 ++- .../datetime_with_min_max_operators.test | 253 ++++++++++++++++++ ...atetime_with_min_max_operators_create.test | 99 +++++++ ...atetime_with_min_max_operators_errors.test | 205 ++++++++++++++ 6 files changed, 594 insertions(+), 9 deletions(-) create mode 100644 test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators.test create mode 100644 test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_create.test create mode 100644 test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_errors.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 1ba267525a..8663908931 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,5 @@ -- Fix: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) +- Add: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) +- Fix: $max and $min operators were not supported with DateTime attributes (#4585) - Fix: wrong date values should not allowed in subscription's expires field (#4541) - Fix: do not raise DB alarm in case of wrong GeoJSON in client request - Upgrade cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 374da2024a..01ab451ca1 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -1618,7 +1618,8 @@ The only exception to "use only one operator" rule is the case of `$set` and Update operators can be used in entity creation or replace operations. In particular: * Numeric operators takes 0 as reference. For instance, `{"$inc": 4}` results in 4, - `{$mul: 1000}` results in 0, etc. + `{"$mul": 1000}` results in 0, etc. +* Epoch time (`"1970-01-01T00:00:00Z"`) is used as reference for `DateTime` when `$min` or `$max` are used. * `$set` takes the empty object (`{}`) as reference. For instance, `"$set": {"X": 1}` results in just `{"X": 1}` * `$push` and `$addToSet` take the empty array (`[]`) as reference. For instance, `{"$push": 4}` results in `[ 4 ]`. diff --git a/src/lib/jsonParseV2/parseContextAttribute.cpp b/src/lib/jsonParseV2/parseContextAttribute.cpp index caf6ac190a..4d4ab692a5 100644 --- a/src/lib/jsonParseV2/parseContextAttribute.cpp +++ b/src/lib/jsonParseV2/parseContextAttribute.cpp @@ -218,16 +218,42 @@ static std::string parseContextAttributeObject // Is it a (not null) date? if (checkAttrSpecialTypes && ((caP->type == DATE_TYPE) || (caP->type == DATE_TYPE_ALT)) && (caP->valueType != orion::ValueTypeNull)) { - caP->numberValue = parse8601Time(caP->stringValue); - - if (caP->numberValue == -1) + if (caP->valueType == orion::ValueTypeObject) { - return "date has invalid format"; + // check if $max or $min operators are being used. Other usages of object values result in parsing error + if ((caP->compoundValueP != NULL) && (caP->compoundValueP->valueType == orion::ValueTypeObject) + && (caP->compoundValueP->childV.size() == 1) && ((caP->compoundValueP->childV[0]->name == "$max") || (caP->compoundValueP->childV[0]->name == "$min"))) + { + orion::CompoundValueNode* upOp = caP->compoundValueP->childV[0]; + upOp->numberValue = parse8601Time(upOp->stringValue); + + if (upOp->numberValue == -1) + { + return "date has invalid format"; + } + + // Probably reseting stringValue is not needed, but let's do it for cleanliness + upOp->stringValue = ""; + upOp->valueType = orion::ValueTypeNumber; + } + else + { + return "date has invalid format"; + } } + else + { + caP->numberValue = parse8601Time(caP->stringValue); + + if (caP->numberValue == -1) + { + return "date has invalid format"; + } - // Probably reseting stringValue is not needed, but let's do it for cleanliness - caP->stringValue = ""; - caP->valueType = orion::ValueTypeNumber; + // Probably reseting stringValue is not needed, but let's do it for cleanliness + caP->stringValue = ""; + caP->valueType = orion::ValueTypeNumber; + } } // It is a safe GeoJSON? diff --git a/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators.test b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators.test new file mode 100644 index 0000000000..49ab277624 --- /dev/null +++ b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators.test @@ -0,0 +1,253 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +DateTime with min and max operators + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z +# 02. Update entity with {$min: 2025-04-15T10:59:59.000Z} +# 03. Get entity see T=2024-04-15T10:59:59.000Z +# 04. Update entity with {$min: 2023-04-15T10:59:59.000Z} +# 05. Get entity see T=2023-04-15T10:59:59.000Z +# 06. Update entity with {$max: 2022-04-15T10:59:59.000Z} +# 07. Get entity see T=2023-04-15T10:59:59.000Z +# 08. Update entity with {$max: 2026-04-15T10:59:59.000Z} +# 09. Get entity see T=2026-04-15T10:59:59.000Z +# + + +echo '01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z' +echo '=============================================================' +payload='{ + "id": "E1", + "type": "T", + "T": { + "value": "2024-04-15T10:59:59.000Z", + "type": "DateTime" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo '02. Update entity with {$min: 2025-04-15T10:59:59.000Z}' +echo "=======================================================" +payload='{ + "T": { + "value": {"$min": "2025-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "03. Get entity see T=2024-04-15T10:59:59.000Z" +echo "=============================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +echo '04. Update entity with {$min: 2023-04-15T10:59:59.000Z}' +echo "=======================================================" +payload='{ + "T": { + "value": {"$min": "2023-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "05. Get entity see T=2023-04-15T10:59:59.000Z" +echo "=============================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +echo '06. Update entity with {$max: 2022-04-15T10:59:59.000Z}' +echo "=======================================================" +payload='{ + "T": { + "value": {"$max": "2022-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "07. Get entity see T=2023-04-15T10:59:59.000Z" +echo "=============================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +echo '08. Update entity with {$max: 2026-04-15T10:59:59.000Z}' +echo "=======================================================" +payload='{ + "T": { + "value": {"$max": "2026-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "09. Get entity see T=2026-04-15T10:59:59.000Z" +echo "=============================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +--REGEXPECT-- +01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z +============================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +02. Update entity with {$min: 2025-04-15T10:59:59.000Z} +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +03. Get entity see T=2024-04-15T10:59:59.000Z +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 74 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2024-04-15T10:59:59.000Z" + } +} + + +04. Update entity with {$min: 2023-04-15T10:59:59.000Z} +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Get entity see T=2023-04-15T10:59:59.000Z +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 74 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2023-04-15T10:59:59.000Z" + } +} + + +06. Update entity with {$max: 2022-04-15T10:59:59.000Z} +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +07. Get entity see T=2023-04-15T10:59:59.000Z +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 74 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2023-04-15T10:59:59.000Z" + } +} + + +08. Update entity with {$max: 2026-04-15T10:59:59.000Z} +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +09. Get entity see T=2026-04-15T10:59:59.000Z +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 74 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2026-04-15T10:59:59.000Z" + } +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_create.test b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_create.test new file mode 100644 index 0000000000..5206388283 --- /dev/null +++ b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_create.test @@ -0,0 +1,99 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +DateTime with min and max operators creating entities + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity with T1={$min: 2025-04-15T10:59:59.000Z}, T2={$max: 2024-04-15T10:59:59.000Z} +# 02. Get entity see T1=1970-01-01T00:00:00.000Z, T2=2024-04-15T10:59:59.000Z +# + + +echo '01. Create entity with T1={$min: 2025-04-15T10:59:59.000Z}, T2={$max: 2024-04-15T10:59:59.000Z}' +echo '===============================================================================================' +payload='{ + "id": "E1", + "type": "T", + "T1": { + "value": {"$min": "2025-04-15T10:59:59.000Z"}, + "type": "DateTime" + }, + "T2": { + "value": {"$max": "2024-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "02. Get entity see T1=1970-01-01T00:00:00.000Z, T2=2024-04-15T10:59:59.000Z" +echo "===========================================================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +--REGEXPECT-- +01. Create entity with T1={$min: 2025-04-15T10:59:59.000Z}, T2={$max: 2024-04-15T10:59:59.000Z} +=============================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +02. Get entity see T1=1970-01-01T00:00:00.000Z, T2=2024-04-15T10:59:59.000Z +=========================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 149 + +{ + "T1": { + "metadata": {}, + "type": "DateTime", + "value": "1970-01-01T00:00:00.000Z" + }, + "T2": { + "metadata": {}, + "type": "DateTime", + "value": "2024-04-15T10:59:59.000Z" + } +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_errors.test b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_errors.test new file mode 100644 index 0000000000..05fdd8de7f --- /dev/null +++ b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_errors.test @@ -0,0 +1,205 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +DateTime with min and max operators errors + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z +# 02. Update entity with {$min: 2025x-04-15T10:59:59.000Z}, see error +# 03. Update entity with {$max: 2023x-04-15T10:59:59.000Z, see error +# 04. Update entity with {$mxn: 2025-04-15T10:59:59.000Z}, see error +# 05. Update entity with {$max: 2025-04-15T10:59:59.000Z, foo: 1}, see error +# 06. Get entity see T=2024-04-15T10:59:59.000Z (entity not modified) +# + + +echo '01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z' +echo '=============================================================' +payload='{ + "id": "E1", + "type": "T", + "T": { + "value": "2024-04-15T10:59:59.000Z", + "type": "DateTime" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo '02. Update entity with {$min: 2025x-04-15T10:59:59.000Z}, see error' +echo "===================================================================" +payload='{ + "T": { + "value": {"$min": "2025x-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo '03. Update entity with {$max: 2023x-04-15T10:59:59.000Z, see error' +echo "==================================================================" +payload='{ + "T": { + "value": {"$max": "2023x-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo '04. Update entity with {$mxn: 2025-04-15T10:59:59.000Z}, see error' +echo "==================================================================" +payload='{ + "T": { + "value": {"$mxn": "2025-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo '05. Update entity with {$max: 2025-04-15T10:59:59.000Z, foo: 1}, see error' +echo "==========================================================================" +payload='{ + "T": { + "value": {"$max": "2025-04-15T10:59:59.000Z", "foo": 1}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "06. Get entity see T=2024-04-15T10:59:59.000Z (entity not modified)" +echo "===================================================================" +orionCurl --url /v2/entities/E1 +echo +echo + + + +--REGEXPECT-- +01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z +============================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +02. Update entity with {$min: 2025x-04-15T10:59:59.000Z}, see error +=================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 62 + +{ + "description": "date has invalid format", + "error": "BadRequest" +} + + +03. Update entity with {$max: 2023x-04-15T10:59:59.000Z, see error +================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 62 + +{ + "description": "date has invalid format", + "error": "BadRequest" +} + + +04. Update entity with {$mxn: 2025-04-15T10:59:59.000Z}, see error +================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 62 + +{ + "description": "date has invalid format", + "error": "BadRequest" +} + + +05. Update entity with {$max: 2025-04-15T10:59:59.000Z, foo: 1}, see error +========================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 62 + +{ + "description": "date has invalid format", + "error": "BadRequest" +} + + +06. Get entity see T=2024-04-15T10:59:59.000Z (entity not modified) +=================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2024-04-15T10:59:59.000Z" + }, + "id": "E1", + "type": "T" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file From 900d9860fef27f072ac5db331bba44a232c92df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 14:59:42 +0200 Subject: [PATCH 2/2] FIX comment raised in code review --- doc/manuals/orion-api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 01ab451ca1..0b12ed9df8 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -1332,7 +1332,7 @@ PUT /v2/entities/E/attrs/A would not change attribute value. -Apart from numbers, other value types are supported (eg, strings). +Apart from numbers, other value types are supported (eg, strings or `DateTime`). #### `$max` @@ -1360,7 +1360,7 @@ PUT /v2/entities/E/attrs/A would not change attribute value. -Apart from numbers, other value types are supported (eg, strings). +Apart from numbers, other value types are supported (eg, strings or `DateTime`). #### `$push`