From 0fe0be5d33ebcec5daffbd625ecd5311735f9ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 2 Jul 2024 13:01:01 +0200 Subject: [PATCH 01/12] ADD evalPriority feature --- src/lib/common/JsonHelper.cpp | 33 ++- src/lib/common/JsonHelper.h | 6 +- src/lib/common/errorMessages.h | 2 +- src/lib/common/limits.h | 10 + src/lib/expressions/ExprContext.cpp | 11 +- src/lib/jsonParseV2/parseSubscription.cpp | 15 +- src/lib/ngsi/ContextAttribute.cpp | 31 ++- src/lib/ngsi/ContextAttribute.h | 4 +- src/lib/ngsi/Metadata.h | 1 + src/lib/ngsiNotify/Notifier.cpp | 62 ++++- .../eval_priority_in_expressions.test | 262 ++++++++++++++++++ 11 files changed, 402 insertions(+), 35 deletions(-) create mode 100644 test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test diff --git a/src/lib/common/JsonHelper.cpp b/src/lib/common/JsonHelper.cpp index 3bd6fd5d2a..9adc597fc2 100644 --- a/src/lib/common/JsonHelper.cpp +++ b/src/lib/common/JsonHelper.cpp @@ -138,7 +138,7 @@ std::string objectToJson(std::map& list) * * JsonObjectHelper - */ -JsonObjectHelper::JsonObjectHelper(): empty(true), closed(false) +JsonObjectHelper::JsonObjectHelper(): empty(true) { ss += '{'; } @@ -280,15 +280,17 @@ void JsonObjectHelper::addNull(const std::string& key) * * JsonObjectHelper::str - */ -std::string JsonObjectHelper::str() +std::string JsonObjectHelper::str(bool closed) { - // This check allows to call str() several times (needed when this is used in ExprContext) - if (!closed) + // closed == false used in ExprContext logic + if (closed) { - ss += '}'; - closed = true; + return ss + '}'; + } + else + { + return ss; } - return ss; } @@ -297,7 +299,7 @@ std::string JsonObjectHelper::str() * * JsonVectorHelper - */ -JsonVectorHelper::JsonVectorHelper(): empty(true), closed(false) +JsonVectorHelper::JsonVectorHelper(): empty(true) { ss += '['; } @@ -425,14 +427,17 @@ void JsonVectorHelper::addNull(void) /* **************************************************************************** * * JsonVectorHelper::str - +* FIXME PR: bool closed probably unneded in vectors */ -std::string JsonVectorHelper::str() +std::string JsonVectorHelper::str(bool closed) { - // This check allows to call str() several times (needed when this is used in ExprContext) - if (!closed) + // closed == false used in ExprContext logic + if (closed) { - ss += ']'; - closed = true; + return ss + ']'; + } + else + { + return ss; } - return ss; } diff --git a/src/lib/common/JsonHelper.h b/src/lib/common/JsonHelper.h index bcb8aaeaf2..5b2633e0ca 100644 --- a/src/lib/common/JsonHelper.h +++ b/src/lib/common/JsonHelper.h @@ -45,12 +45,11 @@ class JsonObjectHelper void addBool(const std::string& key, bool b); void addNull(const std::string& key); - std::string str(); + std::string str(bool closed = true); private: std::string ss; bool empty; - bool closed; }; @@ -68,12 +67,11 @@ class JsonVectorHelper void addNull(void); - std::string str(); + std::string str(bool closed = true); private: std::string ss; bool empty; - bool closed; }; diff --git a/src/lib/common/errorMessages.h b/src/lib/common/errorMessages.h index d50bf8acf7..a5f36a2f3a 100644 --- a/src/lib/common/errorMessages.h +++ b/src/lib/common/errorMessages.h @@ -87,7 +87,7 @@ #define ERROR_DESC_BAD_REQUEST_FORMAT_INVALID "invalid render format for notifications" #define ERROR_DESC_BAD_REQUEST_SERVICE_NOT_FOUND "Service not found. Check your URL as probably it is wrong." #define ERROR_DESC_BAD_REQUEST_WRONG_GEOJSON "Wrong GeoJson" -#define ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF "metadata are not allowed in ngsi field in custom notifications" +#define ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF "only evalPriority metadata is allowed in ngsi field in custom notifications" #define ERROR_NOT_FOUND "NotFound" #define ERROR_DESC_NOT_FOUND_ENTITY "The requested entity has not been found. Check type and id" diff --git a/src/lib/common/limits.h b/src/lib/common/limits.h index 39295ec64d..9a903f7953 100644 --- a/src/lib/common/limits.h +++ b/src/lib/common/limits.h @@ -229,4 +229,14 @@ +/* **************************************************************************** +* +* MIX_PRIORITY and MAX_PRIORITY +* +*/ +#define MIN_PRIORITY 1 +#define MAX_PRIORITY LLONG_MAX + + + #endif // SRC_LIB_COMMON_LIMITS_H_ diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 80148399b3..94bf607a74 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -47,7 +47,7 @@ ExprContextObject::ExprContextObject(bool _basic) */ std::string ExprContextObject::getJexlContext(void) { - return jh.str(); + return jh.str(false) + '}'; } @@ -83,7 +83,14 @@ void ExprContextObject::add(const std::string &key, const std::string &_value, b else { LM_T(LmtExpr, ("adding to JEXL expression context object (string): %s=%s", key.c_str(), _value.c_str())); - jh.addString(key, _value); + if (raw) + { + jh.addRaw(key, _value); + } + else + { + jh.addString(key, _value); + } } } diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index 593161b202..7dd54c9b65 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -554,10 +554,19 @@ static std::string parseCustomPayload return badInput(ciP, r); } - // metadadata are now allowed in this case - if (caP->metadataVector.size() > 0) + // only evalPriority metadadata is allowed in this case + for (unsigned ix = 0; ix < caP->metadataVector.size(); ix++) { - return badInput(ciP, ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF); + if (caP->metadataVector[ix]->name != NGSI_MD_EVAL_PRIORITY) + { + return badInput(ciP, ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF); + } + else + { + // priority must be a number + // priority must be between MIX_PRIORITY and MAX_PRIORITY + // FIXME: PR (include a .test to asses the checkings works) + } } } } diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 79ce3f57fc..5fa1f9f78c 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -763,6 +763,26 @@ bool ContextAttribute::getLocation(orion::BSONObj* attrsP) const +/* **************************************************************************** +* +* getEvalPriority - +*/ +double ContextAttribute::getEvalPriority(void) +{ + for (unsigned int ix = 0; ix < metadataVector.size(); ix++) + { + if (metadataVector[ix]->name == NGSI_MD_EVAL_PRIORITY) + { + return metadataVector[ix]->numberValue; + } + } + + // if the attribute doesn't have evalPriority metadata, then max priority is assumed + return MAX_PRIORITY; +} + + + /* **************************************************************************** * * toJsonV1AsObject - @@ -1140,9 +1160,10 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi * toJsonValue - * * To be used by options=values and options=unique renderings +* Also used by the ngsi expression logic * */ -std::string ContextAttribute::toJsonValue(void) +std::string ContextAttribute::toJsonValue(ExprContextObject* exprContextObjectP) { if (compoundValueP != NULL) { @@ -1164,10 +1185,10 @@ std::string ContextAttribute::toJsonValue(void) } else if (valueType == orion::ValueTypeString) { - std::string out = "\""; - out += toJsonString(stringValue); - out += '"'; - return out; + //std::string out = "\""; + //out += toJsonString(stringValue); + //out += '"'; + return smartStringValue(stringValue, exprContextObjectP, "null"); } else if (valueType == orion::ValueTypeBoolean) { diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index f242bf101d..63e4551690 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -95,6 +95,8 @@ typedef struct ContextAttribute /* Check if attribute means a location */ bool getLocation(orion::BSONObj* attrsP) const; + double getEvalPriority(void); + std::string toJsonV1(bool asJsonObject, RequestType request, const std::vector& metadataFilter, @@ -110,7 +112,7 @@ typedef struct ContextAttribute std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, ExprContextObject* exprContextObjectP = NULL); - std::string toJsonValue(void); + std::string toJsonValue(ExprContextObject* exprContextObjectP = NULL); std::string toJsonAsValue(ApiVersion apiVersion, bool acceptedTextPlain, diff --git a/src/lib/ngsi/Metadata.h b/src/lib/ngsi/Metadata.h index 591df92cb1..e1fcdab163 100644 --- a/src/lib/ngsi/Metadata.h +++ b/src/lib/ngsi/Metadata.h @@ -46,6 +46,7 @@ * * Metadata interpreted by Orion Context Broker, i.e. not custom metadata */ +#define NGSI_MD_EVAL_PRIORITY "evalPriority" #define NGSI_MD_IGNORE_TYPE "ignoreType" #define NGSI_MD_PREVIOUSVALUE "previousValue" // Special metadata #define NGSI_MD_ACTIONTYPE "actionType" // Special metadata diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 14724c9a1e..6fd17245da 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -193,6 +193,51 @@ static bool setJsonPayload +/* **************************************************************************** +* +* orderByPriority - +* +* Return false if some problem occur +*/ +static void orderByPriority +( + const Entity& ngsi, + std::vector* orderedAttrs +) +{ + // Add all the attributes to a temporal vector + std::vector attrs; + + for (unsigned ix = 0; ix < ngsi.attributeVector.size(); ix++) + { + attrs.push_back(ngsi.attributeVector[ix]); + } + + // Pass the content of attrs to orderedAttrs based in the evalPriority element + while (!attrs.empty()) + { + ContextAttribute* selectedAttr = attrs[0]; + unsigned int selectedIx = 0; + double prio = selectedAttr->getEvalPriority(); + + for (unsigned ix = 0; ix < attrs.size(); ix++) + { + double newPrio = attrs[ix]->getEvalPriority(); + if (newPrio < prio) + { + selectedAttr = attrs[ix]; + selectedIx = ix; + prio = newPrio; + } + } + + orderedAttrs->push_back(selectedAttr); + attrs.erase(attrs.begin() + selectedIx); + } +} + + + /* **************************************************************************** * * setNgsiPayload - @@ -209,7 +254,8 @@ static bool setNgsiPayload bool blacklist, const std::vector& metadataFilter, std::string* payloadP, - RenderFormat renderFormat + RenderFormat renderFormat, + bool basic // used by TIME_EXPR_CTXBLD_START/STOP macros ) { NotifyContextRequest ncr; @@ -239,10 +285,16 @@ static bool setNgsiPayload cer.entity.fill(effectiveId, effectiveType, en.isPattern, en.servicePath); - // First we add attributes in the ngsi field - for (unsigned int ix = 0; ix < ngsi.attributeVector.size(); ix++) + // First we add attributes in the ngsi field, adding calculated expressions to context in order of priority + std::vector orderedNgsiAttrs; + orderByPriority(ngsi, &orderedNgsiAttrs); + + for (unsigned int ix = 0; ix < orderedNgsiAttrs.size(); ix++) { - cer.entity.attributeVector.push_back(new ContextAttribute(ngsi.attributeVector[ix], false, true)); + cer.entity.attributeVector.push_back(new ContextAttribute(orderedNgsiAttrs[ix], false, true)); + TIME_EXPR_CTXBLD_START(); + exprContextObjectP->add(orderedNgsiAttrs[ix]->name, orderedNgsiAttrs[ix]->toJsonValue(exprContextObjectP), true); + TIME_EXPR_CTXBLD_STOP(); } // Next, other attributes in the original entity not already added for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) @@ -393,7 +445,7 @@ static SenderThreadParams* buildSenderParamsCustom { // Important to use const& for Entity here. Otherwise problems may occur in the object release logic const Entity& ngsi = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.ngsi : notification.mqttInfo.ngsi); - if (!setNgsiPayload(ngsi, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) + if (!setNgsiPayload(ngsi, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat, basic)) { // Warning already logged in macroSubstitute() return NULL; diff --git a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test new file mode 100644 index 0000000000..5df812c7a1 --- /dev/null +++ b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test @@ -0,0 +1,262 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +Evaluation priority in expressions + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified) +# 02. Create entity E1 with A=1 +# 03. Update entity E1 with A=2.1 +# 04. Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1) +# + + +echo "01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified)" +echo "=========================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "P2": { + "value": "${P1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 2, + "type": "Number" + } + } + }, + "P1": { + "value": "${A+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 1, + "type": "Number" + } + } + }, + "P3": { + "value": "${P2+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 3, + "type": "Number" + } + } + }, + "S": { + "value": "${P3+1}", + "type": "Calculated" + } + } + }, + "attrs": [ "P1", "P2", "S" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A=2.1" +echo "===============================" +payload='{ + "A": { + "value": 2.1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1)" +echo "====================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified) +========================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A=2.1 +=============================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1) +==================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 313 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "P1": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 1 + } + }, + "type": "Calculated", + "value": 2 + }, + "P2": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 2 + } + }, + "type": "Calculated", + "value": 3 + }, + "S": { + "metadata": {}, + "type": "Calculated", + "value": 5 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 319 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "P1": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 1 + } + }, + "type": "Calculated", + "value": 3.1 + }, + "P2": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 2 + } + }, + "type": "Calculated", + "value": 4.1 + }, + "S": { + "metadata": {}, + "type": "Calculated", + "value": 6.1 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 50221f6b3a38a0f78e0098c74b3af0d80262876c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 09:41:47 +0200 Subject: [PATCH 02/12] FIX jexl context addition logic --- src/lib/ngsiNotify/Notifier.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 6fd17245da..fb5070f1ab 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -292,9 +292,14 @@ static bool setNgsiPayload for (unsigned int ix = 0; ix < orderedNgsiAttrs.size(); ix++) { cer.entity.attributeVector.push_back(new ContextAttribute(orderedNgsiAttrs[ix], false, true)); - TIME_EXPR_CTXBLD_START(); - exprContextObjectP->add(orderedNgsiAttrs[ix]->name, orderedNgsiAttrs[ix]->toJsonValue(exprContextObjectP), true); - TIME_EXPR_CTXBLD_STOP(); + + // Avoid to add context if an attribute with the same name exists in the entity + if (cer.entity.attributeVector.get(orderedNgsiAttrs[ix]->name) < 0) + { + TIME_EXPR_CTXBLD_START(); + exprContextObjectP->add(orderedNgsiAttrs[ix]->name, orderedNgsiAttrs[ix]->toJsonValue(exprContextObjectP), true); + TIME_EXPR_CTXBLD_STOP(); + } } // Next, other attributes in the original entity not already added for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) From a67e6f10378bb37517cfb6662ce0cba2d93ede30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 09:49:44 +0200 Subject: [PATCH 03/12] FIX description in 4085 ftest --- .../custom_notification_http_ngsi_errors.test | 4 ++-- .../custom_notification_mqtt_ngsi_errors.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_errors.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_errors.test index 56e39ca359..1406c43de4 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_errors.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_errors.test @@ -243,10 +243,10 @@ HTTP/1.1 400 Bad Request Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 101 +Content-Length: 114 { - "description": "metadata are not allowed in ngsi field in custom notifications", + "description": "only evalPriority metadata is allowed in ngsi field in custom notifications", "error": "BadRequest" } diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_errors.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_errors.test index 59742603a8..78f1212fa3 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_errors.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_errors.test @@ -249,10 +249,10 @@ HTTP/1.1 400 Bad Request Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 101 +Content-Length: 114 { - "description": "metadata are not allowed in ngsi field in custom notifications", + "description": "only evalPriority metadata is allowed in ngsi field in custom notifications", "error": "BadRequest" } From 61a2fa4f0570aad05be0130b1d6c594479ffd526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 10:03:39 +0200 Subject: [PATCH 04/12] FIX jexl context addition logic --- src/lib/ngsiNotify/Notifier.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index fb5070f1ab..0ca4da8582 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -291,15 +291,15 @@ static bool setNgsiPayload for (unsigned int ix = 0; ix < orderedNgsiAttrs.size(); ix++) { - cer.entity.attributeVector.push_back(new ContextAttribute(orderedNgsiAttrs[ix], false, true)); - // Avoid to add context if an attribute with the same name exists in the entity - if (cer.entity.attributeVector.get(orderedNgsiAttrs[ix]->name) < 0) + if (en.attributeVector.get(orderedNgsiAttrs[ix]->name) < 0) { TIME_EXPR_CTXBLD_START(); exprContextObjectP->add(orderedNgsiAttrs[ix]->name, orderedNgsiAttrs[ix]->toJsonValue(exprContextObjectP), true); TIME_EXPR_CTXBLD_STOP(); } + + cer.entity.attributeVector.push_back(new ContextAttribute(orderedNgsiAttrs[ix], false, true)); } // Next, other attributes in the original entity not already added for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) From 3eed5ca99f9d89ed4f40f4b3a6408d3e15cd3447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 10:06:25 +0200 Subject: [PATCH 05/12] FIX expression names as suggested in code review --- .../eval_priority_in_expressions.test | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test index 5df812c7a1..6c40610a65 100644 --- a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test +++ b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test @@ -32,14 +32,14 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified) +# 01. Create custom sub with expressions Z1=A+1, Y2=Z1+1, X3=Y2+1, S=X3+1 (X3 not notified) # 02. Create entity E1 with A=1 # 03. Update entity E1 with A=2.1 -# 04. Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1) +# 04. Dump accumulator and see two notifications (Z1:2, Y2:3, S:5) (Z1:3.1, Y2:4.1, S:6.1) # -echo "01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified)" +echo "01. Create custom sub with expressions Z1=A+1, Y2=Z1+1, X3=Y2+1, S=X3+1 (X3 not notified)" echo "=========================================================================================" payload='{ "subject": { @@ -54,8 +54,8 @@ payload='{ "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", "ngsi": { - "P2": { - "value": "${P1+1}", + "Y2": { + "value": "${Z1+1}", "type": "Calculated", "metadata": { "evalPriority": { @@ -64,7 +64,7 @@ payload='{ } } }, - "P1": { + "Z1": { "value": "${A+1}", "type": "Calculated", "metadata": { @@ -74,8 +74,8 @@ payload='{ } } }, - "P3": { - "value": "${P2+1}", + "X3": { + "value": "${Y2+1}", "type": "Calculated", "metadata": { "evalPriority": { @@ -85,12 +85,12 @@ payload='{ } }, "S": { - "value": "${P3+1}", + "value": "${X3+1}", "type": "Calculated" } } }, - "attrs": [ "P1", "P2", "S" ] + "attrs": [ "Z1", "Y2", "S" ] } }' orionCurl --url /v2/subscriptions --payload "$payload" @@ -126,7 +126,7 @@ echo echo -echo "Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1)" +echo "Dump accumulator and see two notifications (Z1:2, Y2:3, S:5) (Z1:3.1, Y2:4.1, S:6.1)" echo "====================================================================================" accumulatorDump echo @@ -134,7 +134,7 @@ echo --REGEXPECT-- -01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified) +01. Create custom sub with expressions Z1=A+1, Y2=Z1+1, X3=Y2+1, S=X3+1 (X3 not notified) ========================================================================================= HTTP/1.1 201 Created Date: REGEX(.*) @@ -162,7 +162,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1) +Dump accumulator and see two notifications (Z1:2, Y2:3, S:5) (Z1:3.1, Y2:4.1, S:6.1) ==================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / @@ -177,30 +177,30 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "P1": { + "S": { + "metadata": {}, + "type": "Calculated", + "value": 5 + }, + "Y2": { "metadata": { "evalPriority": { "type": "Number", - "value": 1 + "value": 2 } }, "type": "Calculated", - "value": 2 + "value": 3 }, - "P2": { + "Z1": { "metadata": { "evalPriority": { "type": "Number", - "value": 2 + "value": 1 } }, "type": "Calculated", - "value": 3 - }, - "S": { - "metadata": {}, - "type": "Calculated", - "value": 5 + "value": 2 }, "id": "E1", "type": "T" @@ -222,30 +222,30 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "P1": { + "S": { + "metadata": {}, + "type": "Calculated", + "value": 6.1 + }, + "Y2": { "metadata": { "evalPriority": { "type": "Number", - "value": 1 + "value": 2 } }, "type": "Calculated", - "value": 3.1 + "value": 4.1 }, - "P2": { + "Z1": { "metadata": { "evalPriority": { "type": "Number", - "value": 2 + "value": 1 } }, "type": "Calculated", - "value": 4.1 - }, - "S": { - "metadata": {}, - "type": "Calculated", - "value": 6.1 + "value": 3.1 }, "id": "E1", "type": "T" From 611ed6ac18238c9d91da0913720c026ae9371ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 10:56:38 +0200 Subject: [PATCH 06/12] FIX evalPriority CRUD --- src/lib/common/errorMessages.h | 4 + src/lib/common/limits.h | 2 +- src/lib/jsonParseV2/parseSubscription.cpp | 15 +- src/lib/ngsi/ContextAttribute.cpp | 7 +- .../eval_priority_crud_and_errors.test | 672 ++++++++++++++++++ 5 files changed, 693 insertions(+), 7 deletions(-) create mode 100644 test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test diff --git a/src/lib/common/errorMessages.h b/src/lib/common/errorMessages.h index a5f36a2f3a..0d170c7ec9 100644 --- a/src/lib/common/errorMessages.h +++ b/src/lib/common/errorMessages.h @@ -89,6 +89,10 @@ #define ERROR_DESC_BAD_REQUEST_WRONG_GEOJSON "Wrong GeoJson" #define ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF "only evalPriority metadata is allowed in ngsi field in custom notifications" +#define ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MUST_BE_A_NUMBER "evalPriority metadata must be a number" +#define ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MIN_ERROR "evalPriority metadata minimum priority is " STR(MIN_PRIORITY) +#define ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MAX_ERROR "evalPriority metadata maximum priority is " STR(MAX_PRIORITY) + #define ERROR_NOT_FOUND "NotFound" #define ERROR_DESC_NOT_FOUND_ENTITY "The requested entity has not been found. Check type and id" #define ERROR_DESC_NOT_FOUND_ENTITY_TYPE "Entity type not found" diff --git a/src/lib/common/limits.h b/src/lib/common/limits.h index 9a903f7953..0fd1283759 100644 --- a/src/lib/common/limits.h +++ b/src/lib/common/limits.h @@ -235,7 +235,7 @@ * */ #define MIN_PRIORITY 1 -#define MAX_PRIORITY LLONG_MAX +#define MAX_PRIORITY 100000 diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index 7dd54c9b65..77269af0aa 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -563,9 +563,18 @@ static std::string parseCustomPayload } else { - // priority must be a number - // priority must be between MIX_PRIORITY and MAX_PRIORITY - // FIXME: PR (include a .test to asses the checkings works) + if (caP->metadataVector[ix]->valueType != orion::ValueTypeNumber) + { + return badInput(ciP, ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MUST_BE_A_NUMBER); + } + if (caP->metadataVector[ix]->numberValue < MIN_PRIORITY) + { + return badInput(ciP, ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MIN_ERROR); + } + if (caP->metadataVector[ix]->numberValue > MAX_PRIORITY) + { + return badInput(ciP, ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MAX_ERROR); + } } } } diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 5fa1f9f78c..74bebd4e99 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1146,10 +1146,11 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi // // metadata (note that ngsi field in custom notifications doesn't include metadata) // - if (!renderNgsiField) - { + // FIXME PR: cleanup this + //if (!renderNgsiField) + //{ jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); - } + //} return jh.str(); } diff --git a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test new file mode 100644 index 0000000000..741a19dfc8 --- /dev/null +++ b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test @@ -0,0 +1,672 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Evaluation priority in expressions CRUD and errors + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with expressions with evalPriority as string, see error +# 02. Create custom sub with expressions with evalPriority 0, see error +# 03. Create custom sub with expressions with evalPriority 100001, see error +# 04. Create custom sub with expression with evalPriority 1 +# 05. Get sub to see evalPriority 1 +# 06. Update custom sub with expression with evalPriority as string, see error +# 07. Update custom sub with expression with evalPriority 0, see error +# 08. Update custom sub with expressions with evalPriority 100001, see error +# 09. Update custom sub with expression with evalPriority 2 +# 10. Get sub to see evalPriority 2 +# 11. Update custom sub with expression without evalPriority +# 12. Get sub to see no evalPriority +# 13. Update custom sub with expression with evalPriority 3 +# 14. Get sub to see evalPriority 3 +# + + +echo "01. Create custom sub with expressions with evalPriority as string, see error" +echo "=============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": "foo", + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create custom sub with expressions with evalPriority 0, see error" +echo "=====================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 0, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create custom sub with expressions with evalPriority 100001, see error" +echo "=======================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 100001, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "04. Create custom sub with expression with evalPriority 1" +echo "=========================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 1, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "05. Get sub to see evalPriority 1" +echo "=================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "06.Update custom sub with expression with evalPriority as string, see error" +echo "===========================================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": "foo", + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "07. Update custom sub with expression with evalPriority 0, see error" +echo "====================================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 0, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "08. Update custom sub with expressions with evalPriority 100001, see error" +echo "=======================================================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 100001, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "09. Update custom sub with expressions with evalPriority 2" +echo "==========================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 2, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "10. Get sub to see evalPriority 2" +echo "=================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "11. Update custom sub with expression without evalPriority" +echo "==========================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated" + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "12. Get sub to see no evalPriority" +echo "==================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + +echo "13. Update custom sub with expressions with evalPriority 3" +echo "==========================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 3, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "14. Get sub to see evalPriority 3" +echo "=================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +--REGEXPECT-- +01. Create custom sub with expressions with evalPriority as string, see error +============================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 77 + +{ + "description": "evalPriority metadata must be a number", + "error": "BadRequest" +} + + +02. Create custom sub with expressions with evalPriority 0, see error +===================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 82 + +{ + "description": "evalPriority metadata minimum priority is 1", + "error": "BadRequest" +} + + +03. Create custom sub with expressions with evalPriority 100001, see error +======================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 87 + +{ + "description": "evalPriority metadata maximum priority is 100000", + "error": "BadRequest" +} + + +04. Create custom sub with expression with evalPriority 1 +========================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Get sub to see evalPriority 1 +================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 411 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "ngsi": { + "S": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 1 + } + }, + "type": "Calculated", + "value": "${Z1+1}" + } + }, + "url": "http://127.0.0.1:9997/notify" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "id": "E1", + "type": "T" + } + ] + } +} + + +06.Update custom sub with expression with evalPriority as string, see error +=========================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 77 + +{ + "description": "evalPriority metadata must be a number", + "error": "BadRequest" +} + + +07. Update custom sub with expression with evalPriority 0, see error +==================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 82 + +{ + "description": "evalPriority metadata minimum priority is 1", + "error": "BadRequest" +} + + +08. Update custom sub with expressions with evalPriority 100001, see error +======================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 87 + +{ + "description": "evalPriority metadata maximum priority is 100000", + "error": "BadRequest" +} + + +09. Update custom sub with expressions with evalPriority 2 +========================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +10. Get sub to see evalPriority 2 +================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 411 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "ngsi": { + "S": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 2 + } + }, + "type": "Calculated", + "value": "${Z1+1}" + } + }, + "url": "http://127.0.0.1:9997/notify" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "id": "E1", + "type": "T" + } + ] + } +} + + +11. Update custom sub with expression without evalPriority +========================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +12. Get sub to see no evalPriority +================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 369 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "ngsi": { + "S": { + "metadata": {}, + "type": "Calculated", + "value": "${Z1+1}" + } + }, + "url": "http://127.0.0.1:9997/notify" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "id": "E1", + "type": "T" + } + ] + } +} + + +13. Update custom sub with expressions with evalPriority 3 +========================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +14. Get sub to see evalPriority 3 +================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 411 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "ngsi": { + "S": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 3 + } + }, + "type": "Calculated", + "value": "${Z1+1}" + } + }, + "url": "http://127.0.0.1:9997/notify" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "id": "E1", + "type": "T" + } + ] + } +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From f44ab2adac036e44c66d68c6372413407773b793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 11:26:51 +0200 Subject: [PATCH 07/12] ADD evalPriority to doc --- doc/manuals/orion-api.md | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index c4b2e0744d..4d8fdb382a 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -75,6 +75,7 @@ - [Additional considerations](#additional-considerations) - [JEXL Support](#jexl-support) - [JEXL usage example](#jexl-usage-example) + - [Evaluation priority](#evaluation-priority) - [Available Transformations](#available-transformations) - [`uppercase`](#uppercase) - [`lowercase`](#lowercase) @@ -746,6 +747,8 @@ type has an special semantic for Orion: * `DateTime` * `geo:json` +* `evalPriority`: used by expression evaluation. Have a look to [this specific section](#evaluation-priority) for details. + At the present moment `ignoreType` is supported only for geo-location types, this way allowing a mechanism to overcome the limit of only one geo-location per entity (more details in [Geospatial properties of entities](#geospatial-properties-of-entities) section). Support @@ -2507,6 +2510,64 @@ will trigger a notification like this: ] ``` +### Evaluation priority + +Each time an expression is evaluated, it is added to the expression context, so it can be reused by other expressions. However, by default Orion does not guarantee a given evaluation other. + +Thus, if we have this: + +``` +"httpCustom": { + ... + "ngsi": { + "A": { + "value": "${16|sqrt}", + "type": "Calculated" + }, + "B": { + "value": "${A/100}", + "type": "Calculated" + } + }, + "attrs": [ "B" ] +} +``` + +`B` could be set to the desired `0.04` (if `A` is evaluated before `B`) or to the underised `null` (if `B` is evaluated before `A`), randomly. + +In order to overcome this problem, the `evalPriority` metadata can be used to define the evaluation order. It works this way: + +* `evalPriority` metadata is a number from 1 (first evaluation) to 100000 (last evaluation) +* Expressions are evaluated in incresing order of priority +* In case of ties, Orion does not guarantee a particular evaluation order. Thus, expressions in the same priority level must be considered independent or bad things would happen +* If no `evalPriority` is set, default 100000 is used +* `evalPriority` only has a meaning in `notification.httpCustom.ngsi` in subscriptions. As metadata in regular entities (e.g. an entity created with `POST /v2/entities`) Orion doesn't implements any semantic. + +Using `evalPriority` the above example could be reformuled this way: + +``` +"httpCustom": { + ... + "ngsi": { + "A": { + "value": "${16|sqrt}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 1, + "type": "Number" + } + } + }, + "B": { + "value": "${A/100}", + "type": "Calculated" + } + }, + "attrs": [ "B" ] +} +``` + ### Available Transformations #### `uppercase` From 5a3a04efa46c758c106d31a9d4824a7ff9b8ca29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 11:27:00 +0200 Subject: [PATCH 08/12] ADD Changelog entry --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 4c5c912cb9..fa20464f70 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ +- Fix: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) - Fix: wrong date values should not allowed in subscription's expires field (#4541) - Fix: do not raise DB alarm in case of wrong GeoJSON in client request \ No newline at end of file From 23a905f6c9591472110483cdfa17f1e67612bd12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 11:39:47 +0200 Subject: [PATCH 09/12] FIX metadata rendering logic in custom ngsi case --- src/lib/ngsi/ContextAttribute.cpp | 11 +++++------ .../eval_priority_crud_and_errors.test | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 74bebd4e99..58675b6edd 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1144,13 +1144,12 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi filterAndOrderMetadata(metadataFilter, &orderedMetadata); // - // metadata (note that ngsi field in custom notifications doesn't include metadata) + // metadata (note that ngsi field in custom notifications avoids empety metadata array, i.e. "metadata": {}) // - // FIXME PR: cleanup this - //if (!renderNgsiField) - //{ - jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); - //} + if ((!renderNgsiField) || (metadataVector.size() > 0)) + { + jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); + } return jh.str(); } diff --git a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test index 741a19dfc8..a875c8ee21 100644 --- a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test +++ b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test @@ -575,7 +575,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 355 { "id": "REGEX([0-9a-f]{24})", @@ -586,7 +586,6 @@ Content-Length: 369 "httpCustom": { "ngsi": { "S": { - "metadata": {}, "type": "Calculated", "value": "${Z1+1}" } From 553342ac002b1105fb383c2276c0d2aab9efd06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 12:09:14 +0200 Subject: [PATCH 10/12] FIX refactor --- src/lib/common/JsonHelper.cpp | 15 ++++----------- src/lib/common/JsonHelper.h | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/lib/common/JsonHelper.cpp b/src/lib/common/JsonHelper.cpp index 9adc597fc2..65b0e3be64 100644 --- a/src/lib/common/JsonHelper.cpp +++ b/src/lib/common/JsonHelper.cpp @@ -424,20 +424,13 @@ void JsonVectorHelper::addNull(void) + /* **************************************************************************** * * JsonVectorHelper::str - -* FIXME PR: bool closed probably unneded in vectors */ -std::string JsonVectorHelper::str(bool closed) +std::string JsonVectorHelper::str(void) { - // closed == false used in ExprContext logic - if (closed) - { - return ss + ']'; - } - else - { - return ss; - } + ss += ']'; + return ss; } diff --git a/src/lib/common/JsonHelper.h b/src/lib/common/JsonHelper.h index 5b2633e0ca..8f01adcf7b 100644 --- a/src/lib/common/JsonHelper.h +++ b/src/lib/common/JsonHelper.h @@ -67,7 +67,7 @@ class JsonVectorHelper void addNull(void); - std::string str(bool closed = true); + std::string str(void); private: std::string ss; From 1b9b33225ae4d0fb3eded8676c5a0d691a18d00f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 12:36:35 +0200 Subject: [PATCH 11/12] Apply suggestions from code review --- doc/manuals/orion-api.md | 4 ++-- src/lib/ngsi/ContextAttribute.cpp | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 4d8fdb382a..a32334b2aa 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2533,7 +2533,7 @@ Thus, if we have this: } ``` -`B` could be set to the desired `0.04` (if `A` is evaluated before `B`) or to the underised `null` (if `B` is evaluated before `A`), randomly. +in the resulting notification `B` could be set to the desired `0.04` (if `A` is evaluated before `B`) or to the underised `null` (if `B` is evaluated before `A`), randomly. In order to overcome this problem, the `evalPriority` metadata can be used to define the evaluation order. It works this way: @@ -2541,7 +2541,7 @@ In order to overcome this problem, the `evalPriority` metadata can be used to de * Expressions are evaluated in incresing order of priority * In case of ties, Orion does not guarantee a particular evaluation order. Thus, expressions in the same priority level must be considered independent or bad things would happen * If no `evalPriority` is set, default 100000 is used -* `evalPriority` only has a meaning in `notification.httpCustom.ngsi` in subscriptions. As metadata in regular entities (e.g. an entity created with `POST /v2/entities`) Orion doesn't implements any semantic. +* `evalPriority` only has a meaning in `notification.httpCustom.ngsi` in subscriptions. As metadata in regular entities (e.g. an entity created with `POST /v2/entities`) Orion doesn't implements any semantic for it Using `evalPriority` the above example could be reformuled this way: diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 58675b6edd..0fc57e3053 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1144,11 +1144,11 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi filterAndOrderMetadata(metadataFilter, &orderedMetadata); // - // metadata (note that ngsi field in custom notifications avoids empety metadata array, i.e. "metadata": {}) + // metadata (note that ngsi field in custom notifications avoids empty metadata array, i.e. "metadata": {}) // if ((!renderNgsiField) || (metadataVector.size() > 0)) { - jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); + jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); } return jh.str(); @@ -1185,9 +1185,6 @@ std::string ContextAttribute::toJsonValue(ExprContextObject* exprContextObjectP) } else if (valueType == orion::ValueTypeString) { - //std::string out = "\""; - //out += toJsonString(stringValue); - //out += '"'; return smartStringValue(stringValue, exprContextObjectP, "null"); } else if (valueType == orion::ValueTypeBoolean) From 4698f6474ee1bbce113f4d369ce1cf8707851150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 13:33:04 +0200 Subject: [PATCH 12/12] Update doc/manuals/orion-api.md Co-authored-by: mapedraza <40356341+mapedraza@users.noreply.github.com> --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index a32334b2aa..1c1bbc1961 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2539,7 +2539,7 @@ In order to overcome this problem, the `evalPriority` metadata can be used to de * `evalPriority` metadata is a number from 1 (first evaluation) to 100000 (last evaluation) * Expressions are evaluated in incresing order of priority -* In case of ties, Orion does not guarantee a particular evaluation order. Thus, expressions in the same priority level must be considered independent or bad things would happen +* In case of ties, Orion does not guarantee a particular evaluation order. Thus, expressions in the same priority level must be considered independent, and expressions using other attributes with the same or lower priority would result in unexpected values. * If no `evalPriority` is set, default 100000 is used * `evalPriority` only has a meaning in `notification.httpCustom.ngsi` in subscriptions. As metadata in regular entities (e.g. an entity created with `POST /v2/entities`) Orion doesn't implements any semantic for it