diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 4fbc8d5ab0..50493f53c1 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -398,11 +398,12 @@ the following will be used (note that "%25" is the encoding for "%"). GET /v2/entities/E%253C01%253E ``` -There are some exception cases in which the above restrictions do not apply. In particular, in the following fields: +There are some exception cases in which the above restrictions do not apply. In particular, in the following cases: * URL parameter `q` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `mq` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `georel` and `coords` allow `;` +* Within `ngsi` (i.e. `id`, `type` and attribute values) in [NGSI Payload patching](#ngsi-payload-patching) (to support characters used in the JEXL expression syntax) * Whichever attribute value which uses `TextUnrestricted` as attribute type (see [Special Attribute Types](#special-attribute-types) section) ## Identifiers syntax restrictions diff --git a/src/lib/jsonParseV2/parseContextAttribute.cpp b/src/lib/jsonParseV2/parseContextAttribute.cpp index 751dfb66d3..caf6ac190a 100644 --- a/src/lib/jsonParseV2/parseContextAttribute.cpp +++ b/src/lib/jsonParseV2/parseContextAttribute.cpp @@ -254,7 +254,8 @@ std::string parseContextAttribute ConnectionInfo* ciP, const rapidjson::Value::ConstMemberIterator& iter, ContextAttribute* caP, - bool checkAttrSpecialTypes + bool checkAttrSpecialTypes, + bool relaxForbiddenCheck ) { std::string name = iter->name.GetString(); @@ -383,7 +384,7 @@ std::string parseContextAttribute caP->type = (compoundVector)? defaultType(orion::ValueTypeVector) : defaultType(caP->valueType); } - std::string r = caP->check(ciP->apiVersion, ciP->requestType); + std::string r = caP->check(ciP->apiVersion, ciP->requestType, relaxForbiddenCheck); if (r != "OK") { alarmMgr.badInput(clientIp, "JSON Parse Error", r); diff --git a/src/lib/jsonParseV2/parseContextAttribute.h b/src/lib/jsonParseV2/parseContextAttribute.h index e3210e9403..2ab0ed39e6 100644 --- a/src/lib/jsonParseV2/parseContextAttribute.h +++ b/src/lib/jsonParseV2/parseContextAttribute.h @@ -42,7 +42,8 @@ extern std::string parseContextAttribute ConnectionInfo* ciP, const rapidjson::Value::ConstMemberIterator& iter, ContextAttribute* caP, - bool checkAttrSpecialTypes + bool checkAttrSpecialTypes, + bool relaxForbiddenCheck = false ); diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index cea25d5b52..5fbe2faa38 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -525,11 +525,6 @@ static std::string parseCustomPayload } ngsi->id = iter->value.GetString(); - - if (forbiddenIdChars(V2, ngsi->id.c_str(), "")) - { - return badInput(ciP, ERROR_DESC_BAD_REQUEST_INVALID_CHAR_ENTID); - } } else if (name == "type") { @@ -539,11 +534,6 @@ static std::string parseCustomPayload } ngsi->type = iter->value.GetString(); - - if (forbiddenIdChars(V2, ngsi->type.c_str(), "")) - { - return badInput(ciP, ERROR_DESC_BAD_REQUEST_INVALID_CHAR_ENTTYPE); - } } else // attribute { @@ -551,7 +541,9 @@ static std::string parseCustomPayload ngsi->attributeVector.push_back(caP); - std::string r = parseContextAttribute(ciP, iter, caP, false); + // Note we are using relaxForbiddenCheck true in this case, as JEXL expressions typically use forbidden + // chars and we don't want to fail in that case + std::string r = parseContextAttribute(ciP, iter, caP, false, true); if (r == "max deep reached") { diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 77e86480a9..79ce3f57fc 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1363,7 +1363,7 @@ void ContextAttribute::addToContext(ExprContextObject* exprContextObjectP, bool * * ContextAttribute::check - */ -std::string ContextAttribute::check(ApiVersion apiVersion, RequestType requestType) +std::string ContextAttribute::check(ApiVersion apiVersion, RequestType requestType, bool relaxForbiddenCheck) { size_t len; char errorMsg[128]; @@ -1414,14 +1414,14 @@ std::string ContextAttribute::check(ApiVersion apiVersion, RequestType requestTy return "Invalid characters in attribute type"; } - if ((compoundValueP != NULL) && (compoundValueP->childV.size() != 0) && (type != TEXT_UNRESTRICTED_TYPE)) + if ((!relaxForbiddenCheck) && (compoundValueP != NULL) && (compoundValueP->childV.size() != 0) && (type != TEXT_UNRESTRICTED_TYPE)) { return compoundValueP->check(""); } if (valueType == orion::ValueTypeString) { - if ((type != TEXT_UNRESTRICTED_TYPE) && (forbiddenChars(stringValue.c_str()))) + if ((!relaxForbiddenCheck) && (type != TEXT_UNRESTRICTED_TYPE) && (forbiddenChars(stringValue.c_str()))) { alarmMgr.badInput(clientIp, "found a forbidden character in the value of an attribute", stringValue); return "Invalid characters in attribute value"; diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index caa36cbfe5..f242bf101d 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -134,7 +134,7 @@ typedef struct ContextAttribute /* Helper method to be use in some places wher '%s' is needed */ std::string getValue(void) const; - std::string check(ApiVersion apiVersion, RequestType requestType); + std::string check(ApiVersion apiVersion, RequestType requestType, bool relaxForbiddenCheck = false); ContextAttribute* clone(); bool compoundItemExists(const std::string& compoundPath, orion::CompoundValueNode** compoundItemPP = NULL); diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test index c20beb102f..0a3bb36e7d 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test @@ -57,7 +57,7 @@ payload='{ "ngsi": { "cal": { "value": "${a[.b<5]}", - "type": "TextUnrestricted" + "type": "Calculated" } } }, @@ -179,7 +179,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) ===================================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 151 +Content-Length: 145 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -192,7 +192,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": [ { "b": 1 @@ -211,7 +211,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 138 +Content-Length: 132 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -224,7 +224,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": null }, "id": "E1", @@ -236,7 +236,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 138 +Content-Length: 132 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -249,7 +249,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": null }, "id": "E1", @@ -261,7 +261,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 150 +Content-Length: 144 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -274,7 +274,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": [ { "b": -1, diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test new file mode 100644 index 0000000000..fc128a4e4d --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test @@ -0,0 +1,197 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using transformation in id and type) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression id=S|substring(0,2), type=S|substring(3,5) +# 02. Create entity E with S=E1:T1, A:1 +# 03. Update entity E with S=E2:T2, A:2 +# 04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2) +# + + +echo "01. Create custom sub with custom expression id=S|substring(0,2), type=S|substring(3,5)" +echo "=======================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "id": "${S|substring(0,2)}", + "type": "${S|substring(3,5)}" + } + }, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E with S=E1:T1, A:1" +echo "=====================================" +payload='{ + "id": "E", + "type": "T", + "S": { + "value": "E1:T1", + "type": "Text" + }, + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E with S=E2:T2, A:2" +echo "=====================================" +payload='{ + "S": { + "value": "E2:T2", + "type": "Text" + }, + "A": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2)" +echo "=================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression id=S|substring(0,2), type=S|substring(3,5) +======================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E with S=E1:T1, A:1 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. Update entity E with S=E2:T2, A:2 +===================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2) +================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 124 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "id": "E1", + "type": "T1" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 124 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "id": "E2", + "type": "T2" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test index 85075dc93a..5616763fde 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test @@ -56,7 +56,7 @@ payload='{ "ngsi": { "includes": { "value": "${S|includes(R)}", - "type": "TextUnrestricted" + "type": "Boolean" } } }, @@ -169,7 +169,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) =========================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 143 +Content-Length: 134 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -183,7 +183,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "id": "E1", "includes": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Boolean", "value": true }, "type": "T" @@ -194,7 +194,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 144 +Content-Length: 135 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -208,7 +208,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "id": "E1", "includes": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Boolean", "value": false }, "type": "T" @@ -219,7 +219,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 143 +Content-Length: 134 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -233,7 +233,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "id": "E1", "includes": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Boolean", "value": true }, "type": "T"