diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 134c80ddc9..2728600db4 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.2.0) (#4004) +- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.3.0) (#4004) - Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) - Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004) - Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index b0e5027b8a..6202177dbb 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -175,7 +175,7 @@ rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - bash /opt/fiware-orion/get_cjexl.sh 0.2.0 $REPO_ACCESS_TOKEN + bash /opt/fiware-orion/get_cjexl.sh 0.3.0 $REPO_ACCESS_TOKEN fi if [ -n "${branch}" ]; then diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 0d81391493..6cceed1a4a 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -79,27 +79,34 @@ - [`uppercase`](#uppercase) - [`lowercase`](#lowercase) - [`split`](#split) - - [`indexOf`](#indexOf) + - [`indexOf`](#indexof) - [`len`](#len) - [`trim`](#trim) - [`substring`](#substring) - [`includes`](#includes) - [`isNaN`](#isNaN) - - [`parseInt`](#parseInt) - - [`parseFloat`](#parseFloat) + - [`parseInt`](#parseint) + - [`parseFloat`](#parsefloat) - [`typeOf`](#typeOf) - - [`toString`](#toString) + - [`toString`](#tostring) + - [`toJson`](#tojson) - [`floor`](#floor) - [`ceil`](#ceil) - [`round`](#round) - - [`toFixed`](#toFixed) + - [`toFixed`](#tofixed) - [`log`](#log) - [`log10`](#log10) - [`log2`](#log2) - [`sqrt`](#sqrt) - - [`replaceStr`](#replaceStr) + - [`replaceStr`](#replacestr) + - [`replaceRegex`](#replaceregex) + - [`matchRegex`](#matchregex) - [`mapper`](#mapper) - [`thMapper`](#thmapper) + - [`values`](#values) + - [`keys`](#keys) + - [`arrSum`](#arrsum) + - [`arrAvg`](#arravg) - [Failsafe cases](#failsafe-cases) - [Known limitations](#known-limitations) - [Oneshot Subscriptions](#oneshot-subscriptions) @@ -2775,6 +2782,24 @@ results in "23" ``` +#### toJson + +Convert the input string to a JSON document. If the string is not a valid JSON, it returns `null`. + +Extra arguments: none. + +Example (being context `{"c": "[1,2]"}`): + +``` +c|toJson +``` + +results in + +``` +[1, 2] +``` + #### floor Return the closest lower integer of a given number. @@ -2939,6 +2964,44 @@ results in "fuubar" ``` +#### replaceRegex + +Replace tokens matching a given regex in an input string by another string. + +Extra arguments: +* Regex to match tokens to replace +* Destination string to replace + +Example (being context `{"c": "aba1234aba786aba"}`): + +``` +c|replaceRegex('\d+','X') +``` + +results in + +``` +"abaXabaXaba" +``` + +#### matchRegex + +Returns an array of tokens matching a given regular expression in input string. In the case of invalid regex, it returns `null`. If no matches are found, it returns the empty array (`[]`). + +Extra arguments: regular expression. + +Example (being context `{"c": "abc1234fgh897hyt"}`): + +``` +c|matchRegex('\d+`) +``` + +results in + +``` +["1234, "897"]] +``` + #### mapper Returns a value among several choices based in one to one mapping. This function is based in an array of *values* and an array of *choices* (which length is exactly the same). Thus, if the input value is equal to the *i*-th item of *values*, then *i*-th item of *choices* is returned. @@ -2983,6 +3046,78 @@ results in "medium" ``` +#### values + +Returns an array with the values of the keys of a given object (or `null` if input is not an object). + +Extra arguments: none + +Example (being context `{"c": {"x": 1, "y": "foo"}}`): + +``` +c|values +``` + +results in + +``` +[1,"foo"] +``` + +#### keys + +Returns an array with the keys of a given object (or `null` if input is not an object). + +Extra arguments: none + +Example (being context `{"c": {"x": 1, "y": "foo"}}`): + +``` +c|keys +``` + +results in + +``` +["x","y"] +``` + +#### arrSum + +Returns the sum of the elements of an array (or `null` if the input in an array or the array containts some not numberic item). + +Extra arguments: none + +Example (being context `{"c": [1, 5]}`): + +``` +c|arrSum +``` + +results in + +``` +6 +``` + +#### arrAvg + +Returns the average of the elements of an array (or `null` if the input in an array or the array containts some not numberic item). + +Extra arguments: none + +Example (being context `{"c": [1, 5]}`): + +``` +c|arrAvg +``` + +results in + +``` +3 +``` + ### Failsafe cases As failsafe behaviour, evaluation returns `null` in the following cases: diff --git a/docker/Dockerfile b/docker/Dockerfile index e6c7846b56..463a4e3034 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.2.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.3.0 ${REPO_ACCESS_TOKEN} && \ make && \ make install && \ # reduce size of installed binaries diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 9e35eb3aa9..736f2dbf3e 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -110,7 +110,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.2.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.3.0 ${REPO_ACCESS_TOKEN} && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test new file mode 100644 index 0000000000..baa991c929 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test @@ -0,0 +1,220 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +JEXL basic expression in custom notification (binary map accumulation) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with several attributes with transformations +# 02. Create entity E1 +# 03. Update entity E1 +# 05. Dump accumulator and see two expected transformations +# + + +echo "01. Create custom sub with several attributes with transformations" +echo "==================================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "occupied": { + "value": "${slots|values|arrSum}", + "type": "Calculated" + }, + "free": { + "value": "${(slots|values|len)-(slots|values|arrSum)}", + "type": "Calculated" + } + } + }, + "attrs": [ "free", "occupied" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + +#"value": "${{count:count.count+1,sum:count.sum+speed|split('\'' '\'')[0]|parseInt}}", + +echo "02. Create entity E1" +echo "====================" +payload='{ + "id": "E1", + "type": "T", + "slots": { + "value": { + "s1": 0, + "s2": 1, + "s3": 0, + "s4": 1, + "s5": 0 + }, + "type": "Text" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1" +echo "====================" +payload='{ + "slots": { + "value": { + "s1": 0, + "s2": 1, + "s3": 1, + "s4": 1, + "s5": 0 + }, + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see two expected transformations" +echo "=========================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with several attributes with transformations +================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 +==================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 +==================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see two expected transformations +========================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 187 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "free": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "id": "E1", + "occupied": { + "metadata": {}, + "type": "Calculated", + "value": 2 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 187 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "free": { + "metadata": {}, + "type": "Calculated", + "value": 2 + }, + "id": "E1", + "occupied": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index 977301f192..12646463e7 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -33,7 +33,7 @@ accumulatorStart --pretty-print # # 01. Create custom sub using all transformations -# 02. Create entity E1 with A to V attributes +# 02. Create entity E1 with A to AD attributes # 03. Dump accumulator and see expected notification # @@ -73,7 +73,15 @@ payload='{ "S", "T", "U", - "V" + "V", + "W", + "X", + "Y", + "Z", + "AA", + "AB", + "AC", + "AD" ], "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", @@ -136,7 +144,7 @@ payload='{ }, "O": { "type": "Text", - "value": "${Z||'\''Is null'\''}" + "value": "${ZZ||'\''Is null'\''}" }, "P": { "type": "Number", @@ -165,6 +173,38 @@ payload='{ "V": { "type": "Text", "value": "${V|thMapper([1,2],['\''low'\'','\''medium'\'','\''high'\''])}" + }, + "W": { + "type": "StructuredValue", + "value": "${W|toJson}" + }, + "X": { + "type": "Text", + "value": "${X|replaceRegex('\''[0-9]+'\'','\''x'\'')}" + }, + "Y": { + "type": "Text", + "value": "${Y|matchRegex('\''[0-9]+'\'')}" + }, + "Z": { + "type": "StructuredValue", + "value": "${Z|values}" + }, + "AA": { + "type": "StructuredValue", + "value": "${AA|keys}" + }, + "AB": { + "type": "Number", + "value": "${AB|arrSum}" + }, + "AC": { + "type": "Number", + "value": "${AC|arrAvg}" + }, + "AD": { + "type": "Number", + "value": "${AD|len}" } } } @@ -175,8 +215,8 @@ echo echo -echo "02. Create entity E1 with A to V attributes" -echo "===========================================" +echo "02. Create entity E1 with A to AD attributes" +echo "============================================" payload='{ "id": "E1", "type": "T", @@ -272,6 +312,38 @@ payload='{ "V": { "type": "Number", "value": 1.5 + }, + "W": { + "type": "Text", + "value": "[1,2,3]" + }, + "X": { + "type": "Text", + "value": "aba1234aba782" + }, + "Y": { + "type": "Text", + "value": "aba1234aba782" + }, + "Z": { + "type": "StructuredValue", + "value": {"x": 1, "y": {"y1": 2, "y2": 3}} + }, + "AA": { + "type": "StructuredValue", + "value": {"x": 1, "y": {"y1": 2, "y2": 3}} + }, + "AB": { + "type": "StructuredValue", + "value": [1, 2, 3] + }, + "AC": { + "type": "StructuredValue", + "value": [1, 2, 3] + }, + "AD": { + "type": "StructuredValue", + "value": [1, 2, 3] } }' orionCurl --url /v2/entities --payload "$payload" @@ -297,8 +369,8 @@ Content-Length: 0 -02. Create entity E1 with A to V attributes -=========================================== +02. Create entity E1 with A to AD attributes +============================================ HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -311,7 +383,7 @@ Content-Length: 0 ================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 1178 +Content-Length: 1627 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -327,6 +399,29 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Text", "value": "NA3" }, + "AA": { + "metadata": {}, + "type": "StructuredValue", + "value": [ + "x", + "y" + ] + }, + "AB": { + "metadata": {}, + "type": "Number", + "value": 6 + }, + "AC": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "AD": { + "metadata": {}, + "type": "Number", + "value": 3 + }, "B": { "metadata": {}, "type": "Text", @@ -435,6 +530,39 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Text", "value": "medium" }, + "W": { + "metadata": {}, + "type": "StructuredValue", + "value": [ + 1, + 2, + 3 + ] + }, + "X": { + "metadata": {}, + "type": "Text", + "value": "abaxabax" + }, + "Y": { + "metadata": {}, + "type": "Text", + "value": [ + "1234", + "782" + ] + }, + "Z": { + "metadata": {}, + "type": "StructuredValue", + "value": [ + 1, + { + "y1": 2, + "y2": 3 + } + ] + }, "id": "E1", "type": "T" }