From da9c9b51805a542ab2e6e590408420b0b78ac226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 16:41:04 +0200 Subject: [PATCH 1/2] ADD documentation --- CHANGES_NEXT_RELEASE | 9 +- doc/manuals/admin/build_source.md | 2 + doc/manuals/admin/statistics.md | 45 ++- doc/manuals/devel/sourceCode.md | 6 + doc/manuals/orion-api.md | 510 +++++++++++++++++++++++++++++- 5 files changed, 527 insertions(+), 45 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index c44ac67e3f..21a630772a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,7 +1,6 @@ -<<<<<<< HEAD -- Add: expression context build and evaluation counter in timing section in GET /statistics (#4004) -- Fix: lighter operation to get databases list from MongoDB (#4517) -======= +- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.1.0) (#4004) +- Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) +- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviur more consistent - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) ->>>>>>> master + diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index 64064030e4..3292e71d32 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -4,6 +4,8 @@ Orion Context Broker reference distribution is Debian 12. This doesn't mean that You can also have a look to [3.1 Building in not official distributions](../../../docker/README.md#31-building-in-not-official-distributions) section in the Docker documentation to check how to build Docker containers images in distributions other than the official one. +*NOTE:* the build process described in this document does not include the cjexl library, as it is considered optional from the point of view of the basic building process. + ## Debian 12 (officially supported) The Orion Context Broker uses the following libraries as build dependencies: diff --git a/doc/manuals/admin/statistics.md b/doc/manuals/admin/statistics.md index 7a0eb4f487..a44bafcb74 100644 --- a/doc/manuals/admin/statistics.md +++ b/doc/manuals/admin/statistics.md @@ -129,28 +129,22 @@ Provides timing information, i.e. the time that CB passes executing in different "timing": { "accumulated": { "jsonV1Parse": 7.860908311, - "mongoBackend": 416.796091597, - "mongoReadWait": 4656.924425628, - "mongoWriteWait": 259.347915990, - "mongoCommandWait": 0.514811318, - "exprBasicCtxBld": FIXME PR, - "exprBasicEval": FIXME PR, - "exprJexlCtxBld": FIXME PR, - "exprJexlEval": FIXME PR, - "render": 108.162782114, - "total": 6476.593504743 - }, + "jsonV2Parse": 120.680244446, + "mongoBackend": 12778.52734375, + "mongoReadWait": 7532.301757812, + "mongoWriteWait": 3619.282226562, + "mongoCommandWait": 0.120559767, + "exprJexlCtxBld": 27.092681885, + "exprJexlEval": 124.217208862, + "render": 44.540554047, + "total": 25051.384765625 + }, "last": { - "mongoBackend": 0.014752309, - "mongoReadWait": 0.012018445, - "mongoWriteWait": 0.000574611, - "exprBasicCtxBld": FIXME PR, - "exprBasicEval": FIXME PR, - "exprJexlCtxBld": FIXME PR, - "exprJexlEval": FIXME PR, - "render": 0.000019136, - "total": 0.015148915 - } + "mongoBackend": 0.003775352, + "mongoReadWait": 0.0013743, + "render": 0.000286864, + "total": 0.00440685 + } } ... } @@ -164,8 +158,6 @@ The block includes two main sections: The particular counters are as follows: -FIXME PR: explain expr* fields - * `total`: processing time for the whole request, excluding the time that the HTTP library takes for request/response dispatching (pseudo end-to-end time) * `jsonV1Parse`: time passed in NGSIv1 JSON parsing module (pseudo self-time) @@ -177,10 +169,15 @@ FIXME PR: explain expr* fields `last` includes the accumulation for all of them. In the case of mongoReadWait, only the time used to get the results cursor is taken into account, but not the time to process cursors results (which is time that belongs to mongoBackend counters). +* `exprJexlCtxBld`: time passed building context for custom notification expression evaluation (see [macro substitution](../orion-api.md#macro-substitution) and [JEXL support](../orion-api.md#jexl-support)) +* `exprJexlEval`: time passed evaluating custom notification expressions (see [macro substitution](../orion-api.md#macro-substitution) and [JEXL support](../orion-api.md#jexl-support)) + +*NOTE*: if Orion binary is build without using cjexl and only basic replacement is available, then `exprBasicCtxtBld` and `exprBasicEval` +fields appear instead of `exprJexlCtxBld` and `exprJexlEval`. Times are measured from the point in time in which a particular thread request starts using a module until it finishes using it. Thus, if the thread is stopped for some reason (e.g. the kernel decides to give priority to another thread based on its -scheculing policy) the time that the thread was sleeping, waiting to execute again is included in the measurement and thus, the measurement is not accurate. That is why we say *pseudo* selt/end-to-end time. However, +scheculing policy) the time that the thread was sleeping, waiting to execute again is included in the measurement and thus, the measurement is not accurate. That is why we say *pseudo* self/end-to-end time. However, under low load conditions this situation is not expected to have a significant impact. ### NotifQueue block diff --git a/doc/manuals/devel/sourceCode.md b/doc/manuals/devel/sourceCode.md index c9227740b0..951e8777f8 100644 --- a/doc/manuals/devel/sourceCode.md +++ b/doc/manuals/devel/sourceCode.md @@ -24,6 +24,7 @@ * [src/lib/cache/](#srclibcache) (Subscription cache implementation) * [src/lib/logSummary/](#srcliblogsummary) (Log Summary implementation) * [src/lib/metricsMgr/](#srclibmetricsmgr) (Metrics Manager implementation) +* [src/lib/expressions/](#seribexpressions) (Custom notification expressions support) ## src/app/contextBroker/ The main program is found in `contextBroker.cpp` and its purpose it to: @@ -551,3 +552,8 @@ This Metrics Manager resides in the library **metricsMgr**. For information about the metrics, please refer to [this document](../admin/metrics_api.md). [Top](#top) + +## src/lib/expressions/ +Provides support to the [macro substition logic used by custom notifications](../orion-api.md#macro-substitution). This library provides an abstraction for expression evaluation, providing two implementations: JEXL based and basic replacement based (the implementation to use is choosen at building time, based on the availability of the cjex library). + +[Top](#top) \ No newline at end of file diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 5e8e6e9bef..ea23b5f764 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -73,6 +73,32 @@ - [NGSI payload patching](#ngsi-payload-patching) - [Omitting payload](#omitting-payload) - [Additional considerations](#additional-considerations) + - [JEXL Support](#jexl-support) + - [Available Transformations](#available-transformations) + - [`uppercase`](#uppercase) + - [`lowercase`](#lowercase) + - [`split`](#split) + - [`indexOf`](#indexOf) + - [`len`](#len) + - [`trim`](#trim) + - [`substring`](#substring) + - [`includes`](#includes) + - [`isNaN`](#isNaN) + - [`parseInt`](#parseInt) + - [`parseFloat`](#parseFloat) + - [`typeOf`](#typeOf) + - [`toString`](#toString) + - [`floor`](#floor) + - [`ceil`](#ceil) + - [`round`](#round) + - [`toFixed`](#toFixed) + - [`log`](#log) + - [`log10`](#log10) + - [`log2`](#log2) + - [`sqrt`](#sqrt) + - [`replaceStr`](#replaceStr) + - [Failsafe cases](#failsafe-cases) + - [Known limitations](#known-limitations) - [Oneshot Subscriptions](#oneshot-subscriptions) - [Covered Subscriptions](#covered-subscriptions) - [Subscriptions based in alteration type](#subscriptions-based-in-alteration-type) @@ -2022,23 +2048,22 @@ In case of `mqttCustom`: * `payload`, `json` and `ngsi` (all them payload related fields) * `topic` -Macro substitution for templates is based on the syntax `${..}`. In particular: +Macro substitution for templates is based on the syntax `${}`. The support to JEXL +is explained in [JEXL Support](#jexl-support) section. The following identifiers are included in +the context evaluated by the JEXL expression: -* `${id}` is replaced by the `id` of the entity -* `${type}` is replaced by the `type` of the entity -* `${service}` is replaced by the service (i.e. `fiware-service` header value) in the +* `id`: for the `id` of the entity +* `type`: for the `type` of the entity +* `service`: for the service (i.e. `fiware-service` header value) in the update request triggering the subscription. -* `${servicePath}` is replaced by the service path (i.e. `fiware-servicepath` header value) in the +* `servicePath`: for the service path (i.e. `fiware-servicepath` header value) in the update request triggering the subscription. -* `${authToken}` is replaced by the authorization token (i.e. `x-auth-token` header value) in the +* `authToken: for the authorization token (i.e. `x-auth-token` header value) in the update request triggering the subscription. -* Any other `${token}` is replaced by the value of the attribute whose name is `token` or with - an empty string if the attribute is not included in the notification. If the value is a number, - a bool or null then its string representation is used. If the value is a JSON array or object - then its JSON representation as string is used. +* All the attributes in the entity triggering the notification (included in the update triggering the notification or not) -In the rare case an attribute was named in the same way of the `${service}`, `${servicePath}` or -`${authToken}` (e.g. an attribute which name is `service`) then the attribute value takes precedence. +In the rare case an attribute was named in the same way of the `service`, `servicePath` or +`authToken` (e.g. an attribute which name is `service`) then the attribute value takes precedence. Example: @@ -2296,6 +2321,463 @@ Some considerations to take into account when using custom notifications: (i.e. `ngsi` field) then `Ngsiv2-AttrsFormat: normalized` is used, as in a regular notification (given that the notification format is actually the same). +## JEXL Support + +Orion Context Broker supports [JEXL expressions](https://github.com/TomFrost/Jexl) in custom notification [macro replacement](#macro-substitution). Thus, subscriptions like this can be defined: + +``` +"httpCustom": { + ... + "ngsi": { + "relativeHumidity": { + "value": "${humidity/100}", + "type": "Calculated" + } + } +} +``` + +So, if a given update sets entity `humidity` attribute to `84.4` then the notification will include a `relativeHumidity` attribute with value `0.844`. + +A particular case of expressions are the ones in which the expression is a given context identifier, without an actual expression using it. For instance: + +``` +"httpCustom": { + ... + "ngsi": { + "originalHumidity": { + "value": "${humidity}", + "type": "Calculated" + } + } +} +``` + +We also refers to this case as *basic replacement*. + +An useful resource to test JEXL expressions is the [JEXL playground](https://czosel.github.io/jexl-playground). However, take into account the differences between the original JEXL implementation in JavaScript and the one included in Orion, described in the [known limitations](#known-limitations) section. + +Orion relies on cjexl library to provide this functionality. If Orion binary is build without using cjexl, then only basic replacement functionality is available. + +### Available Transformations + +#### `uppercase` + +Convert a string into uppercase. + +Extra arguments: none + +Example (being context `{"c": "fooBAR"}`): + +``` +c|uppercase +``` + +results in + +``` +"FOOBAR" +``` + +#### lowercase + +Convert a string into lowercase. + +Extra arguments: none + +Example (being context `{"c": "fooBAR"}`): + +``` +c|lowercase +``` + +results in + +``` +"foobar" +``` + +#### split + +Split the input string into array items. + +Extra arguments: delimiter to use for the split. + +Example (being context `{"c": "foo,bar,zzz"}`): + +``` +c|split(',') +``` + +results in + +``` +[ "foo", "bar", "zzz" ] +``` + +#### indexOf + +Provides the position of a given string within the input string. In the string is not found, returns `null`. + +Extra arguments: the input string to search. + +Note this function doesn't work if the input is an array (it only works for strings). + +Example (being context `{"c": "fooxybar"}`): + +``` +c|indexOf('xy') +``` + +results in + +``` +3 +``` + +#### len + +Provides the length of a string. + +Extra arguments: none. + +Note this function doesn't work if the input is an array (it only works for strings). + +Example (being context `{"c": "foobar"}`): + +``` +c|len +``` + +results in + +``` +6 +``` + +#### trim + +Removes starting and trailing whitespaces. + +Extra arguments: none. + +Example (being context `{"c": " foo bar "}`): + +``` +c|trim +``` + +results in + +``` +foo bar +``` + +#### substring + +Returns a substring between two positions. + +Extra arguments: +* Initial position +* Final position + +Example (being context `{"c": "foobar"}`): + +``` +c|substring(3,5) +``` + +results in + +``` +ba +``` + +#### includes + +Returns `true` if a given string is contained in the input string, `false` otherwise. + +Extra arguments: the input string to search. + +Example (being context `{"c": "foobar"}`): + +``` +c|includes('ba') +``` + +results in + +``` +true +``` + +#### isNaN + +Returns `true` if the input is not a number, `false` otherwise. + +Extra arguments: none. + +Example (being context `{"c": "foobar"}`): + +``` +c|isNaN +``` + +results in + +``` +true +``` + +#### parseInt + +Parses a string and return the corresponding integer number. + +Extra arguments: none. + +Example (being context `{"c": "25"}`): + +``` +c|parseInt +``` + +results in + +``` +25 +``` + +#### parseFloat + +Parses a string and return the corresponding float number + +Extra arguments: none. + +Example (being context `{"c": "25.33"}`): + +``` +c|parseFloat +``` + +results in + +``` +25.33 +``` + +#### typeOf + +Return a string with the type of the input. + +Extra arguments: none. + +Example (being context `{"c": 23}`): + +``` +c|typeOf +``` + +results in + +``` +"Number" +``` + +#### toString + +Return a string representation of the input. + +Extra arguments: none. + +Example (being context `{"c": 23}`): + +``` +c|toString +``` + +results in + +``` +"23" +``` + +#### floor + +Return the closest lower integer of a given number. + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|floor +``` + +results in + +``` +3 +``` + +#### ceil + +Return the closest upper integer of a given number. + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|ceil +``` + +results in + +``` +4 +``` + +#### round + +Return the closest integer (either lower or upper) of a given number. + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|round +``` + +results in + +``` +3 +``` + +#### toFixed + +Rounds a number to a number of decimal digits. + +Extra arguments: number of decimal digits. + +Example (being context `{"c": 3.18}`): + +``` +c|toFixed(1) +``` + +results in + +``` +3.2 +``` + +#### log + +Return the natural logarithm of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|log +``` + +results in + +``` +1.144222799920162 +``` + +#### log10 + +Return the base 10 logarithm of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|log10 +``` + +results in + +``` +0.49692964807321494 +``` + +#### log2 + +Return the base 2 logarithm of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|log2 +``` + +results in + +``` +1.6507645591169025 +``` + +#### sqrt + +Return the square root of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|sqrt +``` + +results in + +``` +1.772004514666935 +``` + +#### replaceStr + +Replace occurrences of a string with another in the input string. + +Extra arguments: +* Source string to replace +* Destination string to replace + +Example (being context `{"c": "foobar"}`): + +``` +c|replaceStr('o','u') +``` + +results in + +``` +"fuubar" +``` + +### Failsafe cases + +As failsafe behaviour, evaluation returns `null` in the following cases: + +* Some of the transformation used in the expression is unknown +* Operations with identifiers that are not defined in the context are used. For instance, `(A==null)?-1:A` will result in `null` (and not `-1`) if `A` is not in the context, due to `==` is an operation that cannot be done on undefined identifiers. However, `A||-1` will work (i.e. `-1` will result if `A` is not in the context), as `||` is not considered an operation on `A`. +* Syntax error in the JEXL expression (e.g. `A[0|uppercase`) + +### Known limitations + +- The unitary minus operator is not working properly, e.g. the following expression doesn't work (it failsafes to `null`): `A||-1`. However, the following alternatives are working: `A||0-1` and `A||'-1'|parseInt)` +- Negation operator `!` (supported in original JavaScript JEXL) is not supported + ## Oneshot Subscriptions Oneshot subscription provides an option to subscribe an entity only for one time notification. When consumer creates a subscription @@ -2395,10 +2877,6 @@ in which case all attributes are included in the notification, no matter if they entity. For these attributes that don't exist (`brightness` in this example) the `null` value (of type `"None"`) is used. -In the case of custom notifications, if `covered` is set to `true` then `null` will be used to replace `${...}` -for non existing attributes (the default behavior when `covered` is not set to `true` is to replace by the -empty string the non existing attributes). - We use the term "covered" in the sense the notification "covers" completely all the attributes in the `notification.attrs` field. It can be useful for those notification endpoints that are not flexible enough for a variable set of attributes and needs always the same set of incoming attributes From 826ffdd4bcb917de04d14e06eb89d9cc9c4042f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 16:47:30 +0200 Subject: [PATCH 2/2] FIX basic replacement logic --- src/lib/expressions/ExprManager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 4a92d91e34..cd1a3d5182 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -124,8 +124,11 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st // method to free it r.fill(result); +#ifndef EXPR_BASIC // cjexl_eval() allocated memory for us. We have to release it in order to avoid a leak + // (disbled with EXPR_BASIC because in that case result is static memory) free((char*)result); +#endif TIME_EXPR_JEXL_EVAL_STOP(); }