Merge pull request #4511 from telefonicaid/feature/4004_jexl_expressions

JEXL expressions in custom notifications

.github/workflows/functional.yml

@@ -60,4 +60,4 @@ jobs:
 
       - name: Run functional test
         run: |
-          docker run  --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional
+          docker run  --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional

.github/workflows/publishimage.yml → .github/workflows/publishimage-fiware.yml

@@ -1,6 +1,6 @@
-name: Publish pre Image from master
+name: Publish pre Image from master in FIWARE dockerhub
 
-# The workflow will push PRE images from master on every merge. The images will be tagged with PRE and the next minor increase on the
+# The workflow will push PRE images from master on every merge to the fiware dockerhub organization. The images will be tagged with PRE and the next minor increase on the
 # semver(based on the github releases)
 # It will NOT produce releases and release images or the 'latest' tag from master. Both (releases and 'latest' tag) rely on the
 # dockerhub autobuild feature

.github/workflows/publishimage-master.yml

@@ -0,0 +1,32 @@
+name: Publish Docker image (master)
+
+# The workflow will push images for master on every merge
+# Ideally, this should be done at dockerhub, but it doesn't support secrets (see https://stackoverflow.com/questions/78446824/use-environment-variables-with-sensible-information-in-docker-hub-autobuild)
+
+on:
+  push:
+    branches:
+      - master      
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_TEF_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TEF_TOKEN }}
+
+    - name: Build Docker image
+      run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile .
+
+    - name: Push Docker image
+      run: docker push telefonicaiot/fiware-orion:latest

.github/workflows/publishimage-tag.yml

@@ -0,0 +1,36 @@
+name: Publish Docker image (tag)
+
+# The workflow will push images on every tag in the format x.y.z
+# Ideally, this should be done at dockerhub, but it doesn't support secrets (see https://stackoverflow.com/questions/78446824/use-environment-variables-with-sensible-information-in-docker-hub-autobuild)
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_TEF_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TEF_TOKEN }}
+
+    - name: Extract version from tag
+      id: extract_version
+      run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+
+    - name: Build Docker image
+      run: docker build -t telefonicaiot/fiware-orion:${{ env.VERSION }} --build-arg GIT_REV_ORION=${{ env.VERSION }} --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile .
+
+    - name: Push Docker image
+      run: docker push telefonicaiot/fiware-orion:${{ env.VERSION }}

.github/workflows/unit.yml

@@ -31,4 +31,4 @@ jobs:
 
       - name: Run unit tests
         run: |
-          docker run  --network host -t --rm -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts unit
+          docker run  --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts unit

.github/workflows/valgrind-nocache.yml

@@ -58,4 +58,4 @@ jobs:
 
       - name: Run valgrind test
         run: |
-          docker run  --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind
+          docker run  --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind

.github/workflows/valgrind.yml

@@ -55,4 +55,4 @@ jobs:
 
       - name: Run valgrind test
         run: |
-          docker run  --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind
+          docker run  --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind

CHANGES_NEXT_RELEASE

@@ -1,2 +1,6 @@
+- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.2.0) (#4004)
+- Add: expression context build and evaluation counters in timing section in GET /statistics (#4004)
+- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004)
+- Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004)
 - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it)
 - Fix: lighter operation to get databases list from MongoDB (#4517)

CMakeLists.txt

@@ -204,6 +204,7 @@ SET (ORION_LIBS
     common
     alarmMgr
     metricsMgr
+    expressions
     logSummary
     lm
     pa
@@ -229,10 +230,20 @@ SET (BOOST_MT
 # SET for common static libs. We use 1.24.3 as reference version.
 find_package (mongoc-1.0 1.24.3 EXACT)
 
+# Is cjexl lib available?
+find_library (HAVE_CJEXL cjexl PATHS /usr/lib /usr/lib64 /usr/local/lib64 /usr/local/lib)
+if (HAVE_CJEXL)
+    message("Using cjexl")
+else (HAVE_CJEXL)
+    message("Not using cjexl")
+    add_definitions(-DEXPR_BASIC)
+endif (HAVE_CJEXL)
+
 # Static libs common to contextBroker and unitTest binaries
 SET (COMMON_STATIC_LIBS
     microhttpd.a
     mosquitto.a
+    ${HAVE_CJEXL}
     mongo::mongoc_static
 )
 
@@ -264,7 +275,6 @@ endif (UNIT_TEST)
 #
 include_directories("/usr/include")
 
-
 # Needed for the new C driver
 include_directories("/usr/local/include/libmongoc-1.0")
 include_directories("/usr/local/include/libbson-1.0")
@@ -319,6 +329,7 @@ if (error EQUAL 0)
   ADD_SUBDIRECTORY(src/lib/cache)
   ADD_SUBDIRECTORY(src/lib/alarmMgr)
   ADD_SUBDIRECTORY(src/lib/metricsMgr)
+  ADD_SUBDIRECTORY(src/lib/expressions)
   ADD_SUBDIRECTORY(src/lib/logSummary)
   ADD_SUBDIRECTORY(src/lib/mqtt)
   ADD_SUBDIRECTORY(src/app/contextBroker)

ci/deb/build.sh

@@ -172,6 +172,12 @@ echo "===================================== PREPARE ============================
 echo "Builder: create temp folders"
 rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu}
 
+if [ -z "${REPO_ACCESS_TOKEN}" ]; then
+    echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download"
+else
+    bash /opt/fiware-orion/get_cjexl.sh 0.2.0 $REPO_ACCESS_TOKEN
+fi
+
 if [ -n "${branch}" ]; then
     echo "===================================== CLONE ============================================"
 

doc/manuals/admin/build_source.md

@@ -4,6 +4,8 @@ Orion Context Broker reference distribution is Debian 12. This doesn't mean that
 
 You can also have a look to [3.1 Building in not official distributions](../../../docker/README.md#31-building-in-not-official-distributions) section in the Docker documentation to check how to build Docker containers images in distributions other than the official one.
 
+*NOTE:* the build process described in this document does not include the cjexl library, as it is considered optional from the point of view of the basic building process.
+
 ## Debian 12 (officially supported)
 
 The Orion Context Broker uses the following libraries as build dependencies:

doc/manuals/admin/statistics.md

@@ -129,20 +129,22 @@ Provides timing information, i.e. the time that CB passes executing in different
   "timing": {
     "accumulated": {
       "jsonV1Parse": 7.860908311,
-      "mongoBackend": 416.796091597,
-      "mongoReadWait": 4656.924425628,
-      "mongoWriteWait": 259.347915990,
-      "mongoCommandWait": 0.514811318,
-      "render": 108.162782114,
-      "total": 6476.593504743
-     },
+      "jsonV2Parse": 120.680244446,
+      "mongoBackend": 12778.52734375,
+      "mongoReadWait": 7532.301757812,
+      "mongoWriteWait": 3619.282226562,
+      "mongoCommandWait": 0.120559767,
+      "exprJexlCtxBld": 27.092681885,
+      "exprJexlEval": 124.217208862,
+      "render": 44.540554047,
+      "total": 25051.384765625
+    },
     "last": {
-      "mongoBackend": 0.014752309,
-      "mongoReadWait": 0.012018445,
-      "mongoWriteWait": 0.000574611,
-      "render": 0.000019136,
-      "total": 0.015148915
-     }
+      "mongoBackend": 0.003775352,
+      "mongoReadWait": 0.0013743,
+      "render": 0.000286864,
+      "total": 0.00440685
+    }
   }
   ...
 }
@@ -167,10 +169,15 @@ The particular counters are as follows:
   `last` includes the accumulation for all of them. In the case of mongoReadWait, only the time used
   to get the results cursor is taken into account, but not the time to process cursors results (which
   is time that belongs to mongoBackend counters).
+* `exprJexlCtxBld`: time passed building context for custom notification expression evaluation (see [macro substitution](../orion-api.md#macro-substitution) and [JEXL support](../orion-api.md#jexl-support))
+* `exprJexlEval`: time passed evaluating custom notification expressions (see [macro substitution](../orion-api.md#macro-substitution) and [JEXL support](../orion-api.md#jexl-support))
+
+*NOTE*: if Orion binary is build without using cjexl and only basic replacement is available, then `exprBasicCtxtBld` and `exprBasicEval`
+fields appear instead of `exprJexlCtxBld` and `exprJexlEval`.
 
 Times are measured from the point in time in which a particular thread request starts using a module until it finishes using it.
 Thus, if the thread is stopped for some reason (e.g. the kernel decides to give priority to another thread based on its
-scheculing policy) the time that the thread was sleeping, waiting to execute again is included in the measurement and thus, the measurement is not accurate. That is why we say *pseudo* selt/end-to-end time. However,
+scheculing policy) the time that the thread was sleeping, waiting to execute again is included in the measurement and thus, the measurement is not accurate. That is why we say *pseudo* self/end-to-end time. However,
 under low load conditions this situation is not expected to have a significant impact.
 
 ### NotifQueue block

doc/manuals/devel/sourceCode.md

@@ -24,6 +24,7 @@
 * [src/lib/cache/](#srclibcache) (Subscription cache implementation)
 * [src/lib/logSummary/](#srcliblogsummary) (Log Summary implementation)
 * [src/lib/metricsMgr/](#srclibmetricsmgr) (Metrics Manager implementation)
+* [src/lib/expressions/](#srclibexpressions) (Custom notification expressions support)
 
 ## src/app/contextBroker/
 The main program is found in `contextBroker.cpp` and its purpose it to:
@@ -551,3 +552,8 @@ This Metrics Manager resides in the library **metricsMgr**.
 For information about the metrics, please refer to [this document](../admin/metrics_api.md).
 
 [Top](#top)
+
+## src/lib/expressions/
+Provides support to the [macro substition logic used by custom notifications](../orion-api.md#macro-substitution). This library provides an abstraction for expression evaluation, providing two implementations: JEXL based and basic replacement based (the implementation to use is choosen at building time, based on the availability of the cjex library).
+
+[Top](#top)