Merge branch 'master' into hardening/orion-mongodb-7.0

.github/workflows/functional.yml

@@ -60,4 +60,4 @@ jobs:
 
       - name: Run functional test
         run: |
-          docker run  --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional
+          docker run  --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional

.github/workflows/publishimage-master.yml

@@ -0,0 +1,34 @@
+name: Publish Docker image (master)
+
+# The workflow will push images for master on every merge
+# Ideally, this should be done at dockerhub, but it doesn't support secrets (see https://stackoverflow.com/questions/78446824/use-environment-variables-with-sensible-information-in-docker-hub-autobuild)
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_TEF_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TEF_TOKEN }}
+
+    - name: Build Docker image
+      run: |
+        export TOKEN=${{ secrets.REPO_ACCESS_TOKEN }}
+        docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --secret id=repo_token,env=TOKEN --no-cache -f docker/Dockerfile .
+
+    - name: Push Docker image
+      run: docker push telefonicaiot/fiware-orion:latest

.github/workflows/publishimage-tag.yml

@@ -0,0 +1,38 @@
+name: Publish Docker image (tag)
+
+# The workflow will push images on every tag in the format x.y.z
+# Ideally, this should be done at dockerhub, but it doesn't support secrets (see https://stackoverflow.com/questions/78446824/use-environment-variables-with-sensible-information-in-docker-hub-autobuild)
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_TEF_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TEF_TOKEN }}
+
+    - name: Extract version from tag
+      id: extract_version
+      run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+
+    - name: Build Docker image
+      run: |
+        export TOKEN=${{ secrets.REPO_ACCESS_TOKEN }}
+        docker build -t telefonicaiot/fiware-orion:${{ env.VERSION }} --build-arg GIT_REV_ORION=${{ env.VERSION }} --secret id=repo_token,env=TOKEN --no-cache -f docker/Dockerfile .
+
+    - name: Push Docker image
+      run: docker push telefonicaiot/fiware-orion:${{ env.VERSION }}

.github/workflows/unit.yml

@@ -31,4 +31,4 @@ jobs:
 
       - name: Run unit tests
         run: |
-          docker run  --network host -t --rm -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts unit
+          docker run  --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts unit

.github/workflows/valgrind-nocache.yml

@@ -58,4 +58,4 @@ jobs:
 
       - name: Run valgrind test
         run: |
-          docker run  --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind
+          docker run  --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind

.github/workflows/valgrind.yml

@@ -55,4 +55,4 @@ jobs:
 
       - name: Run valgrind test
         run: |
-          docker run  --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind
+          docker run  --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind

CHANGES_NEXT_RELEASE

@@ -1,12 +1,9 @@
-- Add: servicePath field to builtin attributes (#2877)
-- Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388)
-- Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643)
-- Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435)
-- Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated)
-- Fix: improve error traces (#4387)
-- Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794)
-- Fix: improve logs in MongoDB query logic
-- Upgrade Debian version from 11.6 to 12.1 in Dockerfile
-- Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3
-- Reference MongoDB version changed from 4.4 to 6.0
-- Reference distribution changed from Debian 11 to Debian 12
+- Add: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556)
+- Add: attribute metadata to JEXL context (#4594, #4560)
+- Fix: $max and $min operators were not supported with DateTime attributes (#4585)
+- Fix: wrong date values should not allowed in subscription's expires field (#4541)
+- Fix: do not raise DB alarm in case of wrong GeoJSON in client request
+- Fix: metadata modifications are not considered as change (with regards to subscription alterationTypes) if notifyOnMetadataChange is false (#4605)
+- Upgrade cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString)
+- Upgrade Debian version from 12.4 to 12.6 in Dockerfile
+- Fix: invalid date in expires field of subscription (#2303)

CMakeLists.txt

@@ -26,6 +26,10 @@ PROJECT(contextBroker)
 # find_package() for mongo driver doesn't work with cmake 2
 cmake_minimum_required(VERSION 3.0)
 
+# set C++14 standard
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
 #
 # DEBUG or RELEASE build ?
 #
@@ -183,8 +187,8 @@ ENDIF()
 
 SET (ORION_LIBS
     common
-    serviceRoutines
     serviceRoutinesV2
+    serviceRoutines
     ngsiNotify
     rest          # verbName(Verb) from [email protected]; jsonRequestTreat from [email protected];
     jsonParse
@@ -204,6 +208,7 @@ SET (ORION_LIBS
     common
     alarmMgr
     metricsMgr
+    expressions
     logSummary
     lm
     pa
@@ -229,12 +234,28 @@ SET (BOOST_MT
 # SET for common static libs. We use 1.24.3 as reference version.
 find_package (mongoc-1.0 1.24.3 EXACT)
 
+# Is cjexl lib available?
+find_library (HAVE_CJEXL cjexl PATHS /usr/lib /usr/lib64 /usr/local/lib64 /usr/local/lib)
+if (HAVE_CJEXL)
+    message("Using cjexl")
+    SET (COMMON_STATIC_LIBS
+        microhttpd.a
+        mosquitto.a
+        ${HAVE_CJEXL}
+        mongo::mongoc_static
+    )
+else (HAVE_CJEXL)
+    message("Not using cjexl")
+    add_definitions(-DEXPR_BASIC)
+    SET (COMMON_STATIC_LIBS
+        microhttpd.a
+        mosquitto.a
+        mongo::mongoc_static
+    )
+endif (HAVE_CJEXL)
+
 # Static libs common to contextBroker and unitTest binaries
-SET (COMMON_STATIC_LIBS
-    microhttpd.a
-    mosquitto.a
-    mongo::mongoc_static
-)
+
 
 SET (DYNAMIC_LIBS
     curl
@@ -264,7 +285,6 @@ endif (UNIT_TEST)
 #
 include_directories("/usr/include")
 
-
 # Needed for the new C driver
 include_directories("/usr/local/include/libmongoc-1.0")
 include_directories("/usr/local/include/libbson-1.0")
@@ -285,13 +305,13 @@ link_directories("/usr/lib/x86_64-linux-gnu")
 #
 # Enabling test harness
 #
-if (UNIT_TEST)
+if (COVERAGE AND NOT UNIT_TEST)
    ENABLE_TESTING()
    INCLUDE(Dart)
    SET (CMAKE_TEST_TIMEOUT 60)
    SET (DART_TIMEOUT 60)
-   #ADD_SUBDIRECTORY(test/harness)
-endif (UNIT_TEST)
+   ADD_SUBDIRECTORY(test/functionalTest/cases)
+endif (COVERAGE AND NOT UNIT_TEST)
 
 #
 # Continuing if OK
@@ -319,6 +339,7 @@ if (error EQUAL 0)
   ADD_SUBDIRECTORY(src/lib/cache)
   ADD_SUBDIRECTORY(src/lib/alarmMgr)
   ADD_SUBDIRECTORY(src/lib/metricsMgr)
+  ADD_SUBDIRECTORY(src/lib/expressions)
   ADD_SUBDIRECTORY(src/lib/logSummary)
   ADD_SUBDIRECTORY(src/lib/mqtt)
   ADD_SUBDIRECTORY(src/app/contextBroker)

Changelog

@@ -1,3 +1,62 @@
+4.0.0 (June 6th, 2024)
+
+- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.3.0) (#4004)
+- Add: expression context build and evaluation counters in timing section in GET /statistics (#4004)
+- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004)
+- Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004)
+- Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it)
+- Fix: lighter operation to get databases list from MongoDB (#4517)
+- Hardening: compile code using C++14 standard
+- Remove: `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` CLI parameters along with associated env vars, already deprecated in Orion 3.12.0 (use `-dbURI` instead`)
+- Remove: legacy subscription format
+- Remove: all the remaining NGSIv1 operations
+  - PUT /v1/contextEntities/{id}
+  - DELETE /v1/contextEntities/{id}
+  - GET /v1/contextEntities/{id}/attributes/{name}
+  - POST /v1/updateContext
+  - POST /NGSI10/updateContext
+  - POST /v1/queryContext
+  - POST /NGSI10/queryContext
+
+3.12.0 (February 29th, 2024)
+
+- Fix: service path levels with 0 length should not be allowed (#4495)
+- Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) 
+- Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496)
+- Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499)
+- Deprecate: `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` CLI parameters along with associated env vars (use `-dbURI` instead`)
+
+3.11.0 (January 29th, 2024)
+
+- Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388)
+- Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794)
+- Add: servicePath field to builtin attributes (#2877)
+- Add: notification payload in INFO log traces (#4449)
+- Add: log deprecation traces for usage of legacyForwarding mode in registrations
+- Add: log deprecation traces for usage of attrsFormat legacy in subscriptions
+- Add: deprecatedFeatures counters block in GET /v2/statistics
+- Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643)
+- Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451)
+- Fix: builtin attributes alterationType, dateCreated and dateModified included in notifications even when onlyChangedAttrs is true
+- Fix: update forwarding was not working when entity type is not included in the request (#4460)
+- Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435)
+- Fix: in custom notification NGSI payload patching macro subscription covering completely the string where is used, always cast to string for id/type (#4462)
+- Fix: on delete attributes operation, the lack of attribute in entity doesn't preclude the deletion of attributes which do exist (this way behaving same way as update attributes operation)
+- Fix: POST /v2/entities/E/attrs?options=append was wrongly creating entities
+- Fix: provide more informative error description in some error responses in update/delete operations
+- Fix: proper use of "PartialUpdate" (instead of "Unprocessed") in responses in the case of partial updates/deletions (#3499)
+- Fix: response 404 Not Found "NotFound" errors instead of 422 Unprocessable Content "Unprocessed" in the case of missing attribute in existing entity in attribute update operations
+- Fix: "Internal Server Error" changed to "InternalServerError" in error responses
+- Fix: improve logs in MongoDB query logic
+- Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated)
+- Fix: false positive in log deprecation logic when entity name (or other literal) includes the token "v1" (#4454)
+- Fix: improve error traces (#4387)
+- Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3
+- Reference MongoDB version changed from 4.4 to 6.0
+- Reference distribution changed from Debian 11 to Debian 12
+- Upgrade Debian version from 11.6 to 12.4 in Dockerfile
+- Remove: deprecated NGSIv1 location metadata to specify entity geo-location
+
 3.10.0 (June 12th, 2023)
 
 - Add: option to change deprecate log setting in the REST management API

README.md

@@ -93,8 +93,8 @@ recommended to have a look to the brief
 ### Introductory presentations
 
 -   Orion Context Broker
-    [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-20230606)
-    [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-20230606-258279602)
+    [(en)](https://www.slideshare.net/slideshow/orion-context-broker-introduction-20240911/271735060)
+    [(jp)](https://www.slideshare.net/slideshow/orion-context-broker-introduction-20240605/269515246)
 -   NGSIv2 Overview for Developers That Already Know NGSIv1
     [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220523)
     [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220526)
@@ -262,7 +262,7 @@ tag `fiware-orion`
 Orion Context Broker is licensed under [Affero General Public License (GPL)
 version 3](./LICENSE).
 
-© 2023 Telefonica Investigación y Desarrollo, S.A.U
+© 2024 Telefonica Investigación y Desarrollo, S.A.U
 
 <details>
 <summary><strong>Further information on the use of the AGPL open source license</strong></summary>

archive/README.md

@@ -0,0 +1 @@
+After 6 years without touching this directory, we are purging it for Orion 4.0.0 release. If you want to get its content check [the previous version GitHub tree](https://github.com/telefonicaid/fiware-orion/tree/3.12.0/archive).