Merge branch 'master' into hardening/orion-mongodb-5.0

.github/workflows/functional.yml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-   TEST_IMAGE_NAME: fiware/orion-ci:deb
+   TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   functional:

.github/workflows/unit.yml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  TEST_IMAGE_NAME: fiware/orion-ci:deb
+  TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   unit:

.github/workflows/valgrind-nocache.yml

@@ -17,7 +17,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-   TEST_IMAGE_NAME: fiware/orion-ci:deb
+   TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   valgrind:

.github/workflows/valgrind.yml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-   TEST_IMAGE_NAME: fiware/orion-ci:deb
+   TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   valgrind:

.readthedocs.yml

@@ -3,7 +3,11 @@ version: 2
 mkdocs:
   configuration: mkdocs.yml
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+
 python:
-  version: 3.8
   install:
     - requirements: doc/requirements.txt

CHANGES_NEXT_RELEASE

@@ -1,15 +1,9 @@
-- Hardening: upgrade microhttpd dependency from 0.9.73 to 0.9.76
-- Add: subject.condition.notifyOnMetadataChange field to subscriptions, so only-metadata changes doesn't trigger notifications (#3727)
-- Add: new simplifiedNormalized and simplifiedKeyvalues for attrsFormat (#4286)
-- Add: -mqttTimeout (env var ORION_MQTT_TIMEOUT) for MQTT broker connection timeout
-- Fix: alterationType with entityDelete not working if condition.attrs is empty (#4326)
-- Fix: $set/$unset update operator not working properly when the key has dots (#4315)
-- Fix: queue workers deadlock when MQTT broker was not responding to connection attemp
-- Fix: Ngsiv2-AttrsFormat header set to "normalized" in NGSI patching custom notification
-- Fix: memory access potential problem in GET /log/trace request when tracelevel 255 is enabled
-- Fix: Orion should return error and refuse to start when port is negative (#3875)
-- Fix: subscription autodisable based on maxFailsLimit not working when fails does not happen within the same csubs cache refresh cycle
-- Fix: incorrectly sum stale failsCounter from database when subscription has started to work again (only when csubs cache is enabled)
-- Fix: MQTT disconnection when MQTT publish fail (so re-connection is forced at next MQTT notification)
-- Fix: lineMaxSize and infoPayloadMaxSize in the log admin REST API (#3707)
-- Fix: check for service database name too long at cache sync logic (#2848)
+- Add: servicePath field to builtin attributes (#2877)
+- Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388)
+- Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated)
+- Fix: improve error traces (#4387)
+- Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794)
+- Fix: improve logs in MongoDB query logic
+- Upgrade Debian version from 11.6 to 12.1 in Dockerfile
+- Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3
+- Reference distribution changed from Debian 11 to Debian 12

CMakeLists.txt

@@ -171,6 +171,12 @@ endif (${CMAKE_BUILD_TYPE} STREQUAL DEBUG)
 #    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-variable")
 #endif (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
 
+# Enables some some #ifdef in the code for compiling in old system
+# FIXME: cleanup OLD_SSL_VERSION_FORMAT stuff after consolidating the change to Debian 12+
+IF(${DISTRO} MATCHES "Debian_11.*")
+  ADD_DEFINITIONS(-DOLD_SSL_VERSION_FORMAT)
+ENDIF()
+
 #
 # Libraries
 #
@@ -220,8 +226,8 @@ SET (BOOST_MT
 
 # See http://mongoc.org/libmongoc/current/tutorial.html#cmake
 # This find_package() command provides the mongo::mongoc_static used in
-# SET for common static libs. We use 1.23.1 as reference version.
-find_package (mongoc-1.0 1.23.1 EXACT)
+# SET for common static libs. We use 1.24.3 as reference version.
+find_package (mongoc-1.0 1.24.3 EXACT)
 
 # Static libs common to contextBroker and unitTest binaries
 SET (COMMON_STATIC_LIBS

Changelog

@@ -1,3 +1,42 @@
+3.10.0 (June 12th, 2023)
+
+- Add: option to change deprecate log setting in the REST management API
+- Fix: decimal rounding in some cases was generating wrong JSON in query responses and notifications (#4346)
+- Fix: pagination support in request query forwarding to Context Providers (#4149, #847, #1647)
+- Fix: count option not working when georel=near was in use (500 error was returned)
+- Fix: maxFailsLimit didn't work in some cases when update subscription is done just after cache refresh (only when csubs cache is enabled)
+- Fix: conditions.notifyOnMetadataChange false not working if condition.attrs is empty
+- Deprecate: `geo:json`, `geo:line`, `geo:box` and `geo:polygon` attribute types (use `geo:json` instead)
+- Remove: all NGSIv1 operations (deprecated in Orion 2.0.0) except the following ones
+    - PUT /v1/contextEntities/{id}
+    - DELETE /v1/contextEntities/{id}
+    - GET /v1/contextEntities/{id}/attributes/{name}
+    - POST /v1/updateContext
+    - POST /NGSI10/updateContext
+    - POST /v1/queryContext
+    - POST /NGSI10/queryContext
+
+3.9.0 (June 2th, 2023)
+
+- Add: new simplifiedNormalized and simplifiedKeyvalues for attrsFormat (#4286)
+- Add: subject.condition.notifyOnMetadataChange field to subscriptions, so only-metadata changes doesn't trigger notifications (#3727)
+- Add: support for update operators ($set, $addToSet, etc.) in entity creation and replace (#3814, reopnened)
+- Add: -mqttTimeout (env var ORION_MQTT_TIMEOUT) for MQTT broker connection timeout
+- Add: -disableNgsiv1 (env var ORION_DISABLE_NGSIV1) to disable NGSIv1 API requests
+- Add: -logDeprecate (env var ORION_LOG_DEPRECATE) to log deprecate usages at WARN level
+- Fix: alterationType with entityDelete not working if condition.attrs is empty (#4326)
+- Fix: $set/$unset update operator not working properly when the key has dots (#4315)
+- Fix: queue workers deadlock when MQTT broker was not responding to connection attemp
+- Fix: Ngsiv2-AttrsFormat header set to "normalized" in NGSI patching custom notification
+- Fix: memory access potential problem in GET /log/trace request when tracelevel 255 is enabled
+- Fix: Orion should return error and refuse to start when port is negative (#3875)
+- Fix: subscription autodisable based on maxFailsLimit not working when fails does not happen within the same csubs cache refresh cycle
+- Fix: incorrectly sum stale failsCounter from database when subscription has started to work again (only when csubs cache is enabled)
+- Fix: MQTT disconnection when MQTT publish fail (so re-connection is forced at next MQTT notification)
+- Fix: lineMaxSize and infoPayloadMaxSize in the log admin REST API (#3707)
+- Fix: check for service database name too long at cache sync logic (#2848)
+- Hardening: upgrade microhttpd dependency from 0.9.73 to 0.9.76
+
 3.8.0 (January 12th, 2023)
 
 - Add: json field in httpCustom and mqttCustom subscriptions to implement JSON-based payloads in notifications (#2560)

README.md

@@ -92,8 +92,8 @@ recommended to have a look to the brief
 ### Introductory presentations
 
 -   Orion Context Broker
-    [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-20221220)
-    [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-20221220-254969123)
+    [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-20230606)
+    [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-20230606-258279602)
 -   NGSIv2 Overview for Developers That Already Know NGSIv1
     [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220523)
     [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220526)

ci/README.jp.md

@@ -1,13 +1,13 @@
 ## 概要
 このリポジトリでは GitHub Actions が有効になっているため、マージを許可する前に各プルリクエストがチェックされます。
-このシステムは、新しい PR がマスターに到達するたびにマスターブランチから構築される `fiware/orion-ci:deb` に基づいており、
+このシステムは、新しい PR がマスターに到達するたびにマスターブランチから構築される `telefonicaiot/fiware-orion:ci` に基づいており、
 すべてのビルド依存関係がオンボードにあるクリーンな環境を提供します。この Docker のビルドに使用される Dockerfile は、
 `ci/deb` ディレクトリにあります。
 
-テスト対象の PR ブランチが変更されたため、`fiware/orion-ci:deb` は再構築されないことに注意してください。したがって、
+テスト対象の PR ブランチが変更されたため、`telefonicaiot/fiware-orion:ci` は再構築されないことに注意してください。したがって、
 新しいライブラリまたはベースシステムを必要とする機能を開発している場合は、そのようなライブラリまたはベースシステムを
 `ci/deb/build-dep.sh` および/または `Dockerfile` に追加する PR を*最初に*実行する必要があります。その PR がマスターに
-マージされ、`fiware/orion-ci:deb` が再構築されると (Docker Hub の https://hub.docker.com/r/fiware/orion-ci/builds
+マージされ、`telefonicaiot/fiware-orion:ci` が再構築されると (Docker Hub の https://hub.docker.com/r/telefonicaiot/fiware-orion/builds
 で進行状況を確認)、新しい機能は、GitHub アクションでテストする準備ができています。
 
 GitHub Actions チェックは段階に分かれており、 "サポートされているテスト" セクションで説明されています。
@@ -31,23 +31,23 @@ CI の現在のバージョンは以下をサポートします:
 イメージをダウンロードするには:
 
 ```
-docker pull fiware/orion-ci:deb
+docker pull telefonicaiot/fiware-orion:ci
 ```
 
 たとえば、GitHub Actions と同じ方法でイメージを実行するには、次のようにします:
 
 ```
 # Check that MongoDB server is running in your localhost:27017
 cd /path/to/fiware-orion
-docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1201 -v $(pwd):/opt/fiware-orion fiware/orion-ci:deb build -miqts functional
+docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1201 -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci build -miqts functional
 ```
 
 インタラクティブな bash を使用してイメージを実行するには:
 
 ```
 # Check that MongoDB server is running in your localhost:27017
 cd /path/to/fiware-orion
-docker run --network host -ti -v $(pwd):/opt/fiware-orion fiware/orion-ci:deb bash
+docker run --network host -ti -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci bash
 ```
 
 bash シェルを起動したら、同様の実行を行うことができます:

ci/README.md

@@ -1,13 +1,13 @@
 ## Overview
 GitHub Actions is enabled in this repository so each pull request is checked before being allowed to merge.
-The system is based on `fiware/orion-ci:deb` which is built from master branch each time a new PR lands in master,
+The system is based on `telefonicaiot/fiware-orion:ci` which is built from master branch each time a new PR lands in master,
 providing a clean environment with all build dependencies onboard. The Dockerfile used to build this docker is available
 in the `ci/deb` directory.
 
-Note that `fiware/orion-ci:deb` is *not* rebuilt due to changes in the PR branch under test. Thus, if you are developing
+Note that `telefonicaiot/fiware-orion:ci` is *not* rebuilt due to changes in the PR branch under test. Thus, if you are developing
 a functionality that requires a new library or base system you need to do *first* a PR adding such library or base system
-to `ci/deb/build-dep.sh` and/or `Dockerfile`. Once that PR gets merged into master and `fiware/orion-ci:deb` gets rebuild
-(checking progress in Dockerhub at: https://hub.docker.com/r/fiware/orion-ci/builds) your PR branch with the new 
+to `ci/deb/build-dep.sh` and/or `Dockerfile`. Once that PR gets merged into master and `telefonicaiot/fiware-orion:ci` gets rebuild
+(checking progress in Dockerhub at: https://hub.docker.com/r/telefonicaiot/fiware-orion/builds) your PR branch with the new 
 functionality is ready to be tested with GitHub Actions.
 
 The GitHub Actions checks are divided into stages, which are described in "Supported tests" section.
@@ -30,23 +30,23 @@ the following cheatsheet can be useful:
 To download the image:
 
 ```
-docker pull fiware/orion-ci:deb
+docker pull telefonicaiot/fiware-orion:ci
 ```
 
 To run the image in the same way that GitHub Actions does, for instance:
 
 ```
 # Check that MongoDB server is running in your localhost:27017
 cd /path/to/fiware-orion
-docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1201 -v $(pwd):/opt/fiware-orion fiware/orion-ci:deb build -miqts functional
+docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1201 -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci build -miqts functional
 ```
 
 To run the image using an interactive bash on it
 
 ```
 # Check that MongoDB server is running in your localhost:27017
 cd /path/to/fiware-orion
-docker run --network host -ti -v $(pwd):/opt/fiware-orion fiware/orion-ci:deb bash
+docker run --network host -ti -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci bash
 ```
 
 Once have a bash shell, you can do the same execution:

ci/deb/Dockerfile

@@ -1,8 +1,9 @@
-FROM debian:11.6-slim
+FROM debian:12.1-slim
 
 ADD build.sh /opt/bin/
 ADD build-dep.sh /opt/bin/
 ADD makefile /opt/archive/
+ADD fuse_gtest_files.py.patch /opt/archive/
 
 RUN ln -s /opt/bin/build.sh /usr/local/bin/build \
 && /opt/bin/build-dep.sh

ci/deb/build-dep.sh

@@ -25,15 +25,14 @@
 # Install security updates
 apt-get -y update
 apt-get -y upgrade
-# FIXME: python2 required by an installation script in GMock. Sad but true :(
 # Install dependencies
 apt-get -y install \
   curl \
   gnupg \
-  python2 \
   python3 \
   python3-pip \
-  netcat \
+  python3-venv \
+  netcat-traditional \
   bc \
   valgrind \
   cmake \
@@ -50,27 +49,30 @@ apt-get -y install \
   libsasl2-dev \
   libgcrypt-dev
 
+## FIXME: check note in build_source.md about the libssl1 installation hack. It will be no longer needed from MongoDB 6.0 on
 echo "INSTALL: MongoDB shell" \
+&& curl -L http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb --output libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
+&& dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
+&& rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
 && curl -L https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - \
 && echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list \
 && apt-get -y update \
 && apt-get -y install mongodb-org-shell
 
 echo "INSTALL: python special dependencies" \
 && cd /opt \
-&& pip3 install virtualenv \
-&& virtualenv /opt/ft_env --python=/usr/bin/python3 \
+&& python3 -m venv /opt/ft_env \
 && . /opt/ft_env/bin/activate \
 && pip install Flask==2.0.2 \
 && pip install paho-mqtt==1.6.1 \
-&& pip install amqtt==0.10.1 \
+&& pip install amqtt==0.11.0b1 \
 && deactivate
 
 # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0
 # see http://mongoc.org/libmongoc/current/init-cleanup.html#deprecated-feature-automatic-initialization-and-cleanup
-echo "INSTALL: mongodb c driver (required by mongo c++ driver)" \
-&& curl -L https://github.com/mongodb/mongo-c-driver/releases/download/1.23.1/mongo-c-driver-1.23.1.tar.gz | tar xzC /opt/ \
-&& cd /opt/mongo-c-driver-1.23.1 \
+echo "INSTALL: mongodb c driver" \
+&& curl -L https://github.com/mongodb/mongo-c-driver/releases/download/1.24.3/mongo-c-driver-1.24.3.tar.gz | tar xzC /opt/ \
+&& cd /opt/mongo-c-driver-1.24.3 \
 && mkdir cmake-build \
 && cd cmake-build \
 && cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF .. \
@@ -88,11 +90,10 @@ echo "INSTALL: libmicrohttpd" \
 && make \
 && make install
 
-# FIXME: if sometimes python2 goes away the fuse_gtest_files.py would need to be migrated to python3
 echo "INSTALL: gmock" \
 && curl -L https://src.fedoraproject.org/repo/pkgs/gmock/gmock-1.5.0.tar.bz2/d738cfee341ad10ce0d7a0cc4209dd5e/gmock-1.5.0.tar.bz2 | tar xjC /opt/ \
 && cd /opt/gmock-1.5.0 \
-&& sed -i 's/env python/env python2/' gtest/scripts/fuse_gtest_files.py \
+&& patch -p1 gtest/scripts/fuse_gtest_files.py < /opt/archive/fuse_gtest_files.py.patch \
 && ./configure \
 && make \
 && make install
@@ -109,7 +110,7 @@ echo "INSTALL: mosquitto" \
 ldconfig
 
 apt-get -y clean \
-&& rm -Rf /opt/mongo-c-driver-1.23.1 \
+&& rm -Rf /opt/mongo-c-driver-1.24.3 \
 && rm -Rf /opt/rapidjson-1.1.0 \
 && rm -Rf /opt/libmicrohttpd-0.9.76 \
 && rm -Rf /opt/mosquitto-2.0.15 \