Merge pull request #4332 from telefonicaid/hardening/orion-mongodb-6.0

MongoDB 6.0

.github/workflows/functional.yml

@@ -23,7 +23,7 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:4.4
+        image: mongo:6.0
         ports:
           - 27017:27017
 

.github/workflows/unit.yml

@@ -22,7 +22,7 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:4.4
+        image: mongo:6.0
         ports:
           - 27017:27017
 

.github/workflows/valgrind-nocache.yml

@@ -26,7 +26,7 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:4.4
+        image: mongo:6.0
         ports:
           - 27017:27017
 

.github/workflows/valgrind.yml

@@ -23,7 +23,7 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:4.4
+        image: mongo:6.0
         ports:
           - 27017:27017
 

CHANGES_NEXT_RELEASE

@@ -8,4 +8,5 @@
 - Fix: improve logs in MongoDB query logic
 - Upgrade Debian version from 11.6 to 12.1 in Dockerfile
 - Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3
+- Reference MongoDB version changed from 4.4 to 6.0
 - Reference distribution changed from Debian 11 to Debian 12

ci/deb/build-dep.sh

@@ -49,15 +49,11 @@ apt-get -y install \
   libsasl2-dev \
   libgcrypt-dev
 
-## FIXME: check note in build_source.md about the libssl1 installation hack. It will be no longer needed from MongoDB 6.0 on
 echo "INSTALL: MongoDB shell" \
-&& curl -L http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb --output libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-&& dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-&& rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-&& curl -L https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - \
-&& echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list \
+&& curl -L https://www.mongodb.org/static/pgp/server-6.0.asc | apt-key add - \
+&& echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/6.0 main" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list \
 && apt-get -y update \
-&& apt-get -y install mongodb-org-shell
+&& apt-get -y install mongodb-mongosh
 
 echo "INSTALL: python special dependencies" \
 && cd /opt \

doc/manuals/admin/build_source.md

@@ -107,13 +107,7 @@ The Orion Context Broker comes with a suite of unit, valgrind and end-to-end tes
 
 In the case of the aarch64 architecture, install libxslt using apt-get, and run `./configure` with `--build=arm-linux` option.
 
-* Install MongoDB (tests rely on mongod running in localhost). Check [the official MongoDB documentation](hhttps://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-debian/) for details. Recommended version is 4.4 (it may work with previous versions, but we don't recommend it).
-    * Note that mongo legacy shell (the `mongo` command) has been deprecated in MongoDB 5 and removed in MongoDB 6 in favour of the new shell (`mongosh` command). Some functional tests (ftest) will fail due to this if you use MongoDB 6 or beyond, as they are suited to use `mongo` and not `mongosh`.
-    * Debian 12 has stepped to libssl3 but some MongoDB versions may require libssl1. In the case you get a `Depends: libssl1.1 (>= 1.1.1) but it is not installable` error, you can test the following (reference [here](https://askubuntu.com/a/1421959))
-
-        wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
-        sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
-        rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb     # optional, for cleanness
+* Install MongoDB (tests rely on mongod running in localhost). Check [the official MongoDB documentation](hhttps://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-debian/) for details. Recommended version is 6.0 (it may work with previous versions, but we don't recommend it).
 
 * Run unit test
 

doc/manuals/admin/install.md

@@ -26,7 +26,7 @@ In the case you are installing Orion building from sources you need:
 * Operating system: Debian. The reference operating system is Debian 12.1
   but it should work also in any later Debian 12 version.
 * Database: MongoDB is required to run either in the same host where Orion Context Broker is to be installed or in a different host accessible through the network. The recommended MongoDB version
-  is 4.4 (Orion may work with older versions but we don't recommend it at all!).
+  is 6.0 (Orion may work with older versions but we don't recommend it at all!).
 
 For system resources (CPUs, RAM, etc.) see [these recommendations](diagnosis.md#resource-availability).
 

doc/manuals/admin/perf_tuning.md

@@ -20,7 +20,7 @@
 
 ##  MongoDB configuration
 
-From a performance point of view, it is recommended to use MongoDB 4.4 with WireTiger, especially
+From a performance point of view, it is recommended to use MongoDB 6.0 with WireTiger, especially
 in update-intensive scenarios.
 
 In addition, take into account the following information from the official MongoDB documentation, as it may have

docker/README.md

@@ -38,7 +38,7 @@ Follow these steps:
 		    command: -dbhost mongo
 		
 		  mongo:
-		    image: mongo:4.4
+		    image: mongo:6.0
 		    command: --nojournal
 
 3. Using the command-line and within the directory you created type: `sudo docker-compose up`.
@@ -77,7 +77,7 @@ Check that everything works with
 ### 2B. MongoDB runs on another docker container
 In case you want to run MongoDB on another container you can launch it like this
 
-	sudo docker run --name mongodb -d mongo:4.4
+	sudo docker run --name mongodb -d mongo:6.0
 
 And then run Orion with this command
 
@@ -111,7 +111,7 @@ Steps:
 4. Run Orion...
 	* Using an automated scenario with docker-compose and building your new image: `sudo docker-compose up`. You may also modify the provided `docker-compose.yml` file if you need so.
 	* Manually, running MongoDB on another container: 
-		1. `sudo docker run --name mongodb -d mongo:4.4`
+		1. `sudo docker run --name mongodb -d mongo:6.0`
 		2. `sudo docker build -t orion .`
 		3. `sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 orion -dbhost mongodb`.
 	* Manually, specifying where to find your MongoDB host:

docker/docker-compose.yml

@@ -10,6 +10,6 @@ services:
     command: -dbhost mongo
 
   mongo:
-    image: mongo:4.4
+    image: mongo:6.0
     command: --nojournal
 

docker/docker_swarm.md

@@ -86,7 +86,7 @@ Details on how to deploy a MongoDB ReplicaSet in Docker Swarm are available
     services:
 
       mongo:
-        image: mongo:4.4
+        image: mongo:6.0
         entrypoint: [ "/usr/bin/mongod", "--replSet", "rs", "--journal", "--smallfiles", "--bind_ip", "0.0.0.0"]
         volumes:
           - mongodata:/data/db

docker/raspberry_pi.md

@@ -49,7 +49,7 @@ services:
     command: -dbhost mongo
 
   mongo:
-    image: mongo:4.4
+    image: mongo:6.0
     command: --nojournal
 ```
 

scripts/dbList.sh

@@ -24,6 +24,4 @@ then
   exit 1
 fi
 
-mongo --quiet << EOF
-  show dbs
-EOF
+mongosh --eval 'show dbs' --quiet

scripts/dbReset.sh

@@ -32,11 +32,11 @@ then
     db=testharness
   fi
 
-  echo 'db.dropDatabase()' | mongo $db --quiet
+  mongosh $db --eval 'db.dropDatabase()' --quiet
 else
   while [ "$#" != 0 ]
   do
-    echo 'db.dropDatabase()' | mongo $1 --quiet
+    mongosh $1 --eval 'db.dropDatabase()' --quiet
     shift
   done
 fi

scripts/managedb/do_in_all_orion_dbs.sh

@@ -22,7 +22,7 @@
 MONGO_URI="mongodb://127.0.0.1:27017"
 #MONGO_URI="mongodb://mongodb1:27017,mongodb2:27017,mongodb3:27017/?replicaSet=cb_rs0"
 
-for db in $(echo 'show dbs' | mongo $MONGO_URI | grep '^orion' | awk -F ' ' '{print $1}')
+for db in $(mongosh $MONGO_URI --eval 'show dbs' --quiet | grep '^orion' | awk -F ' ' '{print $1}')
 do
   echo "Processing $db db"
   # Edit next line to set the script you want

test/functionalTest/cases/0725_multiservice_ignore/service_ignore_without_multiservice.test

@@ -27,7 +27,7 @@ Ignoring service header when not running in multiservice mode
 dbInit CB
 dbInit ${CB_DB_NAME}-foo
 dbResetAll
-echo "show dbs" | mongo | grep ${CB_DB_NAME} | awk '{ print $1 }'
+mongosh --eval 'show dbs' | grep ${CB_DB_NAME} | awk '{ print $1 }'
 
 brokerStart CB 0-255 IPV4
 
@@ -59,14 +59,14 @@ echo
 echo "2. Check that the number of orion databases is just 1"
 echo "====================================================="
 # In this case mongoCmd helper is not used, due to we don't have
-# a particular database for mongo command and the output could be
+# a particular database for mongosh command and the output could be
 # multiline in a general case
-echo "show dbs" | mongo | grep ${CB_DB_NAME} | wc -l
-echo "show dbs" | mongo | grep ${CB_DB_NAME}-foo | wc -l
+mongosh --eval 'show dbs' | grep ${CB_DB_NAME} | wc -l
+mongosh --eval 'show dbs' | grep ${CB_DB_NAME}-foo | wc -l
 # Next line was added in PR https://github.com/telefonicaid/fiware-orion/pull/1184/files#r37738638
 # in order to help us to detect where the ftest-ftest-ftest-... problem is. It could happen that
 # after fixing #1186 this is no longer needed, but let's keep it for a while as a "quarentine" measure...
-echo "show dbs" | mongo | grep ${CB_DB_NAME} | awk '{ print $1 }'
+mongosh --eval 'show dbs' | grep ${CB_DB_NAME} | awk '{ print $1 }'
 
 --REGEXPECT--
 1. Create an entity using foo as tenant

test/functionalTest/cases/1048_subscription_cache/get_subscription_before_and_after_cache_refresh.test

@@ -26,7 +26,7 @@ Get subscription before and after cache refresh
 --SHELL-INIT--
 dbInit CB
 
-mongoCmd ${CB_DB_NAME} 'db.csubs.insert({
+mongoCmd ${CB_DB_NAME} 'db.csubs.insertOne({
         "_id" : ObjectId("61960a51596aa2623032357d"),
         "expiration" : NumberLong("9223372036854775807"),
         "reference" : "http://localhost:1028/accumulate",
@@ -134,15 +134,15 @@ echo
 
 echo "02. Set dynamic fields in DB to set B"
 echo "====================================="
-mongoCmd ${CB_DB_NAME} 'db.csubs.update({_id: ObjectId("61960a51596aa2623032357d")}, {
+mongoCmd ${CB_DB_NAME} 'db.csubs.updateOne({_id: ObjectId("61960a51596aa2623032357d")}, {
   $set: {
-    lastNotification: NumberLong(1577876400),
-    lastSuccess: NumberLong(1577876400),
-    lastFailure: NumberLong(1577872800),
-    lastSuccessCode: NumberLong(204),
+    lastNotification: NumberLong("1577876400"),
+    lastSuccess: NumberLong("1577876400"),
+    lastFailure: NumberLong("1577872800"),
+    lastSuccessCode: NumberLong("204"),
     lastFailureReason: "Timeout",
-    count: NumberLong(25),
-    failsCounter: NumberLong(8),
+    count: NumberLong("25"),
+    failsCounter: NumberLong("8"),
     status: "inactive",
     statusLastChange: 1578733200
   }
@@ -217,7 +217,7 @@ Content-Length: 551
 
 02. Set dynamic fields in DB to set B
 =====================================
-WriteResult({ "nMatched" : 1, "nUpserted" : 0, "nModified" : 1 })
+{"acknowledged":true,"insertedId":null,"matchedCount":1,"modifiedCount":1,"upsertedCount":0}
 
 
 03. GET /v2/subscriptions and see dynamic fields set B although cache has not been synched yet

test/functionalTest/cases/1048_subscription_cache/get_subscription_before_and_after_cache_refresh_status_in_db_older.test

@@ -26,7 +26,7 @@ Get subscription before and after cache refresh with status older in DB during r
 --SHELL-INIT--
 dbInit CB
 
-mongoCmd ${CB_DB_NAME} 'db.csubs.insert({
+mongoCmd ${CB_DB_NAME} 'db.csubs.insertOne({
         "_id" : ObjectId("61960a51596aa2623032357d"),
         "expiration" : NumberLong("9223372036854775807"),
         "reference" : "http://localhost:1028/accumulate",
@@ -99,15 +99,15 @@ echo
 
 echo "02. Set dynamic fields in DB to set B"
 echo "====================================="
-mongoCmd ${CB_DB_NAME} 'db.csubs.update({_id: ObjectId("61960a51596aa2623032357d")}, {
+mongoCmd ${CB_DB_NAME} 'db.csubs.updateOne({_id: ObjectId("61960a51596aa2623032357d")}, {
   $set: {
-    lastNotification: NumberLong(1577876400),
-    lastSuccess: NumberLong(1577876400),
-    lastFailure: NumberLong(1577872800),
-    lastSuccessCode: NumberLong(204),
+    lastNotification: NumberLong("1577876400"),
+    lastSuccess: NumberLong("1577876400"),
+    lastFailure: NumberLong("1577872800"),
+    lastSuccessCode: NumberLong("204"),
     lastFailureReason: "Timeout",
-    count: NumberLong(25),
-    failsCounter: NumberLong(8),
+    count: NumberLong("25"),
+    failsCounter: NumberLong("8"),
     status: "inactive",
     statusLastChange: 1577869200
   }
@@ -182,7 +182,7 @@ Content-Length: 551
 
 02. Set dynamic fields in DB to set B
 =====================================
-WriteResult({ "nMatched" : 1, "nUpserted" : 0, "nModified" : 1 })
+{"acknowledged":true,"insertedId":null,"matchedCount":1,"modifiedCount":1,"upsertedCount":0}
 
 
 03. GET /v2/subscriptions and see dynamic fields set B although cache has not been synched yet

test/functionalTest/cases/1048_subscription_cache/refresh_cache_wins_notif_disabled.test

@@ -26,7 +26,7 @@ Cache refresh keeps newer information from cache in disabled notification scenar
 --SHELL-INIT--
 dbInit CB
 
-mongoCmd ${CB_DB_NAME} 'db.csubs.insert({
+mongoCmd ${CB_DB_NAME} 'db.csubs.insertOne({
         "_id" : ObjectId("61960a51596aa2623032357d"),
         "expiration" : NumberLong("9223372036854775807"),
         "reference" : "http://localhost:9997/notify",
@@ -131,7 +131,7 @@ echo
 
 echo "05. Get doc in DB before cache refresh and see status active changed in the past and without count and without failsCounter"
 echo "==========================================================================================================================="
-mongoCmd ${CB_DB_NAME} 'db.csubs.find({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | sed 's/NumberLong(\([[:digit:]]\+\))/\1/g' | sed 's/NumberLong("\([[:digit:]]\+\)")/\1/g' | python ${SCRIPT_HOME}/jsonBeautifier.py
+mongoCmd ${CB_DB_NAME} 'db.csubs.findOne({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | python ${SCRIPT_HOME}/jsonBeautifier.py
 echo
 echo
 
@@ -146,7 +146,7 @@ echo
 
 echo "07. Get doc in DB after cache refresh and see status inactive changed in the present and count 2, failsCounter 2"
 echo "================================================================================================================"
-mongoCmd ${CB_DB_NAME} 'db.csubs.find({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | sed 's/NumberLong(\([[:digit:]]\+\))/\1/g' | sed 's/NumberLong("\([[:digit:]]\+\)")/\1/g' | python ${SCRIPT_HOME}/jsonBeautifier.py
+mongoCmd ${CB_DB_NAME} 'db.csubs.findOne({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | python ${SCRIPT_HOME}/jsonBeautifier.py
 echo
 echo
 

test/functionalTest/cases/1048_subscription_cache/refresh_cache_wins_notif_disabled_with_update_in_the_middle.test

@@ -26,7 +26,7 @@ Cache refresh keeps newer information from cache in disabled notification scenar
 --SHELL-INIT--
 dbInit CB
 
-mongoCmd ${CB_DB_NAME} 'db.csubs.insert({
+mongoCmd ${CB_DB_NAME} 'db.csubs.insertOne({
         "_id" : ObjectId("61960a51596aa2623032357d"),
         "expiration" : NumberLong("9223372036854775807"),
         "reference" : "http://localhost:9997/notify",
@@ -133,7 +133,7 @@ echo
 
 echo "05. Get doc in DB before cache refresh and see status active changed in the past and without count and without failsCounter"
 echo "==========================================================================================================================="
-mongoCmd ${CB_DB_NAME} 'db.csubs.find({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | sed 's/NumberLong(\([[:digit:]]\+\))/\1/g' | sed 's/NumberLong("\([[:digit:]]\+\)")/\1/g' | python ${SCRIPT_HOME}/jsonBeautifier.py
+mongoCmd ${CB_DB_NAME} 'db.csubs.findOne({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | python ${SCRIPT_HOME}/jsonBeautifier.py
 echo
 echo
 
@@ -150,7 +150,7 @@ echo
 
 echo "07. Get doc in DB before cache refresh and see status active changed in the past and without count and without failsCounter + description"
 echo "========================================================================================================================================="
-mongoCmd ${CB_DB_NAME} 'db.csubs.find({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | sed 's/NumberLong(\([[:digit:]]\+\))/\1/g' | sed 's/NumberLong("\([[:digit:]]\+\)")/\1/g' | python ${SCRIPT_HOME}/jsonBeautifier.py
+mongoCmd ${CB_DB_NAME} 'db.csubs.findOne({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | python ${SCRIPT_HOME}/jsonBeautifier.py
 echo
 echo
 
@@ -165,7 +165,7 @@ echo
 
 echo "09. Get doc in DB after cache refresh and see status inactive changed in the present and count 2, failsCounter 2 + description"
 echo "=============================================================================================================================="
-mongoCmd ${CB_DB_NAME} 'db.csubs.find({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | sed 's/NumberLong(\([[:digit:]]\+\))/\1/g' | sed 's/NumberLong("\([[:digit:]]\+\)")/\1/g' | python ${SCRIPT_HOME}/jsonBeautifier.py
+mongoCmd ${CB_DB_NAME} 'db.csubs.findOne({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | python ${SCRIPT_HOME}/jsonBeautifier.py
 echo
 echo
 

test/functionalTest/cases/1048_subscription_cache/refresh_cache_wins_notif_fail.test

@@ -26,7 +26,7 @@ Cache refresh keeps newer information from cache in failed notification scenario
 --SHELL-INIT--
 dbInit CB
 
-mongoCmd ${CB_DB_NAME} 'db.csubs.insert({
+mongoCmd ${CB_DB_NAME} 'db.csubs.insertOne({
         "_id" : ObjectId("61960a51596aa2623032357d"),
         "expiration" : NumberLong("9223372036854775807"),
         "reference" : "http://localhost:9997/notify",
@@ -124,7 +124,7 @@ echo
 
 echo "04. Get doc in DB before cache refresh and see lastNotification/Failure in a the past and reason 'some reason' but no count and no failsCounter"
 echo "==============================================================================================================================================="
-mongoCmd ${CB_DB_NAME} 'db.csubs.find({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | sed 's/NumberLong(\([[:digit:]]\+\))/\1/g' | sed 's/NumberLong("\([[:digit:]]\+\)")/\1/g' | python ${SCRIPT_HOME}/jsonBeautifier.py
+mongoCmd ${CB_DB_NAME} 'db.csubs.findOne({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | python ${SCRIPT_HOME}/jsonBeautifier.py
 echo
 echo
 
@@ -139,7 +139,7 @@ echo
 
 echo "06. Get doc in DB after cache refresh and see lastNotification/Failure in the present and reason 'could not connect' and count 1 and failsCounter 1"
 echo "==================================================================================================================================================="
-mongoCmd ${CB_DB_NAME} 'db.csubs.find({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | sed 's/NumberLong(\([[:digit:]]\+\))/\1/g' | sed 's/NumberLong("\([[:digit:]]\+\)")/\1/g' | python ${SCRIPT_HOME}/jsonBeautifier.py
+mongoCmd ${CB_DB_NAME} 'db.csubs.findOne({ _id: ObjectId("61960a51596aa2623032357d")}, {_id:0, expiration: 0})' | python ${SCRIPT_HOME}/jsonBeautifier.py
 echo
 echo