From 400f6b08a1804c5b25705ae2fb443fc9218b5aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 1 Sep 2023 15:20:54 +0200 Subject: [PATCH 01/14] FIX make code to comiple under Debian 12 without losing compatibility with Debian 11 --- CMakeLists.txt | 6 ++++++ src/lib/serviceRoutines/versionTreat.cpp | 5 +++++ test/unittests/CMakeLists.txt | 6 ++---- .../ngsi9/DiscoverContextAvailabilityResponse_test.cpp | 4 ++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 37af9c0a65..22e6b2c052 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -171,6 +171,12 @@ endif (${CMAKE_BUILD_TYPE} STREQUAL DEBUG) # set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-variable") #endif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") +# Enables some some #ifdef in the code for compiling in old system +# FIXME: cleanup OLD_SSL_VERSION_FORMAT stuff after consolidating the change to Debian 12+ +IF(${DISTRO} MATCHES "Debian_11") + ADD_DEFINITIONS(-DOLD_SSL_VERSION_FORMAT) +ENDIF() + # # Libraries # diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index 90ff1e98c0..981c81f2b6 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -88,7 +88,12 @@ std::string libVersions(void) total += curl + "\"" + curlVersion + "\"" + ",\n"; total += mosq + "\"" + mosqVersion + "\"" + ",\n"; total += mhd + "\"" + MHD_get_version() + "\"" + ",\n"; +#ifdef OLD_SSL_VERSION_FORMAT + // Needed by openssl 1.1.1n in Debian 11 and before total += ssl + "\"" + SHLIB_VERSION_NUMBER "\"" + ",\n"; +#else + total += ssl + "\"" + OPENSSL_FULL_VERSION_STR "\"" + ",\n"; +#endif total += rjson + "\"" + RAPIDJSON_VERSION_STRING "\"" + ",\n"; total += mongo + "\"" + MONGOC_VERSION_S "\"" + ",\n"; total += bson + "\"" + BSON_VERSION_S "\"" + "\n"; diff --git a/test/unittests/CMakeLists.txt b/test/unittests/CMakeLists.txt index 9b429548a4..af8fbaf709 100644 --- a/test/unittests/CMakeLists.txt +++ b/test/unittests/CMakeLists.txt @@ -245,11 +245,9 @@ MESSAGE("unitTest distro: '${DISTRO}'") IF(${DISTRO} MATCHES "CentOS_6.*") TARGET_LINK_LIBRARIES(unitTest ${STATIC_LIBS} ${BOOST_MT} ${DYNAMIC_LIBS}) -ELSEIF((${DISTRO} MATCHES "Debian_8.*") OR (${DISTRO} MATCHES "Debian_9.*") OR (${DISTRO} MATCHES "Debian_10.*") OR (${DISTRO} MATCHES "Debian_11.*")) - # It seems that Debian 8.x/9.x doesn't like -mt libraries for unit tests... not sure about other Debian versions - TARGET_LINK_LIBRARIES(unitTest ${STATIC_LIBS} ${BOOST} ${DYNAMIC_LIBS}) +# From Debian 8.x to 12.x: they don't like -mt libraries for unit tests ELSEIF(${DISTRO} MATCHES "Debian_.*") - TARGET_LINK_LIBRARIES(unitTest ${STATIC_LIBS} ${BOOST_MT} ${DYNAMIC_LIBS}) + TARGET_LINK_LIBRARIES(unitTest ${STATIC_LIBS} ${BOOST} ${DYNAMIC_LIBS}) # The following ones are distros that haven't been checked since the refactoring in # PR #2062. They should be "promoted" as soon as we have positive feedback about diff --git a/test/unittests/ngsi9/DiscoverContextAvailabilityResponse_test.cpp b/test/unittests/ngsi9/DiscoverContextAvailabilityResponse_test.cpp index c913603920..35b987f713 100644 --- a/test/unittests/ngsi9/DiscoverContextAvailabilityResponse_test.cpp +++ b/test/unittests/ngsi9/DiscoverContextAvailabilityResponse_test.cpp @@ -117,7 +117,7 @@ TEST(DiscoverContextAvailabilityResponse, jsonRender) rendered = dcarP->toJsonV1(); EXPECT_STREQ(expectedBuf, rendered.c_str()); dcarP->release(); // ... otherwise the 500 remains and "pollutes" next tests - free(dcarP); + delete dcarP; // 2. One contextRegistrationResponse - errorCode inside ContextRegistrationResponse - only providingApplication in ContextRegistration @@ -343,7 +343,7 @@ TEST(DiscoverContextAvailabilityResponse, jsonRender) rendered = dcarP->toJsonV1(); EXPECT_STREQ(expectedBuf, rendered.c_str()); - free(dcarP); + delete dcarP; utExit(); } From 4345f21dde04f5db29573c555d3f21f8ceb3d02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 1 Sep 2023 15:24:07 +0200 Subject: [PATCH 02/14] FIX CMakeList.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22e6b2c052..b260ad8faa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,7 +173,7 @@ endif (${CMAKE_BUILD_TYPE} STREQUAL DEBUG) # Enables some some #ifdef in the code for compiling in old system # FIXME: cleanup OLD_SSL_VERSION_FORMAT stuff after consolidating the change to Debian 12+ -IF(${DISTRO} MATCHES "Debian_11") +IF(${DISTRO} MATCHES "Debian_11.*") ADD_DEFINITIONS(-DOLD_SSL_VERSION_FORMAT) ENDIF() From 546afd60217d52b6a5a211e5588b834eb96634b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 4 Sep 2023 13:42:54 +0200 Subject: [PATCH 03/14] FIX small modifications in ftest framework and gtest patch --- ...ps_notifications_no_accept_selfsigned.test | 2 +- test/functionalTest/harnessFunctions.sh | 5 + test/unittests/fuse_gtest_files.py.patch | 95 +++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 test/unittests/fuse_gtest_files.py.patch diff --git a/test/functionalTest/cases/0706_direct_https_notifications/direct_https_notifications_no_accept_selfsigned.test b/test/functionalTest/cases/0706_direct_https_notifications/direct_https_notifications_no_accept_selfsigned.test index d244c4b284..5450106eb1 100644 --- a/test/functionalTest/cases/0706_direct_https_notifications/direct_https_notifications_no_accept_selfsigned.test +++ b/test/functionalTest/cases/0706_direct_https_notifications/direct_https_notifications_no_accept_selfsigned.test @@ -92,7 +92,7 @@ echo "04. Look in the CB log for the warning about cert not accepted" echo "==============================================================" # Depdending the system the message in the logs may be diferent. For instance, it has been found than in -# CentOS8 and Debian 9 the message is "Peer certificate cannot be authenticated" but in Debian 11 +# CentOS8 and Debian 9 the message is "Peer certificate cannot be authenticated" but in Debian 11/12 # is "Peer certificate cannot be authenticated". At the end this error message comes from the SSH library grep "Peer certificate cannot be authenticated\|SSL peer certificate or SSH remote key was not OK" /tmp/contextBroker.log | grep "notification failure" | wc -l diff --git a/test/functionalTest/harnessFunctions.sh b/test/functionalTest/harnessFunctions.sh index ace7115ebc..08df1697f3 100644 --- a/test/functionalTest/harnessFunctions.sh +++ b/test/functionalTest/harnessFunctions.sh @@ -1047,6 +1047,11 @@ function valgrindSleep() # --quiet, but we need to get rid of some preamble lines about mongo version and # connection information and a final 'bye' line # +# NOTE: this will no longer work with new mongosh shell. Note that legacy shell +# (the 'mongo' command) has been deprecated in MongoDB 5.0 and removed in MongoDB 6.0. +# This function (and many .test using mongoCmd) would need fixing after stepping to +# MongoDB 6.0 +# function mongoCmd() { host="${CB_DATABASE_HOST}" diff --git a/test/unittests/fuse_gtest_files.py.patch b/test/unittests/fuse_gtest_files.py.patch new file mode 100644 index 0000000000..8cc5960853 --- /dev/null +++ b/test/unittests/fuse_gtest_files.py.patch @@ -0,0 +1,95 @@ +This patch adapt the original fuse_gest_files.py to Python 3 +Check build_source.md file to see how it is used + +--- fuse_gtest_files.py.orig 2023-09-01 11:26:02.036070350 +0200 ++++ fuse_gtest_files.py 2023-09-01 11:39:16.422565989 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/env python ++#!/usr/bin/env python3 + # + # Copyright 2009, Google Inc. + # All rights reserved. +@@ -60,7 +60,6 @@ + + import os + import re +-import sets + import sys + + # We assume that this file is in the scripts/ directory in the Google +@@ -90,8 +89,8 @@ + """ + + if not os.path.isfile(os.path.join(directory, relative_path)): +- print 'ERROR: Cannot find %s in directory %s.' % (relative_path, +- directory) ++ print ('ERROR: Cannot find %s in directory %s.' % (relative_path, ++ directory)) + print ('Please either specify a valid project root directory ' + 'or omit it on the command line.') + sys.exit(1) +@@ -123,7 +122,7 @@ + (relative_path, output_dir)) + answer = sys.stdin.readline().strip() + if answer not in ['y', 'Y']: +- print 'ABORTED.' ++ print ('ABORTED.') + sys.exit(1) + + # Makes sure the directory holding the output file exists; creates +@@ -146,8 +145,8 @@ + def FuseGTestH(gtest_root, output_dir): + """Scans folder gtest_root to generate gtest/gtest.h in output_dir.""" + +- output_file = file(os.path.join(output_dir, GTEST_H_OUTPUT), 'w') +- processed_files = sets.Set() # Holds all gtest headers we've processed. ++ output_file = open(os.path.join(output_dir, GTEST_H_OUTPUT), 'w') ++ processed_files = set() # Holds all gtest headers we've processed. + + def ProcessFile(gtest_header_path): + """Processes the given gtest header file.""" +@@ -159,7 +158,7 @@ + processed_files.add(gtest_header_path) + + # Reads each line in the given gtest header. +- for line in file(os.path.join(gtest_root, gtest_header_path), 'r'): ++ for line in open(os.path.join(gtest_root, gtest_header_path), 'r'): + m = INCLUDE_GTEST_FILE_REGEX.match(line) + if m: + # It's '#include ' - let's process it recursively. +@@ -175,7 +174,7 @@ + def FuseGTestAllCcToFile(gtest_root, output_file): + """Scans folder gtest_root to generate gtest/gtest-all.cc in output_file.""" + +- processed_files = sets.Set() ++ processed_files = set() + + def ProcessFile(gtest_source_file): + """Processes the given gtest source file.""" +@@ -187,7 +186,7 @@ + processed_files.add(gtest_source_file) + + # Reads each line in the given gtest source file. +- for line in file(os.path.join(gtest_root, gtest_source_file), 'r'): ++ for line in open(os.path.join(gtest_root, gtest_source_file), 'r'): + m = INCLUDE_GTEST_FILE_REGEX.match(line) + if m: + if 'include/' + m.group(1) == GTEST_SPI_H_SEED: +@@ -218,7 +217,7 @@ + def FuseGTestAllCc(gtest_root, output_dir): + """Scans folder gtest_root to generate gtest/gtest-all.cc in output_dir.""" + +- output_file = file(os.path.join(output_dir, GTEST_ALL_CC_OUTPUT), 'w') ++ output_file = open(os.path.join(output_dir, GTEST_ALL_CC_OUTPUT), 'w') + FuseGTestAllCcToFile(gtest_root, output_file) + output_file.close() + +@@ -242,7 +241,7 @@ + # fuse_gtest_files.py GTEST_ROOT_DIR OUTPUT_DIR + FuseGTest(sys.argv[1], sys.argv[2]) + else: +- print __doc__ ++ print (__doc__) + sys.exit(1) + + From 97c08fc1aae3f0708f8b70fe0d40267ba20bbb01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 4 Sep 2023 13:47:19 +0200 Subject: [PATCH 04/14] ADD .patch files to ingore list in check files compliance --- scripts/check_files_compliance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_files_compliance.py b/scripts/check_files_compliance.py index 9c81db6399..0555aee3c1 100755 --- a/scripts/check_files_compliance.py +++ b/scripts/check_files_compliance.py @@ -129,7 +129,7 @@ def ignore(root, file): return True # Apib files have an "inline" license, so they are ignored - extensions_to_ignore = [ 'apib', 'md' ] + extensions_to_ignore = [ 'apib', 'md', 'patch'] if os.path.splitext(file)[1][1:] in extensions_to_ignore: return True From 224ace3e33c8f26a355146a66d9b6229c98a2eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 4 Sep 2023 18:31:47 +0200 Subject: [PATCH 05/14] FIX Dockerfile --- docker/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 96c143c1ad..79e486a97d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ # FIXME the change from mongoc driver 1.23.1 to 1.24.1 (PR https://github.com/telefonicaid/fiware-orion/pull/4415) # has not been actually tested. Remove this FIXME mark after succesfull test -ARG IMAGE_TAG=11.6-slim +ARG IMAGE_TAG=12.1-slim FROM debian:${IMAGE_TAG} ARG GITHUB_ACCOUNT=telefonicaid @@ -149,8 +149,7 @@ RUN \ apt-get -y autoremove --purge && \ # The above apt removal erases some dependencies needed by contextBroker. So we reinstall it apt-get -y install \ - libicu67 \ - libcurl4 \ + libicu72 \ libboost-thread1.74.0 && \ # Don't need old log files inside docker images rm -f /var/log/*log From 0f45fe86b0286908b3a1efd036e3b82a19f60409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 5 Sep 2023 11:57:06 +0200 Subject: [PATCH 06/14] FIX little fixes in makefile and testHarness.sh --- makefile | 2 ++ test/functionalTest/testHarness.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/makefile b/makefile index a279b0adfb..594f21f2d3 100644 --- a/makefile +++ b/makefile @@ -234,6 +234,7 @@ ftd: functional_test_debug test: unit_test functional_test coverage: install_coverage + # FIXME #4418: the functional test part of this target is not working properly. Check issue for details. # Init coverage echo "Initializing coverage files" mkdir -p coverage @@ -291,6 +292,7 @@ coverage_unit_test: build_unit_test genhtml -o coverage coverage/broker.info coverage_functional_test: install_coverage + # FIXME #4418: this target is not working properly. Check issue for details. # Init coverage echo "Initializing coverage files" mkdir -p coverage diff --git a/test/functionalTest/testHarness.sh b/test/functionalTest/testHarness.sh index 74b799a495..f20328e79b 100755 --- a/test/functionalTest/testHarness.sh +++ b/test/functionalTest/testHarness.sh @@ -914,7 +914,8 @@ function testDisabled dIx=0 while [ $dIx -lt ${#DISABLED[@]} ] do - if [ test/functionalTest/cases/$testcase == ${DISABLED[$dIx]} ] + # Comparison is done based in filename, skipping the path (see https://stackdiary.com/tutorials/bash-get-filename-from-path/) + if [ "${testcase##*/}" == "${DISABLED[$dIx]##*/}" ] then echo "Disabled" From 2d870f70d8866b7f22291c23362d74a6c5b9cb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 5 Sep 2023 14:27:20 +0200 Subject: [PATCH 07/14] FIX ci docker image --- ci/deb/Dockerfile | 3 ++- ci/deb/build-dep.sh | 19 ++++++++++--------- .../deb}/fuse_gtest_files.py.patch | 0 3 files changed, 12 insertions(+), 10 deletions(-) rename {test/unittests => ci/deb}/fuse_gtest_files.py.patch (100%) diff --git a/ci/deb/Dockerfile b/ci/deb/Dockerfile index 3b6a2fa86d..42a635fd3e 100644 --- a/ci/deb/Dockerfile +++ b/ci/deb/Dockerfile @@ -1,8 +1,9 @@ -FROM debian:11.6-slim +FROM debian:12.1-slim ADD build.sh /opt/bin/ ADD build-dep.sh /opt/bin/ ADD makefile /opt/archive/ +ADD fuse_gtest_files.py.patch /opt/archive/ RUN ln -s /opt/bin/build.sh /usr/local/bin/build \ && /opt/bin/build-dep.sh diff --git a/ci/deb/build-dep.sh b/ci/deb/build-dep.sh index 702a6b623c..5ee98b8e5e 100755 --- a/ci/deb/build-dep.sh +++ b/ci/deb/build-dep.sh @@ -25,15 +25,14 @@ # Install security updates apt-get -y update apt-get -y upgrade -# FIXME: python2 required by an installation script in GMock. Sad but true :( # Install dependencies apt-get -y install \ curl \ gnupg \ - python2 \ python3 \ python3-pip \ - netcat \ + python3-venv \ + netcat-traditional \ bc \ valgrind \ cmake \ @@ -50,7 +49,11 @@ apt-get -y install \ libsasl2-dev \ libgcrypt-dev +## FIXME: check note in build_source.md about the libssl1 installation hack. It will be no longer needed from MongoDB 6.0 on echo "INSTALL: MongoDB shell" \ +&& curl -L http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb --output libssl1.1_1.1.1f-1ubuntu2_amd64.deb \ +&& dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb \ +&& rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb \ && curl -L https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - \ && echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list \ && apt-get -y update \ @@ -58,17 +61,16 @@ echo "INSTALL: MongoDB shell" \ echo "INSTALL: python special dependencies" \ && cd /opt \ -&& pip3 install virtualenv \ -&& virtualenv /opt/ft_env --python=/usr/bin/python3 \ +&& python3 -m venv /opt/ft_env \ && . /opt/ft_env/bin/activate \ && pip install Flask==2.0.2 \ && pip install paho-mqtt==1.6.1 \ -&& pip install amqtt==0.10.1 \ +&& pip install amqtt==0.11.0b1 \ && deactivate # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0 # see http://mongoc.org/libmongoc/current/init-cleanup.html#deprecated-feature-automatic-initialization-and-cleanup -echo "INSTALL: mongodb c driver (required by mongo c++ driver)" \ +echo "INSTALL: mongodb c driver" \ && curl -L https://github.com/mongodb/mongo-c-driver/releases/download/1.24.3/mongo-c-driver-1.24.3.tar.gz | tar xzC /opt/ \ && cd /opt/mongo-c-driver-1.24.3 \ && mkdir cmake-build \ @@ -88,11 +90,10 @@ echo "INSTALL: libmicrohttpd" \ && make \ && make install -# FIXME: if sometimes python2 goes away the fuse_gtest_files.py would need to be migrated to python3 echo "INSTALL: gmock" \ && curl -L https://src.fedoraproject.org/repo/pkgs/gmock/gmock-1.5.0.tar.bz2/d738cfee341ad10ce0d7a0cc4209dd5e/gmock-1.5.0.tar.bz2 | tar xjC /opt/ \ && cd /opt/gmock-1.5.0 \ -&& sed -i 's/env python/env python2/' gtest/scripts/fuse_gtest_files.py \ +&& patch -p1 gtest/scripts/fuse_gtest_files.py < /opt/archive/fuse_gtest_files.py.patch \ && ./configure \ && make \ && make install diff --git a/test/unittests/fuse_gtest_files.py.patch b/ci/deb/fuse_gtest_files.py.patch similarity index 100% rename from test/unittests/fuse_gtest_files.py.patch rename to ci/deb/fuse_gtest_files.py.patch From 74364e28f43323c14102d059fb75e7362cc6dbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Sep 2023 08:39:49 +0200 Subject: [PATCH 08/14] FIX align .test with Debian 12 curl new way of working --- .../more_than_one_bad_input.test | 16 +++- .../csub_cache_objects_errors.test | 87 ++++++++++--------- test/functionalTest/testHarness.sh | 2 +- 3 files changed, 58 insertions(+), 47 deletions(-) diff --git a/test/functionalTest/cases/1697_more_than_one_bad_input/more_than_one_bad_input.test b/test/functionalTest/cases/1697_more_than_one_bad_input/more_than_one_bad_input.test index dc105394f8..2f95550b78 100644 --- a/test/functionalTest/cases/1697_more_than_one_bad_input/more_than_one_bad_input.test +++ b/test/functionalTest/cases/1697_more_than_one_bad_input/more_than_one_bad_input.test @@ -71,6 +71,14 @@ echo echo "04. grep BadInput in log-file (4 lines found)" echo "=============================================" +# FIXME: in Debian 11 IP is 0.0.0.0 while in Debian 12 is 127.0.0.1. This could be due to changes in how +# curl (used internally by orionCurl) works, from 7.74 (Debian 11) to 7.88 (Debian 12) (it wouldn't be +# the only change, see https://stackoverflow.com/q/77045755/1485926 ;). I have tried to fix +# using explictely 'curl --interface 0.0.0.0' in orionCurl function, but it doesn't work. Thus, the +# alternative is to use a REGEX() so both IP options get covered. This can be seen as a temporal fix, so +# when we decide to remove compatibility with Debian 11 we can remove the REGEX() and leave only one way +# (it seems we have already used this trick is other .test files, look for '0.0.0.0|127.0.0.1' text pattern, +# maybe all them could be removed at the same time) grep BadInput /tmp/contextBroker.log | awk -F'|' '{ print $10 }' | awk -F= '{ print $2 }' echo echo @@ -138,10 +146,10 @@ Bad input active alarms: 1, raised: _total: 1, new: 1_, released: _total: 0, new 04. grep BadInput in log-file (4 lines found) ============================================= -Raising alarm BadInput 0.0.0.0: service '/v2/noService' not found -Repeated BadInput 0.0.0.0: JSON Parse Error (Invalid value) -Repeated BadInput 0.0.0.0: bad verb for url '/v2/entities', method 'GETT' -Repeated BadInput 0.0.0.0: {"error":"ContentLengthRequired","description":"Zero/No Content-Length in PUT/POST/PATCH request"} +Raising alarm BadInput REGEX((0.0.0.0|127.0.0.1)): service '/v2/noService' not found +Repeated BadInput REGEX((0.0.0.0|127.0.0.1): JSON Parse Error \(Invalid value\)) +Repeated BadInput REGEX((0.0.0.0|127.0.0.1)): bad verb for url '/v2/entities', method 'GETT' +Repeated BadInput REGEX((0.0.0.0|127.0.0.1)): {"error":"ContentLengthRequired","description":"Zero/No Content-Length in PUT/POST/PATCH request"} --TEARDOWN-- diff --git a/test/functionalTest/cases/1705_csub_cache_objects/csub_cache_objects_errors.test b/test/functionalTest/cases/1705_csub_cache_objects/csub_cache_objects_errors.test index 3db0e3161f..beb48d3393 100644 --- a/test/functionalTest/cases/1705_csub_cache_objects/csub_cache_objects_errors.test +++ b/test/functionalTest/cases/1705_csub_cache_objects/csub_cache_objects_errors.test @@ -119,20 +119,23 @@ echo echo -echo "09. GET /v2/entities with only whitespace in q-item (q= )" -echo "===========================================================" -q='q= ' -orionCurl --url "/v2/entities?$q" -echo -echo - - -echo "10. GET /v2/entities with trailing whitespace in q (q=a==12; )" -echo "================================================================" -q='q=a==12; ' -orionCurl --url "/v2/entities?$q" -echo -echo +# FIXME: curl has changed its behaviour from Debian 11 to Debian 12 (see https://stackoverflow.com/q/77045755/1485926) +# and now it is not a valid tool to test malformed URLs. Thus, we are commenting this fields while we look for +# another tool or avoid curl to work in the old way +#echo "09. GET /v2/entities with only whitespace in q-item (q= )" +#echo "===========================================================" +#q='q= ' +#orionCurl --url "/v2/entities?$q" +#echo +#echo +# +# +#echo "10. GET /v2/entities with trailing whitespace in q (q=a==12; )" +#echo "================================================================" +#q='q=a==12; ' +#orionCurl --url "/v2/entities?$q" +#echo +#echo echo "11. GET /v2/entities with > RANGE" @@ -333,34 +336,34 @@ Content-Length: 79 } -09. GET /v2/entities with only whitespace in q-item (q= ) -=========================================================== -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 77 - -{ - "description": "empty q-item (only whitespace)", - "error": "BadRequest" -} - - -10. GET /v2/entities with trailing whitespace in q (q=a==12; ) -================================================================ -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 51 - -{ - "description": "empty q-item", - "error": "BadRequest" -} - - +#09. GET /v2/entities with only whitespace in q-item (q= ) +#=========================================================== +#HTTP/1.1 400 Bad Request +#Date: REGEX(.*) +#Fiware-Correlator: REGEX([0-9a-f\-]{36}) +#Content-Type: application/json +#Content-Length: 77 +# +#{ +# "description": "empty q-item (only whitespace)", +# "error": "BadRequest" +#} +# +# +#10. GET /v2/entities with trailing whitespace in q (q=a==12; ) +#================================================================ +#HTTP/1.1 400 Bad Request +#Date: REGEX(.*) +#Fiware-Correlator: REGEX([0-9a-f\-]{36}) +#Content-Type: application/json +#Content-Length: 51 +# +#{ +# "description": "empty q-item", +# "error": "BadRequest" +#} +# +# 11. GET /v2/entities with > RANGE ================================= HTTP/1.1 400 Bad Request diff --git a/test/functionalTest/testHarness.sh b/test/functionalTest/testHarness.sh index f20328e79b..b49e26172b 100755 --- a/test/functionalTest/testHarness.sh +++ b/test/functionalTest/testHarness.sh @@ -632,7 +632,7 @@ function fileCreation() then TEST_REGEXPECT=${pathWithoutExt}.regexpect vMsg "Creating $TEST_REGEXPECT at $PWD" - sed -n '/--REGEXPECT--/,/^--/p' $path | grep -v "^--" > $TEST_REGEXPECT + sed -n '/--REGEXPECT--/,/^--/p' $path | grep -v "^--" | sed '/^#/d' > $TEST_REGEXPECT else exitFunction 5 "--REGEXPECT-- part is missing" $path "($path)" "" DIE fi From c3c827d3103e44a8f2ff5acd51a79f03e4302b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Sep 2023 09:48:55 +0200 Subject: [PATCH 09/14] FIX ftest --- .../csub_cache_objects_errors.test | 56 +++++++++---------- test/functionalTest/testHarness.sh | 2 +- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/test/functionalTest/cases/1705_csub_cache_objects/csub_cache_objects_errors.test b/test/functionalTest/cases/1705_csub_cache_objects/csub_cache_objects_errors.test index beb48d3393..08804a2c3c 100644 --- a/test/functionalTest/cases/1705_csub_cache_objects/csub_cache_objects_errors.test +++ b/test/functionalTest/cases/1705_csub_cache_objects/csub_cache_objects_errors.test @@ -336,34 +336,34 @@ Content-Length: 79 } -#09. GET /v2/entities with only whitespace in q-item (q= ) -#=========================================================== -#HTTP/1.1 400 Bad Request -#Date: REGEX(.*) -#Fiware-Correlator: REGEX([0-9a-f\-]{36}) -#Content-Type: application/json -#Content-Length: 77 -# -#{ -# "description": "empty q-item (only whitespace)", -# "error": "BadRequest" -#} -# -# -#10. GET /v2/entities with trailing whitespace in q (q=a==12; ) -#================================================================ -#HTTP/1.1 400 Bad Request -#Date: REGEX(.*) -#Fiware-Correlator: REGEX([0-9a-f\-]{36}) -#Content-Type: application/json -#Content-Length: 51 -# -#{ -# "description": "empty q-item", -# "error": "BadRequest" -#} -# -# +##09. GET /v2/entities with only whitespace in q-item (q= ) +##=========================================================== +##HTTP/1.1 400 Bad Request +##Date: REGEX(.*) +##Fiware-Correlator: REGEX([0-9a-f\-]{36}) +##Content-Type: application/json +##Content-Length: 77 +## +##{ +## "description": "empty q-item (only whitespace)", +## "error": "BadRequest" +##} +## +## +##10. GET /v2/entities with trailing whitespace in q (q=a==12; ) +##================================================================ +##HTTP/1.1 400 Bad Request +##Date: REGEX(.*) +##Fiware-Correlator: REGEX([0-9a-f\-]{36}) +##Content-Type: application/json +##Content-Length: 51 +## +##{ +## "description": "empty q-item", +## "error": "BadRequest" +##} +## +## 11. GET /v2/entities with > RANGE ================================= HTTP/1.1 400 Bad Request diff --git a/test/functionalTest/testHarness.sh b/test/functionalTest/testHarness.sh index b49e26172b..c309c4e8c2 100755 --- a/test/functionalTest/testHarness.sh +++ b/test/functionalTest/testHarness.sh @@ -632,7 +632,7 @@ function fileCreation() then TEST_REGEXPECT=${pathWithoutExt}.regexpect vMsg "Creating $TEST_REGEXPECT at $PWD" - sed -n '/--REGEXPECT--/,/^--/p' $path | grep -v "^--" | sed '/^#/d' > $TEST_REGEXPECT + sed -n '/--REGEXPECT--/,/^--/p' $path | grep -v "^--" | sed '/^##/d' > $TEST_REGEXPECT else exitFunction 5 "--REGEXPECT-- part is missing" $path "($path)" "" DIE fi From 16fced2c2a6427eb3a560be3608accdd3f8b364c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Sep 2023 10:18:18 +0200 Subject: [PATCH 10/14] FIX documentation --- doc/manuals.jp/admin/build_source.md | 16 +++++++------- doc/manuals.jp/admin/install.md | 4 ++-- doc/manuals/admin/build_source.md | 31 ++++++++++++++++------------ doc/manuals/admin/install.md | 4 ++-- doc/manuals/devel/cookbook.md | 24 ++++++++------------- test/functionalTest/README.jp.md | 2 +- test/functionalTest/README.md | 2 +- test/functionalTest/cases/README.md | 8 +------ 8 files changed, 42 insertions(+), 49 deletions(-) diff --git a/doc/manuals.jp/admin/build_source.md b/doc/manuals.jp/admin/build_source.md index a6ceb0c257..5fba262d95 100644 --- a/doc/manuals.jp/admin/build_source.md +++ b/doc/manuals.jp/admin/build_source.md @@ -1,6 +1,6 @@ # ソースからのビルド -Orion Context Broker のリファレンス配布は Debian 11 です。これは、broker を他のディストリビューションに組み込むことができないことを意味しません (実際には可能です)。このセクションでは、他のディストリビューションをビルドする方法についても説明しています。Debian を使用していない人に役立つかもしれません。ただし、"公式にサポートされている" 唯一の手順は Debian 11 用の手順です。他のものは "現状のまま" 提供され、随時時代遅れになる可能性があります。 +Orion Context Broker のリファレンス配布は Debian 12 です。これは、broker を他のディストリビューションに組み込むことができないことを意味しません (実際には可能です)。このセクションでは、他のディストリビューションをビルドする方法についても説明しています。Debian を使用していない人に役立つかもしれません。ただし、"公式にサポートされている" 唯一の手順は Debian 12 用の手順です。他のものは "現状のまま" 提供され、随時時代遅れになる可能性があります。 ## Debian 11 (正式サポート) @@ -8,9 +8,9 @@ Orion Context Broker は、以下のライブラリをビルドの依存関係 * boost: 1.74 * libmicrohttpd: 0.9.76 (ソースから) -* libcurl: 7.74.0 -* openssl: 1.1.1n -* libuuid: 2.36.1 +* libcurl: 7.88.1 +* openssl: 3.0.9 +* libuuid: 2.38.1 * libmosquitto: 2.0.15 (ソースから) * Mongo C driver: 1.24.3 (ソースから) * rapidjson: 1.1.0 (ソースから) @@ -94,12 +94,12 @@ Orion Context Broker には、次の手順に従って実行できる一連の * ソースから GoogleTest/Mock をインストールします。以前の URL は http://googlemock.googlecode.com/files/gmock-1.5.0.tar.bz2 でしたが、Google は2016年8月下旬にそのパッケージを削除し、機能しなくなりました。 - sudo apt-get install python2 wget https://nexus.lab.fiware.org/repository/raw/public/storage/gmock-1.5.0.tar.bz2 tar xfvj gmock-1.5.0.tar.bz2 cd gmock-1.5.0 ./configure - sed -i 's/env python/env python2/' gtest/scripts/fuse_gtest_files.py # little hack to make installation to work on Debian 11 + # Adjust /path/to/fiware-orion in the next line accordingly to where you local copy of fiware-orion repo is in your system + patch -p1 gtest/scripts/fuse_gtest_files.py < /path/to/fiware-orion/test/unittests/fuse_gtest_files.py.patch make sudo make install # installation puts .h files in /usr/local/include and library in /usr/local/lib sudo ldconfig # just in case... it doesn't hurt :) @@ -123,9 +123,9 @@ aarch64 アーキテクチャの場合、apt-get を使用して libxslt をイ export PATH=~/bin:$PATH make install_scripts INSTALL_DIR=~ . scripts/testEnv.sh - virtualenv /opt/ft_env --python=/usr/bin/python3 + python3 -m venv /opt/ft_env # or 'virtualenv /opt/ft_env --python=/usr/bin/python3' in some systems . /opt/ft_env/bin/activate - pip install Flask==2.0.2 paho-mqtt==1.6.1 amqtt==0.10.1 + pip install Flask==2.0.2 paho-mqtt==1.6.1 amqtt==0.11.0b1 * この環境でテスト・ハーネスを実行してください (時間がかかりますので、気をつけてください) diff --git a/doc/manuals.jp/admin/install.md b/doc/manuals.jp/admin/install.md index c29306aff9..cd73b7ec67 100644 --- a/doc/manuals.jp/admin/install.md +++ b/doc/manuals.jp/admin/install.md @@ -26,8 +26,8 @@ Docker hub で公式の Orion docker コンテナを使用してインストー 必要なソースから ビルドした Orion をインストールする場合: -* オペレーティングシステム: Debian。リファレンス・オペレーティングシステムは Debian 11.6 ですが、それ以降の - Debian 11 バージョンでも動作するはずです +* オペレーティングシステム: Debian。リファレンス・オペレーティングシステムは Debian 12.1 ですが、それ以降の + Debian 12 バージョンでも動作するはずです * データベース: MongoDB は、Orion Context Broker がインストールされるのと同じホストで実行するか、ネットワーク経由で アクセスできる別のホストで実行する必要があります。推奨される MongoDB バージョンは 4.4 です (Orion は古いバージョンで 動作する可能性がありますが、まったくお勧めしません!) diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index a153b95d31..67a6ed6f48 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -1,16 +1,16 @@ # Building from sources -Orion Context Broker reference distribution is Debian 11. This doesn't mean that the broker cannot be built in other distributions (actually, it can). This section also includes indications on how to build in other distributions, just in the case it may help people that don't use Debian. However, note that the only "officially supported" procedure is the one for Debian 11; the others are provided "as is" and can get obsolete from time to time. +Orion Context Broker reference distribution is Debian 12. This doesn't mean that the broker cannot be built in other distributions (actually, it can). This section also includes indications on how to build in other distributions, just in the case it may help people that don't use Debian. However, note that the only "officially supported" procedure is the one for Debian 12; the others are provided "as is" and can get obsolete from time to time. -## Debian 11 (officially supported) +## Debian 12 (officially supported) The Orion Context Broker uses the following libraries as build dependencies: * boost: 1.74 * libmicrohttpd: 0.9.76 (from source) -* libcurl: 7.74.0 -* openssl: 1.1.1n -* libuuid: 2.36.1 +* libcurl: 7.88.1 +* openssl: 3.0.9 +* libuuid: 2.38.1 * libmosquitto: 2.0.15 (from source) * Mongo C driver: 1.24.3 (from source) * rapidjson: 1.1.0 (from source) @@ -93,19 +93,25 @@ The Orion Context Broker comes with a suite of unit, valgrind and end-to-end tes * Install Google Test/Mock from sources. Previously the URL was http://googlemock.googlecode.com/files/gmock-1.5.0.tar.bz2 but Google removed that package in late August 2016 and it is no longer working. - sudo apt-get install python2 wget https://nexus.lab.fiware.org/repository/raw/public/storage/gmock-1.5.0.tar.bz2 tar xfvj gmock-1.5.0.tar.bz2 cd gmock-1.5.0 ./configure - sed -i 's/env python/env python2/' gtest/scripts/fuse_gtest_files.py # little hack to make installation to work on Debian 11 + # Adjust /path/to/fiware-orion in the next line accordingly to where you local copy of fiware-orion repo is in your system + patch -p1 gtest/scripts/fuse_gtest_files.py < /path/to/fiware-orion/ci/deb/fuse_gtest_files.py.patch make sudo make install # installation puts .h files in /usr/local/include and library in /usr/local/lib sudo ldconfig # just in case... it doesn't hurt :) In the case of the aarch64 architecture, install libxslt using apt-get, and run `./configure` with `--build=arm-linux` option. -* Install MongoDB (tests rely on mongod running in localhost). Check [the official MongoDB documentation](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-red-hat/) for details. Recommended version is 4.4 (it may work with previous versions, but we don't recommend it). +* Install MongoDB (tests rely on mongod running in localhost). Check [the official MongoDB documentation](hhttps://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-debian/) for details. Recommended version is 4.4 (it may work with previous versions, but we don't recommend it). + * Note that mongo legacy shell (the `mongo` command) has been deprecated in MongoDB 5 and removed in MongoDB 6 in favour of the new shell (`mongosh` command). Some functional tests (ftest) will fail due to this if you use MongoDB 6 or beyond, as they are suited to use `mongo` and not `mongosh`. + * Debian 12 has stepped to libssl3 but some MongoDB versions may require libssl1. In the case you get a `Depends: libssl1.1 (>= 1.1.1) but it is not installable` error, you can test the following (reference [here](https://askubuntu.com/a/1421959)) + + wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb + sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb + rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb # optional, for cleanness * Run unit test @@ -113,8 +119,7 @@ In the case of the aarch64 architecture, install libxslt using apt-get, and run * Install additional required tools for functional and valgrind tests: - sudo apt-get install curl nc valgrind bc python3 python3-pip - sudo pip3 install virtualenv + sudo apt-get install curl netcat-traditional valgrind bc python3 python3-pip mosquitto * Prepare the environment for test harness. Basically, you have to install the `accumulator-server.py` script and in a path under your control, `~/bin` is the recommended one. Alternatively, you can install them in a system directory such as `/usr/bin` but it could collide with an other programs, thus it is not recommended. In addition, you have to set several environment variables used by the harness script (see `scripts/testEnv.sh` file) and create a virtualenv environment with the required Python packages. @@ -122,9 +127,9 @@ In the case of the aarch64 architecture, install libxslt using apt-get, and run export PATH=~/bin:$PATH make install_scripts INSTALL_DIR=~ . scripts/testEnv.sh - virtualenv /opt/ft_env --python=/usr/bin/python3 + python3 -m venv /opt/ft_env # or 'virtualenv /opt/ft_env --python=/usr/bin/python3' in some systems . /opt/ft_env/bin/activate - pip install Flask==2.0.2 paho-mqtt==1.6.1 amqtt==0.10.1 + pip install Flask==2.0.2 paho-mqtt==1.6.1 amqtt==0.11.0b1 * Run test harness in this environment (it takes some time, please be patient). @@ -138,7 +143,7 @@ You can generate coverage reports for the Orion Context Broker using the followi * Install the lcov tool - sudo apt-get install lcov + sudo apt-get install lcov xsltproc * Do first a successful pass for unit_test and functional_test, to check that everything is ok (see above) diff --git a/doc/manuals/admin/install.md b/doc/manuals/admin/install.md index a915685d79..80fdf6b09f 100644 --- a/doc/manuals/admin/install.md +++ b/doc/manuals/admin/install.md @@ -23,8 +23,8 @@ In the case you install using the official Orion docker container at Dockerhub, In the case you are installing Orion building from sources you need: -* Operating system: Debian. The reference operating system is Debian 11.6 - but it should work also in any later Debian 11 version. +* Operating system: Debian. The reference operating system is Debian 12.1 + but it should work also in any later Debian 12 version. * Database: MongoDB is required to run either in the same host where Orion Context Broker is to be installed or in a different host accessible through the network. The recommended MongoDB version is 4.4 (Orion may work with older versions but we don't recommend it at all!). diff --git a/doc/manuals/devel/cookbook.md b/doc/manuals/devel/cookbook.md index 100a163302..5d75a4ddff 100644 --- a/doc/manuals/devel/cookbook.md +++ b/doc/manuals/devel/cookbook.md @@ -187,7 +187,7 @@ A functional test file contains six sections: 2. NAME section 3. SHELL-INIT section 4. SHELL section -5. EXPECT/REGEXPECT section +5. REGEXPECT section 6. TEARDOWN section Each section (except the Copyright preamble, that starts at the beginning of the file) must have a header, that tells the functional test harness where every section starts/ends: @@ -195,11 +195,9 @@ Each section (except the Copyright preamble, that starts at the beginning of the * `--NAME--` * `--SHELL-INIT--` * `--SHELL--` -* `--REGEXPECT--` / `--EXPECT--` +* `--REGEXPECT--` * `--TEARDOWN--` -If `--REGEXPECT--` is used (and not `--EXPECT--`), then the expected section permits regular expressions. That is the only different between these two. - ### Copyright section This section is simply for the Copyright header. Copy an old one. Try to remember to change the year, if necessary. @@ -275,7 +273,7 @@ echo "0x. description of test step 0x" echo "===============================" ``` -and the steps end with two calls to `echo`, to separate the current step from the next in the output. This is pretty important as it makes it **so much** easier to read the output, which must match what is in the section that follows, the **EXPECT/REGEXPECT** section. +and the steps end with two calls to `echo`, to separate the current step from the next in the output. This is pretty important as it makes it **so much** easier to read the output, which must match what is in the section that follows, the **REGEXPECT** section. A typical step (e.g. to create an entity) looks like this: @@ -300,22 +298,16 @@ echo echo ``` -### EXPECT/REGEXPECT Section +### REGEXPECT Section First of all, the test harness (`test/functionalTest/testHarness.sh`) admits two types of 'expect sections'. Either ``` ---EXPECT-- -``` - -or - -``` ---REG-EXPECT-- +--REGEXPECT-- ``` -You have to **pick one**. Pretty much **all** current functests use the `--REG-EXPECT--` type. The advantage with --REG-EXPECT-- is that it permits to add regular expressions using the `REGEX()` syntax, which is very important for the comparison of dates, or IDs created by Orion and returned in the response, like a registration id or a correlator or a simple timestamp. An important limitation is that there can only be **one REGEX** per line in the REG-EXPECT section. +It permits to add regular expressions using the `REGEX()` syntax, which is very important for the comparison of dates, or IDs created by Orion and returned in the response, like a registration id or a correlator or a simple timestamp. An important limitation is that there can only be **one REGEX** per line in the REGEXPECT section. -That said, in the REG-EXPECT section, just add what is the expected output from the test step in question. For example, the example "01. Create entity E1 with attribute A1" from the above sub-chapter about the SHELL section would +That said, just add what is the expected output from the test step in question. For example, the example "01. Create entity E1 with attribute A1" from the above sub-chapter about the SHELL section would have this corresponding piece in the --REGEXPECT-- section: ``` @@ -339,6 +331,8 @@ Note the two occurrences of `REGEX()`, for the correlator and the date: where each hyphen must come, however, it's not really necesary. * The second REGEX, for the `Date` HTTP header could also be more elaborated. Also not necessary. +You can include comments in the REGEXPECT section. Any line starting with `##` in the `--REGEXPECT--` section is ignored. + ### TEARDOWN Section This is where processes are killed and databases are removed, so that the following test case will start with a clean slate. The most typical commands used are: diff --git a/test/functionalTest/README.jp.md b/test/functionalTest/README.jp.md index fbb23bf38b..b744839cea 100644 --- a/test/functionalTest/README.jp.md +++ b/test/functionalTest/README.jp.md @@ -28,7 +28,7 @@ virtualenv --python=/usr/bin/python3 /path/to/ft_env ``` pip install Flask==2.0.2 pip install paho-mqtt==1.6.1 -pip install amqtt==0.10.1 # Not actually an accumulator-server.py dependency, but needed by some tests +pip install amqtt==0.11.0b1 # Not actually an accumulator-server.py dependency, but needed by some tests ``` 次に、accumulator-server.py のスクリプト自体をインストールします: diff --git a/test/functionalTest/README.md b/test/functionalTest/README.md index be195dbce6..541e1f02bc 100644 --- a/test/functionalTest/README.md +++ b/test/functionalTest/README.md @@ -26,7 +26,7 @@ Next install accumulator-server.py depencencies: ``` pip install Flask==2.0.2 pip install paho-mqtt==1.6.1 -pip install amqtt==0.10.1 # Not actually an accumulator-server.py dependency, but needed by some tests +pip install amqtt==0.11.0b1 # Not actually an accumulator-server.py dependency, but needed by some tests ``` Next, install the accumulator-server.py script itself: diff --git a/test/functionalTest/cases/README.md b/test/functionalTest/cases/README.md index 104d389a41..924ae74a14 100644 --- a/test/functionalTest/cases/README.md +++ b/test/functionalTest/cases/README.md @@ -1,7 +1 @@ -This directory contains several subdirectories. Each subdirectory starts with a number, which identifies the issue number at GitHub associated with the .test contained in that subdirectory (except the ones starting with `0000`, which are not associated with any GitHub issue). - -Subdirectories can be classified in 3 groups: - -* The ones with "ngsi10" substring in the name, corresponding to NGSI10 functionality. -* The ones with "ngsi9" substring in the name, corresponding to NGSI9 functionality. -* The ones without "ngsi10" or "ngsi9" in the name, corresponding to functionalities not directly related to NGSI9 or NGSI10, because they are orthogonal (i.e. applicable to both) or are not related to NGSI (e.g. CLI tests). +This directory contains several subdirectories. Each subdirectory starts with a number, which identifies the issue number at GitHub associated with the .test contained in that subdirectory (except the ones starting with `0000`, which are not associated with any GitHub issue). \ No newline at end of file From b2f15333d90ba98388ed7ef27a0c1ae83fe8e83d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Sep 2023 10:18:33 +0200 Subject: [PATCH 11/14] FIX CNR --- CHANGES_NEXT_RELEASE | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0fb2c7d025..c3b7ccea03 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -4,4 +4,6 @@ - Fix: improve error traces (#4387) - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) - Fix: improve logs in MongoDB query logic +- Upgrade Debian version from 11.6 to 12.1 in Dockerfile - Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3 +- Reference distribution changed from Debian 11 to Debian 12 From 617f37e406805c7a5ece0194dccf625b6ea998b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Sep 2023 10:25:17 +0200 Subject: [PATCH 12/14] FIX build_source.md --- doc/manuals.jp/admin/build_source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals.jp/admin/build_source.md b/doc/manuals.jp/admin/build_source.md index 5fba262d95..c2636d9b30 100644 --- a/doc/manuals.jp/admin/build_source.md +++ b/doc/manuals.jp/admin/build_source.md @@ -2,7 +2,7 @@ Orion Context Broker のリファレンス配布は Debian 12 です。これは、broker を他のディストリビューションに組み込むことができないことを意味しません (実際には可能です)。このセクションでは、他のディストリビューションをビルドする方法についても説明しています。Debian を使用していない人に役立つかもしれません。ただし、"公式にサポートされている" 唯一の手順は Debian 12 用の手順です。他のものは "現状のまま" 提供され、随時時代遅れになる可能性があります。 -## Debian 11 (正式サポート) +## Debian 12 (正式サポート) Orion Context Broker は、以下のライブラリをビルドの依存関係として使用します : From f33394c4131d6821ec4713dbe852e7e9fea8865f Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Wed, 6 Sep 2023 20:34:56 +0900 Subject: [PATCH 13/14] (JP) ADD doc about Debian 12 (#4417) --- doc/manuals.jp/admin/build_source.md | 15 ++++++++++----- doc/manuals.jp/devel/cookbook.md | 24 +++++++++--------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/doc/manuals.jp/admin/build_source.md b/doc/manuals.jp/admin/build_source.md index c2636d9b30..d494a40200 100644 --- a/doc/manuals.jp/admin/build_source.md +++ b/doc/manuals.jp/admin/build_source.md @@ -98,7 +98,7 @@ Orion Context Broker には、次の手順に従って実行できる一連の tar xfvj gmock-1.5.0.tar.bz2 cd gmock-1.5.0 ./configure - # Adjust /path/to/fiware-orion in the next line accordingly to where you local copy of fiware-orion repo is in your system + # システム内の fiware-orion リポジトリのローカル・コピーの場所に応じて、次の行の /path/to/fiware-orion を調整します。 patch -p1 gtest/scripts/fuse_gtest_files.py < /path/to/fiware-orion/test/unittests/fuse_gtest_files.py.patch make sudo make install # installation puts .h files in /usr/local/include and library in /usr/local/lib @@ -106,7 +106,13 @@ Orion Context Broker には、次の手順に従って実行できる一連の aarch64 アーキテクチャの場合、apt-get を使用して libxslt をインストールし、`--build=arm-linux` オプションを指定して `/configure` を実行します。 -* MongoDB をインストールします (テストはローカルホストで実行されている mongod に依存します)。詳細については、[MongoDB の公式ドキュメント](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-red-hat/)を確認してください。推奨バージョンは 4.4 です (以前のバージョンで動作する可能性がありますが、お勧めしません)。 +* MongoDB をインストールします (テストはローカル・ホストで実行されている mongod に依存します)。詳細については、[MongoDB の公式ドキュメント](hhttps://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-debian/) を確認してください。推奨バージョンは 4.4 です (以前のバージョンでも動作する可能性がありますが、お勧めしません)。 + * mongo レガシー・シェル (`mongo` コマンド) は MongoDB 5 で非推奨となり、MongoDB 6 では新しいシェル (`mongosh` コマンド) が優先されて削除されたことに注意してください。一部の機能テスト (ftest) は、`mongosh` ではなく `mongo` を使用しているため、MongoDB 6 以降を使用している場合、これが原因で失敗します。 + * Debian 12 は libssl3 に移行しましたが、一部の MongoDB バージョンでは libssl1 が必要な場合があります。`Depends: libssl1.1 (>= 1.1.1) but it is not installable` エラーが発生した場合は、次のことをテストできます ([こちら](https://askubuntu.com/a/1421959) を参照)) + + wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb + sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb + rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb # optional, for cleanness * ユニット・テストを実行します @@ -114,8 +120,7 @@ aarch64 アーキテクチャの場合、apt-get を使用して libxslt をイ * 機能テストと valgrind テストに必要な追加のツールをインストールします: - sudo apt-get install curl nc valgrind bc python3 python3-pip - sudo pip3 install virtualenv + sudo apt-get install curl netcat-traditional valgrind bc python3 python3-pip mosquitto * テスト・ハーネスのための環境を準備します。基本的には、`accumulator-server.py` スクリプトをコントロールの下にあるパスにインストールしなければならず、`~/bin` が推奨です。また、`/usr/bin` のようなシステム・ディレクトリにインストールすることもできますが、他のプログラムと衝突する可能性がありますので、お勧めしません。さらに、ハーネス・スクリプト (`scripts/testEnv.sh` ファイル参照) で使用されるいくつかの環境変数を設定し、必要な Python パッケージを使用して virtualenv 環境を作成します。 @@ -139,7 +144,7 @@ aarch64 アーキテクチャの場合、apt-get を使用して libxslt をイ * lcov ツールをインストールします - sudo apt-get install lcov + sudo apt-get install lcov xsltproc * まず、unit_test と functional_test の成功パスを実行して、すべてが正常であることを確認します (上記参照) diff --git a/doc/manuals.jp/devel/cookbook.md b/doc/manuals.jp/devel/cookbook.md index 12433e1008..5da690df72 100644 --- a/doc/manuals.jp/devel/cookbook.md +++ b/doc/manuals.jp/devel/cookbook.md @@ -193,7 +193,7 @@ Orion の機能テストは、`.test` のサフィックスを持つ、テキス 2. NAME セクション 3. SHELL-INIT セクション 4. SHELL セクション -5. EXPECT/REGEXPECT セクション +5. REGEXPECT セクション 6. TEARDOWN セクション 各セクション (ファイルの先頭から始まる著作権プリアンブルを除く) には、すべてのセクションが開始/終了する機能テスト・ハーネスを示すヘッダが必要です : @@ -201,11 +201,9 @@ Orion の機能テストは、`.test` のサフィックスを持つ、テキス * `--NAME--` * `--SHELL-INIT--` * `--SHELL--` -* `--REGEXPECT--` / `--EXPECT--` +* `--REGEXPECT--` * `--TEARDOWN--` -`--REGEXPECT--` が使用されていて、 `--EXPECT--` でなければ、期待されるセクションは正規表現を許可します。これは、これら2つの間で唯一異なるものです。 - ### 著作権のセクション このセクションは単に著作権のヘッダです。古いものをコピーしてください。必要に応じて、年を変更することを忘れないでください。 @@ -278,7 +276,7 @@ echo "0x. description of test step 0x" echo "===============================" ``` -これらのステップは、現在のステップを出力の次のものから分離するために、`echo` を2回呼び出すことで終了します。 これは非常に重要なことです。出力を読むのがずっと**簡単**です。それは、**EXPECT/REGEXPECT** セクションに続くセクションに一致する必要があります。 +これらのステップは、現在のステップを出力の次のものから分離するために、`echo` を2回呼び出すことで終了します。 これは非常に重要なことです。出力を読むのがずっと**簡単**です。それは、**REGEXPECT** セクションに続くセクションに一致する必要があります。 エンティティの作成などの典型的なステップは次のようになります : @@ -303,22 +301,16 @@ echo echo ``` -### EXPECT/REGEXPECT セクション +### REGEXPECT セクション まず、テストハーネス (`test/functionalTest/testHarness.sh`) は、2つのタイプの 'expect sections' を認めます。いずれか : ``` ---EXPECT-- -``` - -または - -``` ---REG-EXPECT-- +--REGEXPECT-- ``` -**1つを選ぶ**必要があります。ほとんど**すべて**の現在の functests が、この `--REG-EXPECT--` タイプを使用します。 --REG-EXPECT-- の利点は、正規表現を `REGEX()` 構文を使用して追加できることです。これは日付の比較や Orion によって作成され、レジストレーション id や 相関器 (correlator)、単純なタイムスタンプのような、レスポンスで返された IDs の比較にとって非常に重要です。重要な制限は、REG-EXPECT セクションには、1行に **REGEX** が1つしかないということです。 +正規表現を `REGEX()` 構文を使用して追加できることです。これは日付の比較や Orion によって作成され、レジストレーション id や 相関器 (correlator)、単純なタイムスタンプのような、レスポンスで返された IDs の比較にとって非常に重要です。重要な制限は、REGEXPECT セクションには、1行に **REGEX** が1つしかないということです。 -つまり、REG-EXPECT セクションでは、問題のテスト・ステップからの予想される出力を追加します。たとえば、SHELL セクションについての上記のサブ・チャプターの例 "01. Create entity E1 with attribute A1" は、 +つまり、問題のテスト・ステップからの予想される出力を追加します。たとえば、SHELL セクションについての上記のサブ・チャプターの例 "01. Create entity E1 with attribute A1" は、 この対応する部分を --REGEXPECT-- セクションに置いてください : ``` @@ -341,6 +333,8 @@ Date: REGEX(.*) * 相関器 (correlator) は36文字の文字列で、ハイフンを含む16進数です。この正規表現は、各ハイフンがどこに来なければならないかを正確に知るようになりましたが、実際には必要ではありません * `Date` HTTP ヘッダの2番目の REGEX もより詳細に記述できます。また必要ありません +REGEXPECT セクションにはコメントを含めることができます。`--REGEXPECT--` セクション内の `##` で始まる行は無視されます。 + ### TEARDOWN セクション ここでプロセスが強制終了され、データベースが削除されるため、次のテスト・ケースがクリーン・スレートで開始されます。最も一般的なコマンドは次のとおりです : From 1f07a161f0f25f9585222e919b88bf1f7eb5c704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Sep 2023 17:33:30 +0200 Subject: [PATCH 14/14] FIX listen only to test topic --- ...ification_mqtt_ngsi_mapping_constant_attribute_compound.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_constant_attribute_compound.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_constant_attribute_compound.test index 8ac16c4dd0..59271fef39 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_constant_attribute_compound.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_constant_attribute_compound.test @@ -26,7 +26,7 @@ Custom notification MQTT with NGSI payload mapping attribute constant compound v --SHELL-INIT-- dbInit CB brokerStart CB -accumulatorStart --pretty-print --mqttHost ${MQTT_HOST} --mqttPort ${MQTT_PORT} --mqttTopic "#" +accumulatorStart --pretty-print --mqttHost ${MQTT_HOST} --mqttPort ${MQTT_PORT} --mqttTopic "topic1" --SHELL--