diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 2a00c05686..179a15a5aa 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -2,6 +2,8 @@ name: Functional Test on: pull_request: + workflow_dispatch: + env: IMAGE_NAME: localhost:5000/orion-ld-test:latest diff --git a/docker/Dockerfile-debug b/docker/Dockerfile-debug index d3f08e213b..a49bc83618 100644 --- a/docker/Dockerfile-debug +++ b/docker/Dockerfile-debug @@ -2,13 +2,16 @@ FROM fiware/orion-ld-base ARG PATH_TO_SRC='opt/orion/' -RUN cp -a /usr/local/lib/. /usr/lib/x86_64-linux-gnu/ -RUN rm -r /usr/local/lib/ +RUN yum -y install hostname && \ + cp -a /usr/local/lib/. /usr/lib/x86_64-linux-gnu/ && \ + rm -r /usr/local/lib/ COPY . ${PATH_TO_SRC} WORKDIR ${PATH_TO_SRC} +ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig + RUN make debug install ENV LD_LIBRARY_PATH=/opt/paho.mqtt.c/build/output:/usr/local/lib64:/opt/prometheus-client-c/prom/build:/opt/prometheus-client-c/promhttp/build:/usr/lib/x86_64-linux-gnu/ diff --git a/docker/Dockerfile-test b/docker/Dockerfile-test index b129430440..6d633518fd 100644 --- a/docker/Dockerfile-test +++ b/docker/Dockerfile-test @@ -5,61 +5,42 @@ RUN ls /usr/local/lib/*.so* RUN echo "======================================================================" ARG PATH_TO_SRC='/opt/orion/' -ENV LD_LIBRARY_PATH=/opt/paho.mqtt.c/build/output:/usr/local/lib64:/opt/prometheus-client-c/prom/build:/opt/prometheus-client-c/promhttp/build:/usr/local/lib -RUN echo LD_LIBRARY_PATH: $LD_LIBRARY_PATH - -RUN wget https://downloads.sourceforge.net/ltp/lcov-1.14.tar.gz -RUN tar -xf lcov-1.14.tar.gz -WORKDIR lcov-1.14 -RUN make install WORKDIR ${PATH_TO_SRC} -RUN yum-config-manager --disable pgdg-common - -## Workaround to refresh the repo key that was baked into the base image -RUN yum --nogpgcheck --disablerepo=* -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm ; exit 0 -RUN mv /etc/yum.repos.d/pgdg-redhat-all.repo.rpmnew /etc/yum.repos.d/pgdg-redhat-all.repo ; exit 0 -RUN yum --nogpgcheck -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - -RUN yum install --nogpgcheck -y gnupg - -RUN yum install --nogpgcheck -y https://repo.mongodb.org/yum/redhat/8Server/mongodb-org/4.4/x86_64/RPMS/mongodb-database-tools-100.4.1.x86_64.rpm -RUN yum install --nogpgcheck -y https://repo.mongodb.org/yum/redhat/8Server/mongodb-org/4.4/x86_64/RPMS/mongodb-org-database-tools-extra-4.4.9-1.el8.x86_64.rpm -RUN yum install --nogpgcheck -y https://repo.mongodb.org/yum/redhat/8Server/mongodb-org/4.4/x86_64/RPMS/mongodb-org-tools-4.4.9-1.el8.x86_64.rpm -RUN yum install --nogpgcheck -y https://repo.mongodb.org/yum/redhat/8Server/mongodb-org/4.4/x86_64/RPMS/mongodb-org-shell-4.4.9-1.el8.x86_64.rpm -RUN yum install --nogpgcheck -y https://repo.mongodb.org/yum/redhat/8Server/mongodb-org/4.4/x86_64/RPMS/mongodb-org-server-4.4.9-1.el8.x86_64.rpm -RUN yum install --nogpgcheck -y https://repo.mongodb.org/yum/redhat/8Server/mongodb-org/4.4/x86_64/RPMS/mongodb-org-mongos-4.4.9-1.el8.x86_64.rpm -RUN yum install --nogpgcheck -y https://repo.mongodb.org/yum/redhat/8Server/mongodb-org/4.4/x86_64/RPMS/mongodb-org-4.4.9-1.el8.x86_64.rpm -RUN yum install --nogpgcheck -y mongodb-org - -RUN yum install --nogpgcheck -y python3 python2 virtualenv -RUN ln --force -s /usr/bin/python2 /usr/bin/python -RUN yum install --nogpgcheck -y openssl-devel libffi-devel - -RUN wget https://bootstrap.pypa.io/pip/2.7/get-pip.py -RUN python2.7 get-pip.py -RUN pip2.7 install Flask -RUN pip2.7 install paho-mqtt -RUN pip2.7 install pyopenssl - -# RUN yum --nogpgcheck -y install https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/gtest-1.8.0-5.el8.x86_64.rpm -# RUN yum --nogpgcheck -y install https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/gmock-1.8.0-5.el8.x86_64.rpm -# RUN yum --nogpgcheck -y install https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/gmock-devel-1.8.0-5.el8.x86_64.rpm -RUN yum --nogpgcheck -y install procps gmock-devel gtest - -RUN wget https://src.fedoraproject.org/repo/pkgs/gmock/gmock-1.5.0.tar.bz2/d738cfee341ad10ce0d7a0cc4209dd5e/gmock-1.5.0.tar.bz2 -RUN tar xfvj gmock-1.5.0.tar.bz2 -WORKDIR gmock-1.5.0 -RUN ./configure -RUN make -RUN make install -WORKDIR ${PATH_TO_SRC} +ENV LD_LIBRARY_PATH=/opt/paho.mqtt.c/build/output:/usr/local/lib64:/opt/prometheus-client-c/prom/build:/opt/prometheus-client-c/promhttp/build:/usr/local/lib +# RUN echo LD_LIBRARY_PATH: $LD_LIBRARY_PATH +COPY docker/mongo.repo /etc/yum.repos.d/mongo.repo + +RUN wget https://downloads.sourceforge.net/ltp/lcov-1.14.tar.gz && \ + tar -xf lcov-1.14.tar.gz && \ + cd lcov-1.14 && \ + make install && \ + cd .. && \ + yum install -y gnupg && \ + yum install -y mongodb-database-tools mongodb-org-database-tools-extra \ + mongodb-org-tools mongodb-org-shell mongodb-org-server \ + mongodb-org-mongos mongodb-org && \ + yum install -y python3 python2 virtualenv && \ + ln --force -s /usr/bin/python2 /usr/bin/python && \ + yum install -y openssl-devel libffi-devel netcat bc diffutils hostname procps gmock-devel gtest && \ + wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ + python2.7 get-pip.py && \ + pip2.7 install Flask && \ + pip2.7 install paho-mqtt && \ + pip2.7 install pyopenssl && \ + wget https://src.fedoraproject.org/repo/pkgs/gmock/gmock-1.5.0.tar.bz2/d738cfee341ad10ce0d7a0cc4209dd5e/gmock-1.5.0.tar.bz2 && \ + tar xfvj gmock-1.5.0.tar.bz2 && \ + cd gmock-1.5.0 && \ + ./configure && \ + make && \ + make install && \ + cd .. && \ + yum clean all && \ + rm lcov-1.14.tar.gz gmock-1.5.0.tar.bz2 -RUN yum install --nogpgcheck -y netcat bc diffutils hostname COPY . ${PATH_TO_SRC} WORKDIR ${PATH_TO_SRC} - ENTRYPOINT ["docker/build.sh"] diff --git a/docker/mongo.repo b/docker/mongo.repo new file mode 100644 index 0000000000..8c9fc19040 --- /dev/null +++ b/docker/mongo.repo @@ -0,0 +1,6 @@ +[mongodb-org-4.4] +name=MongoDB Repository +baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.4/x86_64/ +gpgcheck=1 +enabled=1 +gpgkey=https://www.mongodb.org/static/pgp/server-4.4.asc diff --git a/scripts/check_files_compliance.py b/scripts/check_files_compliance.py index 140fb0475b..3b776ff501 100755 --- a/scripts/check_files_compliance.py +++ b/scripts/check_files_compliance.py @@ -167,7 +167,7 @@ def ignore(root, file): return True # Some files in docker/ directory are not processed - if 'docker' in root and file in ['Dockerfile', 'Dockerfile-base', 'Dockerfile-ubi-base', 'Dockerfile-ubi', 'Dockerfile-test', 'Dockerfile-debug', 'Dockerfile-gdb', 'gdbinit', 'docker-compose.yml', 'subscription-manager.conf', 'ubi.repo', 'other-places.repo']: + if 'docker' in root and file in ['Dockerfile', 'Dockerfile-base', 'Dockerfile-ubi-base', 'Dockerfile-ubi', 'Dockerfile-test', 'Dockerfile-debug', 'Dockerfile-gdb', 'gdbinit', 'docker-compose.yml', 'subscription-manager.conf', 'ubi.repo', 'other-places.repo', 'mongo.repo']: return True # Some files in test/acceptance/behave directory are not processed diff --git a/src/lib/orionld/legacyDriver/legacyPostBatchUpdate.cpp b/src/lib/orionld/legacyDriver/legacyPostBatchUpdate.cpp index eda8afad89..8b83eeaf42 100644 --- a/src/lib/orionld/legacyDriver/legacyPostBatchUpdate.cpp +++ b/src/lib/orionld/legacyDriver/legacyPostBatchUpdate.cpp @@ -56,6 +56,7 @@ extern "C" #include "orionld/common/typeCheckForNonExistingEntities.h" // typeCheckForNonExistingEntities #include "orionld/common/duplicatedInstances.h" // duplicatedInstances #include "orionld/common/performance.h" // PERFORMANCE +#include "orionld/notifications/previousValues.h" // previousValues #include "orionld/service/orionldServiceInit.h" // orionldHostName, orionldHostNameLen #include "orionld/context/orionldCoreContext.h" // orionldDefaultUrl, orionldCoreContext #include "orionld/context/orionldContextPresent.h" // orionldContextPresent @@ -188,6 +189,8 @@ bool legacyPostBatchUpdate(void) // Not existing entities cannot be updated KjNode* dbEntityP = entityLookupById(idTypeAndCreDateFromDb, entityId); + KjNode* dbAttrsP = (dbEntityP != NULL)? kjLookup(dbEntityP, "attrs") : NULL; + if (dbEntityP == NULL) { entityErrorPush(errorsArrayP, entityId, OrionldBadRequestData, "entity does not exist", NULL, 400); @@ -207,6 +210,8 @@ bool legacyPostBatchUpdate(void) kjChildRemove(incomingTree, entityP); } + previousValues(entityP, dbAttrsP); + char* newType; if (entityTypeChange(entityP, dbEntityP, &newType) == true) { diff --git a/src/lib/orionld/serviceRoutines/orionldPostBatchUpsert.cpp b/src/lib/orionld/serviceRoutines/orionldPostBatchUpsert.cpp index 55bd1fb8fc..047085a2d9 100644 --- a/src/lib/orionld/serviceRoutines/orionldPostBatchUpsert.cpp +++ b/src/lib/orionld/serviceRoutines/orionldPostBatchUpsert.cpp @@ -60,6 +60,7 @@ extern "C" #include "orionld/mongoc/mongocEntitiesUpsert.h" // mongocEntitiesUpsert #include "orionld/legacyDriver/legacyPostBatchUpsert.h" // legacyPostBatchUpsert #include "orionld/notifications/alteration.h" // alteration +#include "orionld/notifications/previousValues.h" // previousValues #include "orionld/serviceRoutines/orionldPostBatchUpsert.h" // Own interface @@ -274,7 +275,12 @@ bool orionldPostBatchUpsert(void) finalDbEntityP = batchReplaceEntity(inEntityP, entityId, entityType, entityCreDate); } else + { + KjNode* dbAttrsP = kjLookup(originalDbEntityP, "attrs"); + + previousValues(inEntityP, dbAttrsP); finalDbEntityP = batchUpdateEntity(inEntityP, originalDbEntityP, false); + } if (finalDbEntityP != NULL) { diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_showChanges_issue_1393.test b/test/functionalTest/cases/0000_ngsild/ngsild_showChanges_issue_1393.test new file mode 100644 index 0000000000..4396728726 --- /dev/null +++ b/test/functionalTest/cases/0000_ngsild/ngsild_showChanges_issue_1393.test @@ -0,0 +1,207 @@ +# Copyright 2023 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +showChanges in Subscription - provoke a notification and see the previousValue + +--SHELL-INIT-- +dbInit CB +orionldStart CB -experimental -triggerOperation +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create E1 with an attribute A1 == 1 +# 02. Create a matching subscription S1, with showChanges set to true +# 03. See S1 in the database - make sure showChanges is there +# 04. Update urn:E1/attrs/A1, using upsert update, setting its value to 4 +# 05. Dump/Reset accumulator, see A1's value and previousValue +# + +echo "01. Create E1 with an attribute A1 == 1" +echo "=======================================" +payload='{ + "id": "urn:E1", + "type": "Device", + "A1": 1 +}' +orionCurl --url /ngsi-ld/v1/entities --payload "$payload" +echo +echo + + +echo "02. Create a matching subscription S1, with showChanges set to true" +echo "===================================================================" +payload='{ + "id": "urn:ngsi-ld:subs:S1", + "type": "Subscription", + "entities": [ + { + "id": "urn:E1", + "type": "Device" + } + ], + "notification": { + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + }, + "showChanges": true + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "03. See S1 in the database - make sure showChanges is there" +echo "===========================================================" +mongoCmd2 ftest "db.csubs.findOne()" +echo +echo + + +echo "04. Update urn:E1/attrs/A1, using upsert update, setting its value to 4" +echo "=======================================================================" +payload='[{ + "id": "urn:E1", + "A1":{ + "value": 4 + } +}]' +orionCurl --url /ngsi-ld/v1/entityOperations/upsert/?options=update --payload "$payload" -X POST +echo +echo + + +echo "05. Dump/Reset accumulator, see A1's value and previousValue" +echo "============================================================" +accumulatorDump +accumulatorReset +echo +echo + + + + +--REGEXPECT-- +01. Create E1 with an attribute A1 == 1 +======================================= +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/entities/urn:E1 + + + +02. Create a matching subscription S1, with showChanges set to true +=================================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:ngsi-ld:subs:S1 + + + +03. See S1 in the database - make sure showChanges is there +=========================================================== +MongoDB shell version REGEX(.*) +connecting to: REGEX(.*) +MongoDB server version: REGEX(.*) +{ + "_id" : "urn:ngsi-ld:subs:S1", + "entities" : [ + { + "id" : "urn:E1", + "type" : "https://uri.etsi.org/ngsi-ld/default-context/Device", + "isPattern" : "false", + "isTypePattern" : false + } + ], + "createdAt" : REGEX(.*), + "modifiedAt" : REGEX(.*), + "throttling" : 0, + "expression" : { + "geometry" : "", + "coords" : "", + "georel" : "", + "geoproperty" : "", + "q" : "", + "mq" : "" + }, + "reference" : "http://REGEX(.*)/notify", + "mimeType" : "application/json", + "showChanges" : true, + "attrs" : [ ], + "format" : "normalized", + "conditions" : [ ], + "status" : "active", + "custom" : false, + "servicePath" : "/#", + "blacklist" : false, + "ldContext" : "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld" +} +bye + + +04. Update urn:E1/attrs/A1, using upsert update, setting its value to 4 +======================================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) + + + +05. Dump/Reset accumulator, see A1's value and previousValue +============================================================ +POST http://REGEX(.*)/notify?subscriptionId=urn:ngsi-ld:subs:S1 +Content-Length: 318 +User-Agent: orionld/REGEX(.*) +Host: REGEX(.*) +Accept: application/json +Content-Type: application/json +Link: