From 2cfed29565359411fd07a333cd5f25f7dd15b839 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sun, 2 Jul 2023 09:47:51 +0200 Subject: [PATCH 01/29] Just to keep working on this in my other laptop, in Athens - doesn't even compile --- CMakeLists.txt | 2 + src/app/orionld/orionld.cpp | 4 + src/lib/orionld/pernot/CMakeLists.txt | 34 +++++ src/lib/orionld/pernot/PernotSubscription.h | 72 +++++++++ src/lib/orionld/pernot/pernotLoop.cpp | 153 ++++++++++++++++++++ src/lib/orionld/pernot/pernotLoop.h | 37 +++++ 6 files changed, 302 insertions(+) create mode 100644 src/lib/orionld/pernot/CMakeLists.txt create mode 100644 src/lib/orionld/pernot/PernotSubscription.h create mode 100644 src/lib/orionld/pernot/pernotLoop.cpp create mode 100644 src/lib/orionld/pernot/pernotLoop.h diff --git a/CMakeLists.txt b/CMakeLists.txt index fa986db3af..217c8c80f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,6 +208,7 @@ SET (ORION_LIBS orionld_troe orionld_dbModel orionld_apiModel + orionld_pernot serviceRoutines serviceRoutinesV2 ngsiNotify @@ -366,6 +367,7 @@ if (error EQUAL 0) ADD_SUBDIRECTORY(src/lib/jsonParse) ADD_SUBDIRECTORY(src/lib/jsonParseV2) ADD_SUBDIRECTORY(src/lib/rest) + ADD_SUBDIRECTORY(src/lib/orionld/pernot) ADD_SUBDIRECTORY(src/lib/orionld/socketService) ADD_SUBDIRECTORY(src/lib/orionld/notifications) ADD_SUBDIRECTORY(src/lib/orionld/forwarding) diff --git a/src/app/orionld/orionld.cpp b/src/app/orionld/orionld.cpp index 3b18a4be03..f881de1619 100644 --- a/src/app/orionld/orionld.cpp +++ b/src/app/orionld/orionld.cpp @@ -129,6 +129,7 @@ extern "C" #include "orionld/regCache/regCacheInit.h" // regCacheInit #include "orionld/regCache/regCacheCreate.h" // regCacheCreate #include "orionld/regCache/regCacheRelease.h" // regCacheRelease +#include "orionld/pernot/pernotLoop.h" // pernotLoopStart #include "orionld/version.h" #include "orionld/orionRestServices.h" @@ -1315,6 +1316,9 @@ int main(int argC, char* argV[]) kaBufferReset(&orionldState.kalloc, KFALSE); + // Start the thread for periodic notifications + pernotLoopStart(); + if (socketService == true) { int fd; diff --git a/src/lib/orionld/pernot/CMakeLists.txt b/src/lib/orionld/pernot/CMakeLists.txt new file mode 100644 index 0000000000..6d92b4a5b2 --- /dev/null +++ b/src/lib/orionld/pernot/CMakeLists.txt @@ -0,0 +1,34 @@ +# Copyright 2023 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +CMAKE_MINIMUM_REQUIRED(VERSION 3.5) + +SET (SOURCES + pernotLoop.cpp +) + +# Include directories +# ----------------------------------------------------------------- +include_directories("${PROJECT_SOURCE_DIR}/src/lib") + + +# Library declaration +# ----------------------------------------------------------------- +ADD_LIBRARY(orionld_pernot STATIC ${SOURCES}) diff --git a/src/lib/orionld/pernot/PernotSubscription.h b/src/lib/orionld/pernot/PernotSubscription.h new file mode 100644 index 0000000000..4db3c7635d --- /dev/null +++ b/src/lib/orionld/pernot/PernotSubscription.h @@ -0,0 +1,72 @@ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTSUBSCRIPTION_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTSUBSCRIPTION_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ + + + +// ----------------------------------------------------------------------------- +// +// PernotState - +// +typedef enum PernotState +{ + SubActive = 1, + SubPaused = 2, + SubErroneous = 3, + SubExpired = 4 +} PernotState; + + + +// ----------------------------------------------------------------------------- +// +// PernotSubscription - Periodic Notification Subscription +// +typedef struct PernotSubscription +{ + char* subscriptionId; + PernotState state; + char tenant[64]; + KjNode* kjSubP; // OR, I split the entire tree into binary fields inside PernotSubscription ... + + // Timestamps + double lastNotificationAttempt; // In seconds + double lastSuccessTime; + double lastFailureTime; + double expiresAt; + + // Error handling + uint32_t consecutiveErrors; + uint32_t timeInterval; // In seconds (Subscription::timeInterval is an integer in seconds) + uint32_t cooldown; + + CURL* curlHandle; + + struct PernotSubscription* next; +} PernotSubscription; + +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSUBSCRIPTION_H_ diff --git a/src/lib/orionld/pernot/pernotLoop.cpp b/src/lib/orionld/pernot/pernotLoop.cpp new file mode 100644 index 0000000000..ff0eb8e019 --- /dev/null +++ b/src/lib/orionld/pernot/pernotLoop.cpp @@ -0,0 +1,153 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include // strerror +#include // gettimeofday +#include // pthread_create + +#include "logMsg/logMsg.h" // LM_x + +#include "orionld/common/orionldState.h" // orionldState +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/pernotLoop.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// PernotSubCache - move to pernot/PernotSubCache.h +// +typedef struct PernotSubCache +{ + PernotSubscription* head; + PernotSubscription* tail; +} PernotSubCache; + + +PernotSubCache pernotSubCache; + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheInit - move to pernot/pernotSubCacheInit.h/cpp +// +void pernotSubCacheInit(void) +{ + bzero(&pernotSubCache, sizeof(pernotSubCache)); +} + + + +// ----------------------------------------------------------------------------- +// +// pernotSubInsert - move to pernot/pernotSubInsert.h/cpp +// +bool pernotSubInsert(void) +{ + return true; +} + + + +// ----------------------------------------------------------------------------- +// +// currentTime - +// +static double currentTime(void) +{ + // int gettimeofday(struct timeval *tv, struct timezone *tz); + struct timeval tv; + + if (gettimeofday(&tv, NULL) != 0) + LM_RE(0, ("gettimeofday error: %s", strerror(errno))); + + return tv.tv_sec + tv.tv_usec / 1000000; +} + + + +// ----------------------------------------------------------------------------- +// +// pernotLoop - +// +static void* pernotLoop(void* vP) +{ + while (1) + { + for (PernotSubscription* subP = pernotSubCache.head; subP != NULL; subP = subP->next) + { + double now = currentTime(); + + if (subP->state == SubPaused) + continue; + + if (subP->expiresAt <= now) + subP->state = SubExpired; + + if (subP->state == SubExpired) + continue; + + if (subP->state == SubErroneous) + { + // Check for cooldown - take it out of error if cooldown time has passwd + if (subP->lastFailureTime + subP->cooldown < now) + subP->state = SubActive; + else + continue; + } + + if (subP->lastNotificationAttempt + subP->timeInterval < now) + { + subP->lastNotificationAttempt = now; // Either it works or fails, the timestamp needs to be updated + if (pernotTreat(subP) == true) // Just send the notification, don't await any response + { + subP->lastSuccessTime = now; + subP->consecutiveErrors = 0; + } + else + { + subP->lastFailureTime = now; + subP->consecutiveErrors += 1; + + if (subP->consecutiveErrors >= 3) + { + subP->state = SubErroneous; + } + } + } + } + } +} + + + +// ----------------------------------------------------------------------------- +// +// pernotLoopStart - +// +void pernotLoopStart(void) +{ + pthread_create(&pernotThreadID, NULL, pernotLoop, NULL); +} diff --git a/src/lib/orionld/pernot/pernotLoop.h b/src/lib/orionld/pernot/pernotLoop.h new file mode 100644 index 0000000000..6936c2858a --- /dev/null +++ b/src/lib/orionld/pernot/pernotLoop.h @@ -0,0 +1,37 @@ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTSTART_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTSTART_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ + + + +// ----------------------------------------------------------------------------- +// +// pernotStart - +// +extern void pernotStart(void); + +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSTART_H_ From 556b20005b829f66b546766451e958a101abc70f Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sun, 9 Jul 2023 17:48:32 +0200 Subject: [PATCH 02/29] Small step further - commit just to synchronize between computers --- .../orionld/kjTree/kjTreeToSubscription.cpp | 10 ++- src/lib/orionld/pernot/CMakeLists.txt | 1 + src/lib/orionld/pernot/PernotSubCache.h | 42 +++++++++++ src/lib/orionld/pernot/PernotSubscription.h | 8 ++ src/lib/orionld/pernot/pernotLoop.cpp | 14 ++-- src/lib/orionld/pernot/pernotLoop.h | 10 +-- src/lib/orionld/pernot/pernotTreat.cpp | 37 ++++++++++ src/lib/orionld/pernot/pernotTreat.h | 38 ++++++++++ .../cases/0000_ngsild/ngsild_pernot.test | 74 +++++++++++++++++++ 9 files changed, 218 insertions(+), 16 deletions(-) create mode 100644 src/lib/orionld/pernot/PernotSubCache.h create mode 100644 src/lib/orionld/pernot/pernotTreat.cpp create mode 100644 src/lib/orionld/pernot/pernotTreat.h create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_pernot.test diff --git a/src/lib/orionld/kjTree/kjTreeToSubscription.cpp b/src/lib/orionld/kjTree/kjTreeToSubscription.cpp index 1d539f5cbd..1af156d43b 100644 --- a/src/lib/orionld/kjTree/kjTreeToSubscription.cpp +++ b/src/lib/orionld/kjTree/kjTreeToSubscription.cpp @@ -256,8 +256,14 @@ bool kjTreeToSubscription(ngsiv2::Subscription* subP, char** subIdPP, KjNode** e } else if (strcmp(kNodeP->name, "timeInterval") == 0) { - orionldError(OrionldBadRequestData, "Not Implemented", "Subscription::timeInterval is not implemented", 501); - return false; + DUPLICATE_CHECK(timeIntervalP, "Subscription::timeInterval", kNodeP); + INTEGER_CHECK(timeIntervalP, "Subscription::timeInterval"); + subP->timeInterval = timeIntervalP->value.i; + if (subP->timeInterval <= 0) + { + orionldError(OrionldBadRequestData, "Invalid value for Subscription::timeInterval", "must be an integer value > 0", 400); + return false; + } } else if ((kNodeP->name[0] == 'q') && (kNodeP->name[1] == 0)) { diff --git a/src/lib/orionld/pernot/CMakeLists.txt b/src/lib/orionld/pernot/CMakeLists.txt index 6d92b4a5b2..304a5e8347 100644 --- a/src/lib/orionld/pernot/CMakeLists.txt +++ b/src/lib/orionld/pernot/CMakeLists.txt @@ -22,6 +22,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) SET (SOURCES pernotLoop.cpp + pernotTreat.cpp ) # Include directories diff --git a/src/lib/orionld/pernot/PernotSubCache.h b/src/lib/orionld/pernot/PernotSubCache.h new file mode 100644 index 0000000000..b4a6fd9e2f --- /dev/null +++ b/src/lib/orionld/pernot/PernotSubCache.h @@ -0,0 +1,42 @@ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHE_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHE_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription + + + +// ----------------------------------------------------------------------------- +// +// PernotSubCache - +// +typedef struct PernotSubCache +{ + PernotSubscription* head; + PernotSubscription* tail; +} PernotSubCache; + +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHE_H_ diff --git a/src/lib/orionld/pernot/PernotSubscription.h b/src/lib/orionld/pernot/PernotSubscription.h index 4db3c7635d..b1c91e6347 100644 --- a/src/lib/orionld/pernot/PernotSubscription.h +++ b/src/lib/orionld/pernot/PernotSubscription.h @@ -25,6 +25,14 @@ * * Author: Ken Zangelin */ +#include // types: uint64_t, ... +#include // CURL + +extern "C" +{ +#include "kjson/KjNode.h" // KjNode +} + diff --git a/src/lib/orionld/pernot/pernotLoop.cpp b/src/lib/orionld/pernot/pernotLoop.cpp index ff0eb8e019..7e5506db40 100644 --- a/src/lib/orionld/pernot/pernotLoop.cpp +++ b/src/lib/orionld/pernot/pernotLoop.cpp @@ -30,21 +30,16 @@ #include "orionld/common/orionldState.h" // orionldState #include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/PernotSubCache.h" // PernotSubCache +#include "orionld/pernot/pernotTreat.h" // pernotTreat #include "orionld/pernot/pernotLoop.h" // Own interface // ----------------------------------------------------------------------------- // -// PernotSubCache - move to pernot/PernotSubCache.h +// pernotSubCache - // -typedef struct PernotSubCache -{ - PernotSubscription* head; - PernotSubscription* tail; -} PernotSubCache; - - PernotSubCache pernotSubCache; @@ -104,7 +99,7 @@ static void* pernotLoop(void* vP) continue; if (subP->expiresAt <= now) - subP->state = SubExpired; + subP->state = SubExpired; // Should it be removed? if (subP->state == SubExpired) continue; @@ -143,6 +138,7 @@ static void* pernotLoop(void* vP) +pthread_t pernotThreadID; // ----------------------------------------------------------------------------- // // pernotLoopStart - diff --git a/src/lib/orionld/pernot/pernotLoop.h b/src/lib/orionld/pernot/pernotLoop.h index 6936c2858a..9587f07583 100644 --- a/src/lib/orionld/pernot/pernotLoop.h +++ b/src/lib/orionld/pernot/pernotLoop.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTSTART_H_ -#define SRC_LIB_ORIONLD_PERNOT_PERNOTSTART_H_ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTLOOP_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTLOOP_H_ /* * @@ -30,8 +30,8 @@ // ----------------------------------------------------------------------------- // -// pernotStart - +// pernotLoopStart - // -extern void pernotStart(void); +extern void pernotLoopStart(void); -#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSTART_H_ +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTLOOP_H_ diff --git a/src/lib/orionld/pernot/pernotTreat.cpp b/src/lib/orionld/pernot/pernotTreat.cpp new file mode 100644 index 0000000000..a126923dab --- /dev/null +++ b/src/lib/orionld/pernot/pernotTreat.cpp @@ -0,0 +1,37 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/pernotTreat.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// pernotTreat - +// +bool pernotTreat(PernotSubscription* subP) +{ + return false; +} diff --git a/src/lib/orionld/pernot/pernotTreat.h b/src/lib/orionld/pernot/pernotTreat.h new file mode 100644 index 0000000000..dc43fbe4f1 --- /dev/null +++ b/src/lib/orionld/pernot/pernotTreat.h @@ -0,0 +1,38 @@ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTTREAT_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTTREAT_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription + + + +// ----------------------------------------------------------------------------- +// +// pernotTreat - +// +extern bool pernotTreat(PernotSubscription* subP); + +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTTREAT_H_ diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test new file mode 100644 index 0000000000..1ab6bff6b2 --- /dev/null +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test @@ -0,0 +1,74 @@ +# Copyright 2023 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Pernot Subscription - periodic notifications + +--SHELL-INIT-- +dbInit CB +orionldStart CB + +--SHELL-- + +# +# 01. Create a subscription urn:S1 with timeInterval +# 02. See urn:S1 in the database +# 03. GET urn:S1 from cache +# 04. GET urn:S1 from database +# 05. Restart broker +# 06. GET urn:S1 from cache +# 07. GET urn:S1 from database +# + +echo "01. Create a subscription urn:S1 with timeInterval" +echo "==================================================" +payload='{ + "id": "urn:S1", + "type": "Subscription", + "entities": [ + { + "type": "T" + } + ], + "timeInterval": 2, + "notification": { + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "02. See urn:S1 in the database" +echo "==============================" +mongoCmd2 ftest "db.csubs.findOne()" +echo +echo + + +--REGEXPECT-- +--TEARDOWN-- +brokerStop CB +dbDrop CB From e830882f83abef5b8da20bb347a9665d52963d70 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sat, 22 Jul 2023 16:45:25 +0200 Subject: [PATCH 03/29] one more step --- src/app/orionld/orionld.cpp | 4 +- src/lib/logMsg/traceLevels.h | 5 + src/lib/orionld/common/orionldState.cpp | 7 +- src/lib/orionld/common/orionldState.h | 2 + .../common/subCacheApiSubscriptionInsert.cpp | 36 +- .../dbModel/dbModelToApiSubscription.cpp | 72 ++-- .../dbModel/dbModelToApiSubscription.h | 3 +- src/lib/orionld/kjTree/CMakeLists.txt | 1 + .../kjTree/kjTreeFromCachedSubscription.cpp | 21 +- .../kjTree/kjTreeFromPernotSubscription.cpp | 223 +++++++++++ .../kjTree/kjTreeFromPernotSubscription.h | 43 +++ .../orionld/kjTree/kjTreeToSubscription.cpp | 9 +- .../mongoCppLegacyEntitiesDelete.cpp | 2 +- .../mongoCppLegacyEntityFieldDelete.cpp | 2 +- .../mongoCppLegacyEntityFieldReplace.cpp | 2 +- ...egacyEntityListLookupWithIdTypeCreDate.cpp | 2 +- .../mongoCppLegacyEntityUpdate.cpp | 2 +- ...SubscriptionMatchEntityIdAndAttributes.cpp | 2 +- .../mongoc/mongocSubCachePopulateByTenant.cpp | 10 +- src/lib/orionld/payloadCheck/PCHECK.h | 47 +++ src/lib/orionld/payloadCheck/fieldPaths.cpp | 1 + src/lib/orionld/payloadCheck/fieldPaths.h | 1 + .../payloadCheck/pCheckSubscription.cpp | 66 ++-- .../orionld/payloadCheck/pCheckSubscription.h | 1 + src/lib/orionld/pernot/CMakeLists.txt | 4 + src/lib/orionld/pernot/PernotSubCache.cpp | 62 ++++ src/lib/orionld/pernot/PernotSubCache.h | 2 +- src/lib/orionld/pernot/PernotSubscription.h | 10 +- src/lib/orionld/pernot/pernotLoop.cpp | 21 +- src/lib/orionld/pernot/pernotSubCacheAdd.cpp | 183 +++++++++ src/lib/orionld/pernot/pernotSubCacheAdd.h | 63 ++++ src/lib/orionld/pernot/pernotSubCacheInit.cpp | 38 ++ src/lib/orionld/pernot/pernotSubCacheInit.h | 37 ++ .../orionld/pernot/pernotSubCacheLookup.cpp | 49 +++ src/lib/orionld/pernot/pernotSubCacheLookup.h | 42 +++ .../orionld/pernot/pernotSubCacheRemove.cpp | 37 ++ src/lib/orionld/pernot/pernotSubCacheRemove.h | 45 +++ src/lib/orionld/q/qAliasCompact.cpp | 12 + .../orionldGetSubscription.cpp | 84 ++++- .../orionldGetSubscriptions.cpp | 13 +- .../orionldPatchSubscription.cpp | 35 +- .../orionldPostSubscriptions.cpp | 104 ++++-- ...ngsild_langprop-and-notifications-new.test | 5 +- ...angprop-and-notifications-with-arrays.test | 3 +- .../ngsild_langprop-and-notifications.test | 7 +- ...-error-for-optional-field-not-present.test | 6 +- .../ngsild_new_first_notification.test | 1 - ...ormat-in-notifications-with-keyValues.test | 3 +- ...d_new_geojson-format-in-notifications.test | 3 +- ...sild_new_issue_1248_q_in_subscription.test | 10 +- ...ld_new_notification-in-geojson-format.test | 3 +- ...ification_with-q-and-subcache-refresh.test | 6 +- ...ew_notification_with-q-commalist-attr.test | 3 +- .../ngsild_new_notification_with_concise.test | 1 - .../ngsild_new_notification_with_https.test | 1 - ...gsild_new_notification_with_keyValues.test | 1 - ...sild_new_notification_with_simplified.test | 1 - .../ngsild_new_notifications_issue-1244.test | 2 - .../0000_ngsild/ngsild_new_subCache.test | 3 +- ...ld_new_subscription-cache-propagation.test | 14 +- ...ew_subscription-create-error-handling.test | 11 +- .../ngsild_new_subscription-get.test | 10 +- .../ngsild_new_subscription-list.test | 102 +++++- ...w_subscription-with-mqtt-notification.test | 9 +- ...ild_new_subscription_and_q_issue_1180.test | 6 +- ...ption_counters-for-http-notifications.test | 15 +- ...tion_counters-for-https-notifications.test | 15 +- ...ption_counters-for-mqtt-notifications.test | 15 +- .../ngsild_new_subscription_create.test | 28 +- .../ngsild_new_subscription_delete.test | 5 +- .../ngsild_new_subscription_patch_geoq.test | 15 +- ...ild_new_subscription_patch_issue_1374.test | 21 +- .../ngsild_new_subscription_patch_q.test | 15 +- ...ubscription_patch_two_subs-issue_1368.test | 4 +- ...on_patch_update_all_individual_fields.test | 80 ++-- .../ngsild_new_subscription_timestamps.test | 18 +- ..._subscription_with_mqtt_issue-1178-II.test | 3 +- .../ngsild_notification-stats.test | 12 +- .../ngsild_notification_issue-1322.test | 3 +- ...d_patch_entity2-notification-counters.test | 6 +- .../cases/0000_ngsild/ngsild_pernot.test | 346 +++++++++++++++++- .../cases/0000_ngsild/ngsild_showChanges.test | 24 +- .../ngsild_subCache-counters-II.test | 9 +- ...gsild_subCache-with-mongoc-on-startup.test | 30 +- .../ngsild_subscription_cache_issue-1316.test | 6 +- .../ngsild_subscription_create.test | 8 +- .../ngsild_subscription_creation_errors.test | 8 +- ...cription_issue_1179-with-experimental.test | 5 +- ...on_patch_update_all_individual_fields.test | 10 +- .../ngsild_subscription_with_sysAttrs.test | 6 +- 90 files changed, 1954 insertions(+), 369 deletions(-) create mode 100644 src/lib/orionld/kjTree/kjTreeFromPernotSubscription.cpp create mode 100644 src/lib/orionld/kjTree/kjTreeFromPernotSubscription.h create mode 100644 src/lib/orionld/pernot/PernotSubCache.cpp create mode 100644 src/lib/orionld/pernot/pernotSubCacheAdd.cpp create mode 100644 src/lib/orionld/pernot/pernotSubCacheAdd.h create mode 100644 src/lib/orionld/pernot/pernotSubCacheInit.cpp create mode 100644 src/lib/orionld/pernot/pernotSubCacheInit.h create mode 100644 src/lib/orionld/pernot/pernotSubCacheLookup.cpp create mode 100644 src/lib/orionld/pernot/pernotSubCacheLookup.h create mode 100644 src/lib/orionld/pernot/pernotSubCacheRemove.cpp create mode 100644 src/lib/orionld/pernot/pernotSubCacheRemove.h diff --git a/src/app/orionld/orionld.cpp b/src/app/orionld/orionld.cpp index f881de1619..519a6496c8 100644 --- a/src/app/orionld/orionld.cpp +++ b/src/app/orionld/orionld.cpp @@ -129,6 +129,7 @@ extern "C" #include "orionld/regCache/regCacheInit.h" // regCacheInit #include "orionld/regCache/regCacheCreate.h" // regCacheCreate #include "orionld/regCache/regCacheRelease.h" // regCacheRelease +#include "orionld/pernot/pernotSubCacheInit.h" // pernotSubCacheInit #include "orionld/pernot/pernotLoop.h" // pernotLoopStart #include "orionld/version.h" @@ -1165,6 +1166,7 @@ int main(int argC, char* argV[]) // Note that regCacheInit uses the tenantList, so orionldTenantInit must be called before regCacheInit // regCacheInit(); + pernotSubCacheInit(); orionldServiceInit(restServiceVV, 9); @@ -1174,7 +1176,7 @@ int main(int argC, char* argV[]) mongoInit(dbHost, rplSet, dbName, dbUser, dbPwd, multitenancy, dbTimeout, writeConcern, dbPoolSize, statSemWait); } - // Initialize orion libs + // Initialize libs alarmMgr.init(relogAlarms); metricsMgr.init(!disableMetrics, statSemWait); logSummaryInit(&lsPeriod); diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index 094a9123c8..6898d988e1 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -90,6 +90,11 @@ typedef enum TraceLevels // GeoJSON LmtGeoJSON = 90, // GeoJSON ... everything (for now) + // + // Pernot sub-cache + // + LmtPernot = 100, // Periodic Notification Subscription cache + // // Misc // diff --git a/src/lib/orionld/common/orionldState.cpp b/src/lib/orionld/common/orionldState.cpp index a9adf48d4e..d32f274d50 100644 --- a/src/lib/orionld/common/orionldState.cpp +++ b/src/lib/orionld/common/orionldState.cpp @@ -44,6 +44,7 @@ extern "C" #include "orionld/common/numberToDate.h" // numberToDate #include "orionld/q/QNode.h" // QNode #include "orionld/common/performance.h" // REQUEST_PERFORMANCE +#include "orionld/pernot/PernotSubCache.h" // PernotSubCache #include "orionld/common/orionldState.h" // Own interface @@ -106,13 +107,13 @@ char userAgentHeaderNoLF[64]; // "User-Agent: orionld/" + ORION char hostHeaderNoLF[128]; char hostHeader[128]; // Host: xxx size_t hostHeaderLen; +PernotSubCache pernotSubCache; + + // // Variables for Mongo C Driver // -mongoc_collection_t* mongoEntitiesCollectionP = NULL; // Deprecated -mongoc_collection_t* mongoRegistrationsCollectionP = NULL; // Deprecated - mongoc_uri_t* mongocUri; mongoc_client_pool_t* mongocPool; sem_t mongocContextsSem; diff --git a/src/lib/orionld/common/orionldState.h b/src/lib/orionld/common/orionldState.h index 06d71102cf..379edbb432 100644 --- a/src/lib/orionld/common/orionldState.h +++ b/src/lib/orionld/common/orionldState.h @@ -61,6 +61,7 @@ extern "C" #include "orionld/types/OrionldAlteration.h" // OrionldAlteration #include "orionld/types/StringArray.h" // StringArray #include "orionld/troe/troe.h" // TroeMode +#include "orionld/pernot/PernotSubCache.h" // PernotSubCache #include "orionld/context/OrionldContext.h" // OrionldContext @@ -593,6 +594,7 @@ extern size_t hostHeaderLen; // move to orionld.cpp? extern bool debugCurl; // From orionld.cpp extern bool noCache; // From orionld.cpp extern int cSubCounters; // Number of subscription counter updates before flush from sub-cache to DB +extern PernotSubCache pernotSubCache; extern char localIpAndPort[135]; // Local address for X-Forwarded-For (from orionld.cpp) extern unsigned long long inReqPayloadMaxSize; extern unsigned long long outReqMsgMaxSize; diff --git a/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp b/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp index a0462e33a2..8037c69da8 100644 --- a/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp +++ b/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp @@ -227,20 +227,36 @@ CachedSubscription* subCacheApiSubscriptionInsert { for (KjNode* eSelectorP = entitiesP->value.firstChildP; eSelectorP != NULL; eSelectorP = eSelectorP->next) { - KjNode* idP = kjLookup(eSelectorP, "id"); - KjNode* idPatternP = kjLookup(eSelectorP, "idPattern"); - KjNode* typeP = kjLookup(eSelectorP, "type"); - char* id = (idP != NULL)? idP->value.s : (char*) ""; - const char* idPattern = (idPatternP != NULL)? idPatternP->value.s : ""; - const char* type = (typeP != NULL)? typeP->value.s : ""; - const char* isPattern = "false"; - - if (idP == NULL) + KjNode* idP = kjLookup(eSelectorP, "id"); + KjNode* idPatternP = kjLookup(eSelectorP, "idPattern"); + KjNode* typeP = kjLookup(eSelectorP, "type"); + char* id = (idP != NULL)? idP->value.s : NULL; + char* idPattern = (idPatternP != NULL)? idPatternP->value.s : NULL; + char* type = (typeP != NULL)? typeP->value.s : NULL; + char* isPattern = (char*) "false"; + + LM_T(LmtSR, ("id: '%s'", id)); + LM_T(LmtSR, ("idPattern: '%s'", idPattern)); + LM_T(LmtSR, ("type: '%s'", type)); + + if ((idP == NULL) && (idPattern == NULL)) { - id = (char*) ((idPatternP != NULL)? idPattern : ".*"); + id = (char*) ".*"; isPattern = (char*) "true"; } + else + { + if ((idP != NULL) && (idPatternP != NULL)) + idPatternP = NULL; + + if (idPattern != NULL) + { + id = idPatternP->value.s; + isPattern = (char*) "true"; + } + } + LM_T(LmtSR, ("Creating a new EntityInfo (id: '%s', idPattern: '%s', type: '%s')", id, idPattern, type)); EntityInfo* eP = new EntityInfo(id, type, isPattern, false); cSubP->entityIdInfos.push_back(eP); } diff --git a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp index c43c56066d..acb3112316 100644 --- a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp +++ b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp @@ -134,7 +134,7 @@ static bool notificationStatus(KjNode* dbLastSuccessP, KjNode* dbLastFailureP) // "custom": false, // "servicePath": "/#", // "blacklist": false, -// "ldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testFullContext.jsonld" +// "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testFullContext.jsonld" // } // // An API Subscription looks like this: @@ -181,9 +181,11 @@ KjNode* dbModelToApiSubscription KjNode** contextNodePP, KjNode** showChangesP, KjNode** sysAttrsP, - RenderFormat* renderFormatP + RenderFormat* renderFormatP, + double* timeIntervalP ) { + LM_T(LmtSR, ("Here")); KjNode* dbSubIdP = kjLookup(dbSubP, "_id"); DB_ITEM_NOT_FOUND(dbSubIdP, "id", tenant); KjNode* dbNameP = kjLookup(dbSubP, "name"); KjNode* dbDescriptionP = kjLookup(dbSubP, "description"); @@ -210,6 +212,7 @@ KjNode* dbModelToApiSubscription KjNode* dbSysAttrsP = kjLookup(dbSubP, "sysAttrs"); KjNode* dbCreatedAtP = NULL; KjNode* dbModifiedAtP = NULL; + KjNode* timeIntervalNodeP = kjLookup(dbSubP, "timeInterval"); if (orionldState.uriParamOptions.sysAttrs == true) { @@ -277,7 +280,6 @@ KjNode* dbModelToApiSubscription if (dbDescriptionP != NULL) kjChildAdd(apiSubP, dbDescriptionP); - // // "entities" // @@ -287,39 +289,35 @@ KjNode* dbModelToApiSubscription // for (KjNode* entityP = dbEntitiesP->value.firstChildP; entityP != NULL; entityP = entityP->next) { + KjNode* idP = kjLookup(entityP, "id"); KjNode* isPatternP = kjLookup(entityP, "isPattern"); - KjNode* isTypePatternP = kjLookup(entityP, "isTypePattern"); KjNode* typeP = kjLookup(entityP, "type"); + KjNode* isTypePatternP = kjLookup(entityP, "isTypePattern"); + // There is no "Type Pattern" in NGSI-LD + kjChildRemove(entityP, isTypePatternP); - // bool ngsild = (dbLdContextP != NULL); - // if (ngsild) - { - if (isTypePatternP != NULL) - kjChildRemove(entityP, isTypePatternP); + // There is no "isPattern" in NGSI-LD + kjChildRemove(entityP, isPatternP); - if (isPatternP != NULL) - { - if (strcmp(isPatternP->value.s, "true") == 0) - { - KjNode* idP = kjLookup(entityP, "id"); - if (strcmp(idP->value.s, ".*") == 0) - kjChildRemove(entityP, idP); - else - idP->name = (char*) "idPattern"; - } - - kjChildRemove(entityP, isPatternP); - - // type must be compacted - if (typeP != NULL) // Can't really be NULL though ... - { - // But, for sub-cache we need the long names - if (forSubCache == false) - typeP->value.s = orionldContextItemAliasLookup(orionldState.contextP, typeP->value.s, NULL, NULL); - } - } + LM_T(LmtSR, ("entityId: %s", idP->value.s)); + LM_T(LmtSR, ("isPattern: %s", isPatternP->value.s)); + LM_T(LmtSR, ("type: %s", typeP->value.s)); + + if (strcmp(isPatternP->value.s, "true") == 0) + { + if (strcmp(idP->value.s, ".*") == 0) + kjChildRemove(entityP, idP); + else + idP->name = (char*) "idPattern"; } + + kjChildRemove(entityP, isPatternP); + + // type must be compacted + // However, for sub-cache we need the long names + if (forSubCache == false) + typeP->value.s = orionldContextItemAliasLookup(orionldState.contextP, typeP->value.s, NULL, NULL); } kjChildAdd(apiSubP, dbEntitiesP); @@ -339,6 +337,13 @@ KjNode* dbModelToApiSubscription } } + // timeInterval + if (timeIntervalNodeP != NULL) + { + kjChildAdd(apiSubP, timeIntervalNodeP); + *timeIntervalP = (timeIntervalNodeP->type == KjInt)? timeIntervalNodeP->value.i : timeIntervalNodeP->value.f; + } + // "q" for NGSI-LD if ((dbLdqP != NULL) && (dbLdqP->value.s[0] != 0)) { @@ -635,11 +640,12 @@ KjNode* dbModelToApiSubscription kjChildAdd(apiSubP, modifiedAtNodeP); } -#if 0 - // ldContext ... not yet implemented in the API SPEC for subscriptions to have their own @context + // jsonldContext if (dbLdContextP != NULL) + { kjChildAdd(apiSubP, dbLdContextP); -#endif + dbLdContextP->name = (char*) "jsonldContext"; + } // count - take sub-cache delta into consideration if (dbCountP != NULL) diff --git a/src/lib/orionld/dbModel/dbModelToApiSubscription.h b/src/lib/orionld/dbModel/dbModelToApiSubscription.h index 256ecbf0c9..fa43666f0b 100644 --- a/src/lib/orionld/dbModel/dbModelToApiSubscription.h +++ b/src/lib/orionld/dbModel/dbModelToApiSubscription.h @@ -50,7 +50,8 @@ extern KjNode* dbModelToApiSubscription KjNode** contextNodePP, KjNode** showChangesP, KjNode** sysAttrsP, - RenderFormat* renderFormatP + RenderFormat* renderFormatP, + double* timeIntervalP ); #endif // SRC_LIB_ORIONLD_DBMODEL_DBMODELTOAPISUBSCRIPTION_H_ diff --git a/src/lib/orionld/kjTree/CMakeLists.txt b/src/lib/orionld/kjTree/CMakeLists.txt index 7f253b1c74..b30da0dc3f 100644 --- a/src/lib/orionld/kjTree/CMakeLists.txt +++ b/src/lib/orionld/kjTree/CMakeLists.txt @@ -60,6 +60,7 @@ SET (SOURCES kjNodeDecouple.cpp kjNavigate.cpp kjTreeFromCachedSubscription.cpp + kjTreeFromPernotSubscription.cpp kjAttributeNormalizedToSimplified.cpp kjAttributeNormalizedToConcise.cpp kjSysAttrsRemove.cpp diff --git a/src/lib/orionld/kjTree/kjTreeFromCachedSubscription.cpp b/src/lib/orionld/kjTree/kjTreeFromCachedSubscription.cpp index 9537f3b001..4aa74482bb 100644 --- a/src/lib/orionld/kjTree/kjTreeFromCachedSubscription.cpp +++ b/src/lib/orionld/kjTree/kjTreeFromCachedSubscription.cpp @@ -132,14 +132,21 @@ KjNode* kjTreeFromCachedSubscription(CachedSubscription* cSubP, bool sysAttrs, b KjNode* eObjectP = kjObject(orionldState.kjsonP, NULL); NULL_CHECK(eObjectP); + LM_T(LmtSR, ("Got an entity")); // // id/idPattern // // SPECIAL CASE: // If it's a .* pattern, then it is omitted // - const char* entityId = eiP->entityId.c_str(); - bool noIdNeeded = (eiP->isPattern == true) && (strcmp(entityId, ".*") == 0); + const char* entityId = eiP->entityId.c_str(); + const char* entityType = eiP->entityType.c_str(); + bool noIdNeeded = (eiP->isPattern == true) && (strcmp(entityId, ".*") == 0); + + LM_T(LmtSR, ("entityId: %s", entityId)); + LM_T(LmtSR, ("isPattern: %s", (eiP->isPattern == true)? "true" : "false")); + LM_T(LmtSR, ("entityType: %s", entityType)); + LM_T(LmtSR, ("noIdNeeded: %s", (noIdNeeded == false)? "FALSE" : "TRUE")); if ((noIdNeeded == false) && (*entityId != 0)) { @@ -503,6 +510,16 @@ KjNode* kjTreeFromCachedSubscription(CachedSubscription* cSubP, bool sysAttrs, b kjChildAdd(sP, nodeP); } + // + // jsonldCcontext + // + if (cSubP->ldContext != "") + { + nodeP = kjString(orionldState.kjsonP, "jsonldContext", cSubP->ldContext.c_str()); + NULL_CHECK(nodeP); + kjChildAdd(sP, nodeP); + } + // // @context // diff --git a/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.cpp b/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.cpp new file mode 100644 index 0000000000..14e1199cef --- /dev/null +++ b/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.cpp @@ -0,0 +1,223 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +extern "C" +{ +#include "kjson/KjNode.h" // KjNode +#include "kjson/kjClone.h" // kjClone +#include "kjson/kjLookup.h" // kjLookup +#include "kjson/kjBuilder.h" // kjString, kjChildAdd, ... +} + +#include "logMsg/logMsg.h" // LM_* + +#include "orionld/common/orionldState.h" // orionldState +#include "orionld/common/orionldError.h" // orionldError +#include "orionld/common/numberToDate.h" // numberToDate +#include "orionld/context/orionldContextItemAliasLookup.h" // orionldContextItemAliasLookup +#include "orionld/dbModel/dbModelValueStrip.h" // dbModelValueStrip +#include "orionld/q/qAliasCompact.h" // qAliasCompact +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription + + + +// ----------------------------------------------------------------------------- +// +// NULL_CHECK - +// +#define NULL_CHECK(pointer) if (pointer == NULL) return outOfMemory() + + + +// ----------------------------------------------------------------------------- +// +// outOfMemory - +// +static KjNode* outOfMemory(void) +{ + LM_E(("Internal Error (out of memory creating a KjNode-tree for a PernotSubscription)")); + return NULL; +} + + + +// ----------------------------------------------------------------------------- +// +// timestampAdd - +// +static void timestampAdd(KjNode* containerP, const char* fieldName, double ts) +{ + if (ts <= 0) + return; + + char buf[64]; + if (numberToDate(ts, buf, sizeof(buf) - 1) == true) + { + KjNode* nodeP = kjString(orionldState.kjsonP, fieldName, buf); + kjChildAdd(containerP, nodeP); + } +} + + + +// ----------------------------------------------------------------------------- +// +// counterAdd - +// +static void counterAdd(KjNode* containerP, const char* fieldName, uint32_t count) +{ + if (count == 0) + return; + + KjNode* nodeP = kjInteger(orionldState.kjsonP, fieldName, count); + kjChildAdd(containerP, nodeP); +} + + + +// ----------------------------------------------------------------------------- +// +// kjTreeFromPernotSubscription - in NGSI-LD API format +// +KjNode* kjTreeFromPernotSubscription(PernotSubscription* pSubP, bool sysAttrs, bool contextInBody) +{ + KjNode* nodeP; + + // + // Top level object - the Subscription + // + // 1. We start with what we already get from the Pernot sub-cache. + // + // 2. And, we remove the sysAttrs, unless asked for, + // + // 3. We add the "volatile" fields: + // "isActive", + // "status" + // "notification::status" + // + // 4. We add counters and timestamps: + // - timesSent + // - timesFailed + // - lastNotification + // - lastFailure + // - lastSuccess + // + // 5. And we compact all fields that need it, according with the current @context + // + // 6. Compact attribute names in q + // + KjNode* sP = kjClone(orionldState.kjsonP, pSubP->kjSubP); + NULL_CHECK(sP); + + KjNode* notificationP = kjLookup(sP, "notification"); + if (notificationP == NULL) + { + orionldError(OrionldInternalError, "Database Error (subscription without notification field)", pSubP->subscriptionId, 500); + return NULL; + } + + // 2. Remove modifiedAt, createdAt? + if (sysAttrs == false) + { + nodeP = kjLookup(sP, "modifiedAt"); + if (nodeP != NULL) + kjChildRemove(sP, nodeP); + + nodeP = kjLookup(sP, "createdAt"); + if (nodeP != NULL) + kjChildRemove(sP, nodeP); + } + + // 3. Add the "volatile" fields, but, first decide the states + bool isActive = true; + char* status = (char*) "active"; + char* notificationStatus = (char*) "ok"; + // + // Lookup and compare with: + // * expiresAt + // * any error + // * paused, ... + // + KjNode* isActiveP = kjBoolean(orionldState.kjsonP, "isActive", isActive); + kjChildAdd(sP, isActiveP); + KjNode* statusP = kjString(orionldState.kjsonP, "status", status); + kjChildAdd(sP, statusP); + KjNode* notificationStatusP = kjString(orionldState.kjsonP, "status", notificationStatus); + kjChildAdd(notificationP, notificationStatusP); + + // 4. counters and timestamps + counterAdd(notificationP, "timesSent", pSubP->dbNotificationAttempts + pSubP->notificationAttempts); + counterAdd(notificationP, "timesFailed", pSubP->dbNotificationErrors + pSubP->notificationErrors); + timestampAdd(notificationP, "lastNotification", pSubP->lastNotificationAttempt); + timestampAdd(notificationP, "lastSuccess", pSubP->lastSuccessTime); + timestampAdd(notificationP, "lastFailure", pSubP->lastFailureTime); + + // 5.1. Compact entity types + KjNode* entitiesP = kjLookup(sP, "entities"); + if (entitiesP != NULL) + { + for (KjNode* entitiesItemP = entitiesP->value.firstChildP; entitiesItemP != NULL; entitiesItemP = entitiesItemP->next) + { + KjNode* typeP = kjLookup(entitiesItemP, "type"); + + if (typeP != NULL) + typeP->value.s = orionldContextItemAliasLookup(orionldState.contextP, typeP->value.s, NULL, NULL); + } + } + + // 5.2. Compact notification::attributes + KjNode* attributesP = kjLookup(notificationP, "attributes"); + if (attributesP != NULL) + { + for (KjNode* attributeP = attributesP->value.firstChildP; attributeP != NULL; attributeP = attributeP->next) + { + attributeP->value.s = orionldContextItemAliasLookup(orionldState.contextP, attributeP->value.s, NULL, NULL); + } + } + + // + // 6. Compact attribute names in q + // + KjNode* qP = kjLookup(sP, "q"); + if (qP != NULL) + { + LM_T(LmtPernot, ("Found 'q' - fixing it", qP->value.s)); + dbModelValueStrip(qP); + qAliasCompact(qP, true); // qAliasCompact uses orionldState.contextP - which is what we want + qP->name = (char*) "q"; + } + else + LM_T(LmtPernot, ("No 'q'")); + + // + // name => subscriptionName + // FIXME: don't do this every time! + // do it in pernotSubCacheAdd! + // + KjNode* nameP = kjLookup(sP, "name"); + if (nameP != NULL) + nameP->name = (char*) "subscriptionName"; + + return sP; +} diff --git a/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.h b/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.h new file mode 100644 index 0000000000..515e0a901a --- /dev/null +++ b/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.h @@ -0,0 +1,43 @@ +#ifndef SRC_LIB_ORIONLD_KJTREE_KJTREEFROMPERNOTSUBSCRIPTION_H_ +#define SRC_LIB_ORIONLD_KJTREE_KJTREEFROMPERNOTSUBSCRIPTION_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +extern "C" +{ +#include "kjson/KjNode.h" // KjNode +} + +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription + + + +// ----------------------------------------------------------------------------- +// +// kjTreeFromPernotSubscription - in NGSI-LD API format +// +extern KjNode* kjTreeFromPernotSubscription(PernotSubscription* cSubP, bool sysAttrs, bool contextInBody); + +#endif // SRC_LIB_ORIONLD_KJTREE_KJTREEFROMPERNOTSUBSCRIPTION_H_ diff --git a/src/lib/orionld/kjTree/kjTreeToSubscription.cpp b/src/lib/orionld/kjTree/kjTreeToSubscription.cpp index 1af156d43b..5b69331dda 100644 --- a/src/lib/orionld/kjTree/kjTreeToSubscription.cpp +++ b/src/lib/orionld/kjTree/kjTreeToSubscription.cpp @@ -257,11 +257,12 @@ bool kjTreeToSubscription(ngsiv2::Subscription* subP, char** subIdPP, KjNode** e else if (strcmp(kNodeP->name, "timeInterval") == 0) { DUPLICATE_CHECK(timeIntervalP, "Subscription::timeInterval", kNodeP); - INTEGER_CHECK(timeIntervalP, "Subscription::timeInterval"); - subP->timeInterval = timeIntervalP->value.i; + NUMBER_CHECK(timeIntervalP, "Subscription::timeInterval"); + subP->timeInterval = (timeIntervalP->type == KjInt)? timeIntervalP->value.i : timeIntervalP->value.f; + if (subP->timeInterval <= 0) { - orionldError(OrionldBadRequestData, "Invalid value for Subscription::timeInterval", "must be an integer value > 0", 400); + orionldError(OrionldBadRequestData, "Invalid value for Subscription::timeInterval", "must be a Number > 0", 400); return false; } } @@ -363,7 +364,7 @@ bool kjTreeToSubscription(ngsiv2::Subscription* subP, char** subIdPP, KjNode** e if ((timeIntervalP != NULL) && (watchedAttributesPresent == true)) { - orionldError(OrionldBadRequestData, "Both 'timeInterval' and 'watchedAttributes' present", NULL, 400); + orionldError(OrionldBadRequestData, "Inconsistent subscription", "Both 'timeInterval' and 'watchedAttributes' present", 400); return false; } diff --git a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntitiesDelete.cpp b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntitiesDelete.cpp index ccd775ce71..9c9d88b3ce 100644 --- a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntitiesDelete.cpp +++ b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntitiesDelete.cpp @@ -33,7 +33,7 @@ extern "C" #include "logMsg/logMsg.h" // LM_* #include "logMsg/traceLevels.h" // Lmt* -#include "orionld/common/orionldState.h" // orionldState, dbName, mongoEntitiesCollectionP +#include "orionld/common/orionldState.h" // orionldState, dbName #include "mongoBackend/MongoGlobal.h" // getMongoConnection, releaseMongoConnection, ... #include "orionld/mongoCppLegacy/mongoCppLegacyEntitiesDelete.h" // Own interface diff --git a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityFieldDelete.cpp b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityFieldDelete.cpp index 9318e67679..b636d1d7f3 100644 --- a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityFieldDelete.cpp +++ b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityFieldDelete.cpp @@ -32,7 +32,7 @@ extern "C" #include "logMsg/logMsg.h" // LM_* #include "logMsg/traceLevels.h" // Lmt* -#include "orionld/common/orionldState.h" // orionldState, dbName, mongoEntitiesCollectionP +#include "orionld/common/orionldState.h" // orionldState, dbName #include "mongoBackend/MongoGlobal.h" // getMongoConnection, releaseMongoConnection, ... #include "orionld/mongoCppLegacy/mongoCppLegacyEntityFieldDelete.h" // Own interface diff --git a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityFieldReplace.cpp b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityFieldReplace.cpp index c3f92e359c..0fc699a78e 100644 --- a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityFieldReplace.cpp +++ b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityFieldReplace.cpp @@ -34,7 +34,7 @@ extern "C" #include "mongoBackend/MongoGlobal.h" // getMongoConnection, releaseMongoConnection, ... -#include "orionld/common/orionldState.h" // orionldState, dbName, mongoEntitiesCollectionP +#include "orionld/common/orionldState.h" // orionldState, dbName #include "orionld/mongoCppLegacy/mongoCppLegacyKjTreeToBsonObj.h" // mongoCppLegacyKjTreeToBsonObj #include "orionld/mongoCppLegacy/mongoCppLegacyEntityFieldReplace.h" // Own interface diff --git a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityListLookupWithIdTypeCreDate.cpp b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityListLookupWithIdTypeCreDate.cpp index 1f3a54c1c5..20ef6208dd 100644 --- a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityListLookupWithIdTypeCreDate.cpp +++ b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityListLookupWithIdTypeCreDate.cpp @@ -39,7 +39,7 @@ extern "C" #include "mongoBackend/MongoGlobal.h" // getMongoConnection, releaseMongoConnection, ... #include "mongoBackend/safeMongo.h" // getStringFieldF, ... -#include "orionld/common/orionldState.h" // orionldState, dbName, mongoEntitiesCollectionP +#include "orionld/common/orionldState.h" // orionldState, dbName #include "orionld/mongoCppLegacy/mongoCppLegacyDbNumberFieldGet.h" // mongoCppLegacyDbNumberFieldGet #include "orionld/mongoCppLegacy/mongoCppLegacyDbStringFieldGet.h" // mongoCppLegacyDbStringFieldGet #include "orionld/mongoCppLegacy/mongoCppLegacyDbObjectFieldGet.h" // mongoCppLegacyDbObjectFieldGet diff --git a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityUpdate.cpp b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityUpdate.cpp index e8bd60d76f..1d92744a70 100644 --- a/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityUpdate.cpp +++ b/src/lib/orionld/mongoCppLegacy/mongoCppLegacyEntityUpdate.cpp @@ -32,7 +32,7 @@ extern "C" #include "logMsg/logMsg.h" // LM_* #include "logMsg/traceLevels.h" // Lmt* -#include "orionld/common/orionldState.h" // orionldState, dbName, mongoEntitiesCollectionP +#include "orionld/common/orionldState.h" // orionldState, dbName #include "mongoBackend/MongoGlobal.h" // getMongoConnection, releaseMongoConnection, ... #include "orionld/mongoCppLegacy/mongoCppLegacyDataToKjTree.h" // mongoCppLegacyDataToKjTree diff --git a/src/lib/orionld/mongoCppLegacy/mongoCppLegacySubscriptionMatchEntityIdAndAttributes.cpp b/src/lib/orionld/mongoCppLegacy/mongoCppLegacySubscriptionMatchEntityIdAndAttributes.cpp index 11e57d70c8..03eb047fab 100644 --- a/src/lib/orionld/mongoCppLegacy/mongoCppLegacySubscriptionMatchEntityIdAndAttributes.cpp +++ b/src/lib/orionld/mongoCppLegacy/mongoCppLegacySubscriptionMatchEntityIdAndAttributes.cpp @@ -34,7 +34,7 @@ extern "C" #include "logMsg/traceLevels.h" // Lmt* #include "mongoBackend/MongoGlobal.h" // getMongoConnection, releaseMongoConnection, ... -#include "orionld/common/orionldState.h" // orionldState, dbName, mongoEntitiesCollectionP +#include "orionld/common/orionldState.h" // orionldState, dbName #include "orionld/mongoCppLegacy/mongoCppLegacyDataToKjTree.h" // mongoCppLegacyDataToKjTree #include "orionld/mongoCppLegacy/mongoCppLegacySubscriptionMatchEntityIdAndAttributes.h" // Own interface diff --git a/src/lib/orionld/mongoc/mongocSubCachePopulateByTenant.cpp b/src/lib/orionld/mongoc/mongocSubCachePopulateByTenant.cpp index fede4a67d4..e3b7fbbc38 100644 --- a/src/lib/orionld/mongoc/mongocSubCachePopulateByTenant.cpp +++ b/src/lib/orionld/mongoc/mongocSubCachePopulateByTenant.cpp @@ -34,6 +34,7 @@ extern "C" #include "orionld/common/orionldState.h" // mongocPool #include "orionld/common/subCacheApiSubscriptionInsert.h" // subCacheApiSubscriptionInsert +#include "orionld/pernot/pernotSubCacheAdd.h" // pernotSubCacheAdd #include "orionld/dbModel/dbModelToApiSubscription.h" // dbModelToApiSubscription #include "orionld/types/OrionldTenant.h" // OrionldTenant #include "orionld/kjTree/kjTreeLog.h" // kjTreeLog @@ -104,6 +105,7 @@ bool mongocSubCachePopulateByTenant(OrionldTenant* tenantP) KjNode* showChangesP = NULL; KjNode* sysAttrsP = NULL; RenderFormat renderFormat = RF_NORMALIZED; + double timeInterval = 0; KjNode* apiSubP = dbModelToApiSubscription(dbSubP, tenantP->tenant, true, @@ -112,7 +114,8 @@ bool mongocSubCachePopulateByTenant(OrionldTenant* tenantP) &contextNodeP, &showChangesP, &sysAttrsP, - &renderFormat); + &renderFormat, + &timeInterval); if (apiSubP == NULL) continue; @@ -121,7 +124,10 @@ bool mongocSubCachePopulateByTenant(OrionldTenant* tenantP) if (contextNodeP != NULL) contextP = orionldContextFromUrl(contextNodeP->value.s, NULL); - subCacheApiSubscriptionInsert(apiSubP, qTree, coordinatesP, contextP, tenantP->tenant, showChangesP, sysAttrsP, renderFormat); + if (timeInterval == 0) + subCacheApiSubscriptionInsert(apiSubP, qTree, coordinatesP, contextP, tenantP->tenant, showChangesP, sysAttrsP, renderFormat); + else + pernotSubCacheAdd(NULL, apiSubP, NULL, qTree, coordinatesP, contextP, tenantP->tenant, showChangesP, sysAttrsP, renderFormat, timeInterval); } mongoc_client_pool_push(mongocPool, connectionP); diff --git a/src/lib/orionld/payloadCheck/PCHECK.h b/src/lib/orionld/payloadCheck/PCHECK.h index 10d8a6d75e..7d25162bfe 100644 --- a/src/lib/orionld/payloadCheck/PCHECK.h +++ b/src/lib/orionld/payloadCheck/PCHECK.h @@ -96,6 +96,25 @@ do +// ----------------------------------------------------------------------------- +// +// PCHECK_INTEGER - +// +#define PCHECK_INTEGER(kNodeP, _type, _title, detail, status) \ +do \ +{ \ + if (kNodeP->type != KjInt) \ + { \ + int type = (_type == 0)? OrionldBadRequestData : _type; \ + const char* title = (_title == NULL)? "Not a JSON Integer" : _title; \ + \ + orionldError((OrionldResponseErrorType) type, title, detail, status); \ + return false; \ + } \ +} while (0) + + + // ----------------------------------------------------------------------------- // // PCHECK_STRING_OR_ARRAY - @@ -168,6 +187,34 @@ do +// ----------------------------------------------------------------------------- +// +// PCHECK_NUMBER_GT - +// +#define PCHECK_NUMBER_GT(kNodeP, _type, _title, detail, status, minValue) \ +do \ +{ \ + if ((kNodeP->type == KjInt) && (kNodeP->value.i <= minValue)) \ + { \ + int type = (_type == 0)? OrionldBadRequestData : _type; \ + const char* title = (_title == NULL)? "Too small a Number" : _title; \ + \ + orionldError((OrionldResponseErrorType) type, title, detail, status); \ + return false; \ + } \ + \ + if ((kNodeP->type == KjFloat) && (kNodeP->value.f <= minValue)) \ + { \ + int type = (_type == 0)? OrionldBadRequestData : _type; \ + const char* title = (_title == NULL)? "Too small a Number" : _title; \ + \ + orionldError((OrionldResponseErrorType) type, title, detail, status); \ + return false; \ + } \ +} while (0) + + + // ----------------------------------------------------------------------------- // // PCHECK_OBJECT - diff --git a/src/lib/orionld/payloadCheck/fieldPaths.cpp b/src/lib/orionld/payloadCheck/fieldPaths.cpp index 98d0771f24..a9849a278b 100644 --- a/src/lib/orionld/payloadCheck/fieldPaths.cpp +++ b/src/lib/orionld/payloadCheck/fieldPaths.cpp @@ -40,6 +40,7 @@ const char* SubscriptionEntitiesIdPatternPath = "Subscription::entities const char* SubscriptionEntitiesTypePath = "Subscription::entities::type"; const char* SubscriptionWatchedAttributesPath = "Subscription::watchedAttributes"; const char* SubscriptionWatchedAttributesItemPath = "Subscription::watchedAttributes[X]"; +const char* SubscriptionTimeIntervalPath = "Subscription::timeInterval"; const char* SubscriptionNotificationPath = "Subscription::notification"; const char* SubscriptionNotificationFormatPath = "Subscription::notification::format"; const char* SubscriptionNotificationAttributesPath = "Subscription::notification::attributes"; diff --git a/src/lib/orionld/payloadCheck/fieldPaths.h b/src/lib/orionld/payloadCheck/fieldPaths.h index 274502887a..777f7f9e5b 100644 --- a/src/lib/orionld/payloadCheck/fieldPaths.h +++ b/src/lib/orionld/payloadCheck/fieldPaths.h @@ -47,6 +47,7 @@ extern const char* SubscriptionNotificationShowChangesPath; extern const char* SubscriptionNotificationSysAttrsPath; extern const char* SubscriptionWatchedAttributesPath; extern const char* SubscriptionWatchedAttributesItemPath; +extern const char* SubscriptionTimeIntervalPath; extern const char* SubscriptionQPath; extern const char* SubscriptionGeoqPath; extern const char* SubscriptionIsActivePath; diff --git a/src/lib/orionld/payloadCheck/pCheckSubscription.cpp b/src/lib/orionld/payloadCheck/pCheckSubscription.cpp index a20662c161..56afbbe5b4 100644 --- a/src/lib/orionld/payloadCheck/pCheckSubscription.cpp +++ b/src/lib/orionld/payloadCheck/pCheckSubscription.cpp @@ -114,6 +114,7 @@ bool pCheckSubscription bool* mqttChangeP, KjNode** showChangesP, KjNode** sysAttrsP, + double* timeInterval, RenderFormat* renderFormatP ) { @@ -130,11 +131,13 @@ bool pCheckSubscription KjNode* qP = NULL; KjNode* geoqP = NULL; KjNode* langP = NULL; + KjNode* timeIntervalP = NULL; double expiresAt; *mqttChangeP = NULL; *showChangesP = NULL; *sysAttrsP = NULL; + *timeInterval = 0; if (idP != NULL) { @@ -199,8 +202,11 @@ bool pCheckSubscription } else if (strcmp(subItemP->name, "timeInterval") == 0) { - orionldError(OrionldOperationNotSupported, "Not Implemented", "Time-Interval for Subscriptions", 501); - return false; + PCHECK_DUPLICATE(timeIntervalP, subItemP, 0, NULL, SubscriptionTimeIntervalPath, 400); + PCHECK_NUMBER(timeIntervalP, 0, NULL, SubscriptionTimeIntervalPath, 400); + PCHECK_NUMBER_GT(timeIntervalP, 0, "Non-supported timeInterval (must be greater than zero)", SubscriptionTimeIntervalPath, 400, 0); + + *timeInterval = (timeIntervalP->type == KjInt)? timeIntervalP->value.i : timeIntervalP->value.f; } else if (strcmp(subItemP->name, "q") == 0) { @@ -251,20 +257,7 @@ bool pCheckSubscription { PCHECK_DUPLICATE(throttlingP, subItemP, 0, NULL, SubscriptionThrottlingPath, 400); PCHECK_NUMBER(throttlingP, 0, NULL, SubscriptionThrottlingPath, 400); - - // - // Can't be negative - // - if ((throttlingP->type == KjInt) && (throttlingP->value.i < 0)) - { - orionldError(OrionldBadRequestData, "Negative Number not allowed in this position", SubscriptionThrottlingPath, 400); - return false; - } - else if ((throttlingP->type == KjFloat) && (throttlingP->value.f < 0)) - { - orionldError(OrionldBadRequestData, "Negative Number not allowed in this position", SubscriptionThrottlingPath, 400); - return false; - } + PCHECK_NUMBER_GT(throttlingP, 0, "Negative Number not allowed in this position", SubscriptionThrottlingPath, 400, 0); } else if (strcmp(subItemP->name, "lang") == 0) { @@ -281,9 +274,14 @@ bool pCheckSubscription orionldError(OrionldOperationNotSupported, "Not Implemented", SubscriptionScopePath, 501); return false; } - else if (strcmp(subItemP->name, "status") == 0) { kjChildRemove(subP, subItemP); } // Silently REMOVED - else if (strcmp(subItemP->name, "createdAt") == 0) { kjChildRemove(subP, subItemP); } // Silently REMOVED - else if (strcmp(subItemP->name, "modifiedAt") == 0) { kjChildRemove(subP, subItemP); } // Silently REMOVED + else if (strcmp(subItemP->name, "status") == 0) { kjChildRemove(subP, subItemP); } // Silently REMOVED + else if (strcmp(subItemP->name, "createdAt") == 0) { kjChildRemove(subP, subItemP); } // Silently REMOVED + else if (strcmp(subItemP->name, "modifiedAt") == 0) { kjChildRemove(subP, subItemP); } // Silently REMOVED + else if (strcmp(subItemP->name, "notificationTrigger") == 0) + { + orionldError(OrionldOperationNotSupported, "Not Implemented", subItemP->name, 501); + return false; + } else { orionldError(OrionldBadRequestData, "Unknown field for subscription", subItemP->name, 400); @@ -307,10 +305,34 @@ bool pCheckSubscription return false; } - if ((entitiesP == NULL) && (watchedAttributesP == NULL)) + if (*timeInterval != 0) { - orionldError(OrionldBadRequestData, "Mandatory field missing", "At least one of 'entities' and 'watchedAttributes' must be present" , 400); - return false; + if (entitiesP == NULL) + { + orionldError(OrionldBadRequestData, "Mandatory field missing", "'entities' is mandatory for timeInterval subscriptions" , 400); + return false; + } + + // Make sure it is consistent + if (watchedAttributesP != NULL) + { + orionldError(OrionldBadRequestData, "Inconsistent subscription", "Both 'timeInterval' and 'watchedAttributes' are present", 400); + return false; + } + + if (throttlingP != NULL) + { + orionldError(OrionldBadRequestData, "Inconsistent subscription", "Both 'timeInterval' and 'throttling' are present", 400); + return false; + } + } + else + { + if ((entitiesP == NULL) && (watchedAttributesP == NULL)) + { + orionldError(OrionldBadRequestData, "Mandatory field missing", "At least one of 'entities' and 'watchedAttributes' must be present" , 400); + return false; + } } } diff --git a/src/lib/orionld/payloadCheck/pCheckSubscription.h b/src/lib/orionld/payloadCheck/pCheckSubscription.h index b8c3087640..c6c5f79329 100644 --- a/src/lib/orionld/payloadCheck/pCheckSubscription.h +++ b/src/lib/orionld/payloadCheck/pCheckSubscription.h @@ -58,6 +58,7 @@ extern bool pCheckSubscription bool* mqttChangeP, KjNode** showChangesP, KjNode** sysAttrsP, + double* timeInterval, RenderFormat* renderFormatP ); diff --git a/src/lib/orionld/pernot/CMakeLists.txt b/src/lib/orionld/pernot/CMakeLists.txt index 304a5e8347..4c31454352 100644 --- a/src/lib/orionld/pernot/CMakeLists.txt +++ b/src/lib/orionld/pernot/CMakeLists.txt @@ -21,6 +21,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) SET (SOURCES + pernotSubCacheInit.cpp + pernotSubCacheAdd.cpp + pernotSubCacheRemove.cpp + pernotSubCacheLookup.cpp pernotLoop.cpp pernotTreat.cpp ) diff --git a/src/lib/orionld/pernot/PernotSubCache.cpp b/src/lib/orionld/pernot/PernotSubCache.cpp new file mode 100644 index 0000000000..60f1c34265 --- /dev/null +++ b/src/lib/orionld/pernot/PernotSubCache.cpp @@ -0,0 +1,62 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "common/RenderFormat.h" // RenderFormat + +#include "orionld/q/QNode.h" // QNode +#include "orionld/context/OrionldContext.h" // OrionldContext +#include "orionld/pernot/PernotSubCache.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheAdd - +// +PernotSubscription* pernotSubCacheAdd +( + const char* subscriptionId, + KjNode* subP, + QNode* qTree, + KjNode* geoCoordinatesP, + OrionldContext* contextP, + const char* tenant, + KjNode* showChangesP, + KjNode* sysAttrsP, + RenderFormat renderFormat +) +{ + return NULL; +} + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheRemove - +// +bool pernotSubCacheRemove(PernotSubCache* pSubP) +{ + return true; +} diff --git a/src/lib/orionld/pernot/PernotSubCache.h b/src/lib/orionld/pernot/PernotSubCache.h index b4a6fd9e2f..8ff6440f44 100644 --- a/src/lib/orionld/pernot/PernotSubCache.h +++ b/src/lib/orionld/pernot/PernotSubCache.h @@ -25,7 +25,7 @@ * * Author: Ken Zangelin */ -#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription diff --git a/src/lib/orionld/pernot/PernotSubscription.h b/src/lib/orionld/pernot/PernotSubscription.h index b1c91e6347..d808e9f80e 100644 --- a/src/lib/orionld/pernot/PernotSubscription.h +++ b/src/lib/orionld/pernot/PernotSubscription.h @@ -67,13 +67,19 @@ typedef struct PernotSubscription double lastFailureTime; double expiresAt; + // Counters + uint32_t dbNotificationAttempts; // Total number of notification attempts, in DB + uint32_t notificationAttempts; // Total number of notification attempts, in cache (to be added to dbCount) + uint32_t dbNotificationErrors; // Total number of FAILED notification attempts, in DB + uint32_t notificationErrors; // Total number of FAILED notification attempts, in cache (to be added to dbNotificationErrors) + // Error handling uint32_t consecutiveErrors; - uint32_t timeInterval; // In seconds (Subscription::timeInterval is an integer in seconds) + double timeInterval; // In seconds uint32_t cooldown; CURL* curlHandle; - + struct PernotSubscription* next; } PernotSubscription; diff --git a/src/lib/orionld/pernot/pernotLoop.cpp b/src/lib/orionld/pernot/pernotLoop.cpp index 7e5506db40..fb7bdffa69 100644 --- a/src/lib/orionld/pernot/pernotLoop.cpp +++ b/src/lib/orionld/pernot/pernotLoop.cpp @@ -28,7 +28,7 @@ #include "logMsg/logMsg.h" // LM_x -#include "orionld/common/orionldState.h" // orionldState +#include "orionld/common/orionldState.h" // orionldState, pernotSubCache #include "orionld/pernot/PernotSubscription.h" // PernotSubscription #include "orionld/pernot/PernotSubCache.h" // PernotSubCache #include "orionld/pernot/pernotTreat.h" // pernotTreat @@ -36,25 +36,6 @@ -// ----------------------------------------------------------------------------- -// -// pernotSubCache - -// -PernotSubCache pernotSubCache; - - - -// ----------------------------------------------------------------------------- -// -// pernotSubCacheInit - move to pernot/pernotSubCacheInit.h/cpp -// -void pernotSubCacheInit(void) -{ - bzero(&pernotSubCache, sizeof(pernotSubCache)); -} - - - // ----------------------------------------------------------------------------- // // pernotSubInsert - move to pernot/pernotSubInsert.h/cpp diff --git a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp new file mode 100644 index 0000000000..1e619902ef --- /dev/null +++ b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp @@ -0,0 +1,183 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +extern "C" +{ +#include "kjson/KjNode.h" // KjNode +#include "kjson/kjLookup.h" // kjLookup +#include "kjson/kjClone.h" // kjClone +#include "kjson/kjBuilder.h" // kjString, kjChildAdd +} + +#include "logMsg/logMsg.h" // LM_* + +#include "common/RenderFormat.h" // RenderFormat + +#include "orionld/common/orionldState.h" // pernotSubCache +#include "orionld/q/QNode.h" // QNode +#include "orionld/context/OrionldContext.h" // OrionldContext +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/PernotSubCache.h" // PernotSubCache + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheAdd - +// +PernotSubscription* pernotSubCacheAdd +( + char* subscriptionId, + KjNode* apiSubP, + KjNode* endpointP, + QNode* qTree, + KjNode* geoCoordinatesP, + OrionldContext* contextP, + const char* tenant, + KjNode* showChangesP, + KjNode* sysAttrsP, + RenderFormat renderFormat, + double timeInterval +) +{ + PernotSubscription* pSubP = (PernotSubscription*) malloc(sizeof(PernotSubscription)); + + if (subscriptionId == NULL) + { + KjNode* idP = kjLookup(apiSubP, "id"); + if (idP == NULL) + LM_RE(NULL, ("No subscription id")); + subscriptionId = idP->value.s; + } + + LM_T(LmtPernot, ("Adding pernot subscription '%s' to cache", subscriptionId)); + + pSubP->subscriptionId = strdup(subscriptionId); + pSubP->timeInterval = timeInterval; + pSubP->kjSubP = kjClone(NULL, apiSubP); + + kjTreeLog(pSubP->kjSubP, "Initial Pernot Subscription", LmtPernot); + + if (tenant != NULL) + strncpy(pSubP->tenant, tenant, sizeof(pSubP->tenant) - 1); + + // State - check also expiresAt+status + KjNode* isActiveNodeP = kjLookup(apiSubP, "isActive"); + if (isActiveNodeP != NULL) + pSubP->state = (isActiveNodeP->value.b == true)? SubActive : SubPaused; + + pSubP->lastNotificationAttempt = 0; // FIXME: get info from apiSubP + pSubP->lastSuccessTime = 0; // FIXME: get info from apiSubP + pSubP->lastFailureTime = 0; // FIXME: get info from apiSubP + pSubP->expiresAt = 0; // FIXME: get info from apiSubP + pSubP->dbNotificationAttempts = 0; // FIXME: get info from apiSubP + pSubP->notificationAttempts = 0; + pSubP->dbNotificationErrors = 0; // FIXME: get info from apiSubP + pSubP->notificationErrors = 0; + pSubP->consecutiveErrors = 0; // FIXME: get info from apiSubP + pSubP->cooldown = 0; // FIXME: get info from apiSubP + pSubP->curlHandle = NULL; + + // + // Add the subscription to the cache + // + pSubP->next = NULL; + + if (pernotSubCache.head == NULL) + { + pernotSubCache.head = pSubP; + pernotSubCache.tail = pSubP; + } + else + { + pernotSubCache.tail->next = pSubP; + pernotSubCache.tail = pSubP; + } + + // + // Prepare the kjSubP for future GET requests + // Must add the "type", and, perhaps also the "jsonldContext" + // Also, remove "q" and "mq" (that's NGSIv2), + // and then rename "ldQ" to "q" + // AND: + // isActive + // notification::endpoint::accept + // notification::format + // notification::status + // origin (cache) + // status + // + KjNode* typeP = kjString(NULL, "type", "Subscription"); + kjChildAdd(pSubP->kjSubP, typeP); + + KjNode* jsonldContextP = kjLookup(pSubP->kjSubP, "jsonldContext"); + if (jsonldContextP == NULL) + { + jsonldContextP = kjString(NULL, "jsonldContext", orionldState.contextP->url); + kjChildAdd(pSubP->kjSubP, jsonldContextP); + } + + KjNode* qP = kjLookup(pSubP->kjSubP, "q"); + if (qP != NULL) + kjChildRemove(pSubP->kjSubP, qP); + + KjNode* mqP = kjLookup(pSubP->kjSubP, "mq"); + if (mqP != NULL) + kjChildRemove(pSubP->kjSubP, mqP); + + qP = kjLookup(pSubP->kjSubP, "ldQ"); + if (qP != NULL) + qP->name = (char*) "q"; + + // origin + KjNode* originP = kjString(NULL, "origin", "cache"); + kjChildAdd(pSubP->kjSubP, originP); + + // notification + KjNode* notificationP = kjLookup(pSubP->kjSubP, "notification"); + + // notification::format + KjNode* formatP = kjLookup(notificationP, "format"); + if (formatP == NULL) + { + formatP = kjString(NULL, "format", "normalized"); + kjChildAdd(notificationP, formatP); + } + + // notification::endpoint + if (endpointP == NULL) + endpointP = kjLookup(notificationP, "endpoint"); + + // notification::endpoint::accept + KjNode* acceptP = kjLookup(endpointP, "accept"); + if (acceptP == NULL) + { + acceptP = kjString(NULL, "accept", "application/json"); + kjChildAdd(endpointP, acceptP); + } + + kjTreeLog(pSubP->kjSubP, "Pernot Subscription in Cache", LmtPernot); + + return NULL; +} diff --git a/src/lib/orionld/pernot/pernotSubCacheAdd.h b/src/lib/orionld/pernot/pernotSubCacheAdd.h new file mode 100644 index 0000000000..525ed75056 --- /dev/null +++ b/src/lib/orionld/pernot/pernotSubCacheAdd.h @@ -0,0 +1,63 @@ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHEADD_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHEADD_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +extern "C" +{ +#include "kjson/KjNode.h" // KjNode +} + +#include "common/RenderFormat.h" // RenderFormat + +#include "orionld/common/orionldState.h" // pernotSubCache +#include "orionld/q/QNode.h" // QNode +#include "orionld/context/OrionldContext.h" // OrionldContext +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/PernotSubCache.h" // PernotSubCache +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheAdd - +// +extern PernotSubscription* pernotSubCacheAdd +( + char* subscriptionId, + KjNode* subP, + KjNode* endpointP, + QNode* qTree, + KjNode* geoCoordinatesP, + OrionldContext* contextP, + const char* tenant, + KjNode* showChangesP, + KjNode* sysAttrsP, + RenderFormat renderFormat, + double timeInterval +); + +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHEADD_H_ diff --git a/src/lib/orionld/pernot/pernotSubCacheInit.cpp b/src/lib/orionld/pernot/pernotSubCacheInit.cpp new file mode 100644 index 0000000000..0422def956 --- /dev/null +++ b/src/lib/orionld/pernot/pernotSubCacheInit.cpp @@ -0,0 +1,38 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include // bzero + +#include "orionld/common/orionldState.h" // pernotSubCache + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheInit - +// +void pernotSubCacheInit(void) +{ + bzero(&pernotSubCache, sizeof(pernotSubCache)); +} diff --git a/src/lib/orionld/pernot/pernotSubCacheInit.h b/src/lib/orionld/pernot/pernotSubCacheInit.h new file mode 100644 index 0000000000..7a371619c4 --- /dev/null +++ b/src/lib/orionld/pernot/pernotSubCacheInit.h @@ -0,0 +1,37 @@ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHEINIT_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHEINIT_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheInit - +// +extern void pernotSubCacheInit(void); + +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHEINIT_H_ diff --git a/src/lib/orionld/pernot/pernotSubCacheLookup.cpp b/src/lib/orionld/pernot/pernotSubCacheLookup.cpp new file mode 100644 index 0000000000..07b3566872 --- /dev/null +++ b/src/lib/orionld/pernot/pernotSubCacheLookup.cpp @@ -0,0 +1,49 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "orionld/common/orionldState.h" // pernotSubCache +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/PernotSubCache.h" // PernotSubCache +#include "orionld/pernot/pernotSubCacheLookup.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheLookup - +// +PernotSubscription* pernotSubCacheLookup +( + const char* subscriptionId, + const char* tenant +) +{ + for (PernotSubscription* pSubP = pernotSubCache.head; pSubP != NULL; pSubP = pSubP->next) + { + if ((strcmp(pSubP->subscriptionId, subscriptionId) == 0) && (strcmp(pSubP->tenant, tenant) == 0)) + return pSubP; + } + + return NULL; +} diff --git a/src/lib/orionld/pernot/pernotSubCacheLookup.h b/src/lib/orionld/pernot/pernotSubCacheLookup.h new file mode 100644 index 0000000000..b4d7620e30 --- /dev/null +++ b/src/lib/orionld/pernot/pernotSubCacheLookup.h @@ -0,0 +1,42 @@ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHELOOKUP_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHELOOKUP_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheLookup - +// +extern PernotSubscription* pernotSubCacheLookup +( + const char* subscriptionId, + const char* tenant +); + +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHELOOKUP_H_ diff --git a/src/lib/orionld/pernot/pernotSubCacheRemove.cpp b/src/lib/orionld/pernot/pernotSubCacheRemove.cpp new file mode 100644 index 0000000000..357b970c71 --- /dev/null +++ b/src/lib/orionld/pernot/pernotSubCacheRemove.cpp @@ -0,0 +1,37 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/pernotSubCacheRemove.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheRemove - +// +bool pernotSubCacheRemove(PernotSubscription* pSubP) +{ + return true; +} diff --git a/src/lib/orionld/pernot/pernotSubCacheRemove.h b/src/lib/orionld/pernot/pernotSubCacheRemove.h new file mode 100644 index 0000000000..b40898124d --- /dev/null +++ b/src/lib/orionld/pernot/pernotSubCacheRemove.h @@ -0,0 +1,45 @@ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHEREMOVE_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHEREMOVE_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "common/RenderFormat.h" // RenderFormat + +#include "orionld/common/orionldState.h" // pernotSubCache +#include "orionld/q/QNode.h" // QNode +#include "orionld/context/OrionldContext.h" // OrionldContext +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/PernotSubCache.h" // PernotSubCache +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription + + + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheRemove - +// +extern bool pernotSubCacheRemove(PernotSubscription* pSubP); + +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHEREMOVE_H_ diff --git a/src/lib/orionld/q/qAliasCompact.cpp b/src/lib/orionld/q/qAliasCompact.cpp index b6ab959861..bd9c84c49d 100644 --- a/src/lib/orionld/q/qAliasCompact.cpp +++ b/src/lib/orionld/q/qAliasCompact.cpp @@ -71,6 +71,17 @@ bool qAliasCompact(KjNode* qP, bool compact) char c0 = cP[0]; char c1 = cP[1]; + if ((c0 == '(') || (c0 == ')')) + { + LM_T(LmtPernot, ("Found a parenthesis - skipping it")); + ++cP; + varStart = cP; + out[outIx] = c0; + ++outIx; + + continue; + } + if (c0 == '!') { varStart = &cP[1]; @@ -103,6 +114,7 @@ bool qAliasCompact(KjNode* qP, bool compact) ++eqP; } + LM_T(LmtPernot, ("Compacting '%s'", varStart)); alias = orionldContextItemAliasLookup(orionldState.contextP, varStart, NULL, NULL); } else diff --git a/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp b/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp index 496002d845..47e8eae1ee 100644 --- a/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp +++ b/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp @@ -37,8 +37,10 @@ extern "C" #include "orionld/common/orionldError.h" // orionldError #include "orionld/common/numberToDate.h" // numberToDate #include "cache/subCache.h" // CachedSubscription, subCacheItemLookup +#include "orionld/pernot/pernotSubCacheLookup.h" // pernotSubCacheLookup #include "orionld/legacyDriver/legacyGetSubscription.h" // legacyGetSubscription #include "orionld/kjTree/kjTreeFromCachedSubscription.h" // kjTreeFromCachedSubscription +#include "orionld/kjTree/kjTreeFromPernotSubscription.h" // kjTreeFromPernotSubscription #include "orionld/payloadCheck/PCHECK.h" // PCHECK_URI #include "orionld/dbModel/dbModelToApiSubscription.h" // dbModelToApiSubscription #include "orionld/mongoc/mongocSubscriptionLookup.h" // mongocSubscriptionLookup @@ -101,9 +103,20 @@ static void subCounterSet(KjNode* apiSubP, const char* fieldName, int64_t valueI // // orionldSubCounters - FIXME: Own Module // -void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP) +void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP, PernotSubscription* pSubP) { - if (cSubP == NULL) + // + // Three options: + // 1. cSubP given + // 2. pSubP given + // 3, None of them (look up apiSubP::subscriptionId in both cashes) + // + double lastNotificationTime = -1; + double lastSuccess = -1; + double lastFailure = -1; + int timesSent = -1; + + if ((cSubP == NULL) && (pSubP == NULL)) { KjNode* subIdP = kjLookup(apiSubP, "id"); @@ -111,18 +124,36 @@ void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP) return; cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subIdP->value.s); - if (cSubP == NULL) - return; + pSubP = pernotSubCacheLookup(orionldState.tenantP->tenant, subIdP->value.s); + if ((cSubP == NULL) && (pSubP == NULL)) + LM_RVE(("Can't find subscription '%s'", subIdP->value.s)); } - KjNode* notificationNodeP = kjLookup(apiSubP, "notification"); + if (cSubP != NULL) + { + lastNotificationTime = cSubP->lastNotificationTime; + lastSuccess = cSubP->lastSuccess; + lastFailure = cSubP->lastFailure; + timesSent = cSubP->dbCount + cSubP->count; + } + else if (pSubP != NULL) + { + lastNotificationTime = pSubP->lastNotificationAttempt; + lastSuccess = pSubP->lastSuccessTime; + lastFailure = pSubP->lastFailureTime; + timesSent = pSubP->dbNotificationAttempts + pSubP->notificationAttempts; + } + // + // Set the values + // + KjNode* notificationNodeP = kjLookup(apiSubP, "notification"); if (notificationNodeP != NULL) { - subTimestampSet(notificationNodeP, "lastNotification", cSubP->lastNotificationTime); - subTimestampSet(notificationNodeP, "lastSuccess", cSubP->lastSuccess); - subTimestampSet(notificationNodeP, "lastFailure", cSubP->lastFailure); - subCounterSet(notificationNodeP, "timesSent", cSubP->dbCount + cSubP->count); + subTimestampSet(notificationNodeP, "lastNotification", lastNotificationTime); + subTimestampSet(notificationNodeP, "lastSuccess", lastSuccess); + subTimestampSet(notificationNodeP, "lastFailure", lastFailure); + subCounterSet(notificationNodeP, "timesSent", timesSent); } } @@ -149,6 +180,7 @@ static bool orionldGetSubscriptionFromDb(void) KjNode* showChangesP = NULL; // Not needed here, but dbModelToApiSubscription requires it KjNode* sysAttrsP = NULL; // Not needed here, but dbModelToApiSubscription requires it RenderFormat renderFormat = RF_NORMALIZED; // Not needed here, but dbModelToApiSubscription requires it + double timeInterval = 0; KjNode* apiSubP = dbModelToApiSubscription(dbSubP, orionldState.tenantP->tenant, false, @@ -157,7 +189,8 @@ static bool orionldGetSubscriptionFromDb(void) &contextNodeP, &showChangesP, &sysAttrsP, - &renderFormat); + &renderFormat, + &timeInterval); if (apiSubP == NULL) { @@ -166,10 +199,18 @@ static bool orionldGetSubscriptionFromDb(void) } // Need to take counters and timestamps from sub-cache - CachedSubscription* cSubP = subCacheItemLookup(orionldState.tenantP->tenant, orionldState.wildcard[0]); - - if (cSubP != NULL) - orionldSubCounters(apiSubP, cSubP); + if (timeInterval == 0) + { + CachedSubscription* cSubP = subCacheItemLookup(orionldState.tenantP->tenant, orionldState.wildcard[0]); + if (cSubP != NULL) + orionldSubCounters(apiSubP, cSubP, NULL); + } + else + { + PernotSubscription* pSubP = pernotSubCacheLookup(orionldState.wildcard[0], orionldState.tenantP->tenant); + if (pSubP != NULL) + orionldSubCounters(apiSubP, NULL, pSubP); + } orionldState.httpStatusCode = 200; orionldState.responseTree = apiSubP; @@ -200,8 +241,9 @@ bool orionldGetSubscription(void) } char* subscriptionId = orionldState.wildcard[0]; - CachedSubscription* cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subscriptionId); + // "Normal" (onchange) subscription? + CachedSubscription* cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subscriptionId); if (cSubP != NULL) { orionldState.httpStatusCode = 200; @@ -210,6 +252,18 @@ bool orionldGetSubscription(void) return true; } + // pernot subscription? + PernotSubscription* pSubP = pernotSubCacheLookup(subscriptionId, orionldState.tenantP->tenant); + if (pSubP != NULL) + { + kjTreeLog(pSubP->kjSubP, "pernot sub in cache", LmtSR); + orionldState.httpStatusCode = 200; + orionldState.responseTree = kjTreeFromPernotSubscription(pSubP, orionldState.uriParamOptions.sysAttrs, orionldState.out.contentType == JSONLD); + kjTreeLog(orionldState.responseTree, "pernot sub after kjTreeFromPernotSubscription", LmtSR); + + return true; + } + orionldError(OrionldResourceNotFound, "subscription not found", subscriptionId, 404); return false; } diff --git a/src/lib/orionld/serviceRoutines/orionldGetSubscriptions.cpp b/src/lib/orionld/serviceRoutines/orionldGetSubscriptions.cpp index 611e1ee585..8534a5a7d7 100644 --- a/src/lib/orionld/serviceRoutines/orionldGetSubscriptions.cpp +++ b/src/lib/orionld/serviceRoutines/orionldGetSubscriptions.cpp @@ -67,7 +67,7 @@ static bool tenantMatch(OrionldTenant* requestTenantP, const char* subscriptionT } -extern void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP); +extern void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP, PernotSubscription* pSubP); // ----------------------------------------------------------------------------- // // orionldGetSubscriptionsFromDb - @@ -113,6 +113,7 @@ static bool orionldGetSubscriptionsFromDb(void) KjNode* showChangesP = NULL; KjNode* sysAttrsP = NULL; RenderFormat renderFormat = RF_NORMALIZED; + double timeInterval = 0; KjNode* apiSubP = dbModelToApiSubscription(dbSubP, orionldState.tenantP->tenant, false, @@ -121,7 +122,8 @@ static bool orionldGetSubscriptionsFromDb(void) &contextNodeP, &showChangesP, &sysAttrsP, - &renderFormat); + &renderFormat, + &timeInterval); if (apiSubP == NULL) { @@ -138,7 +140,7 @@ static bool orionldGetSubscriptionsFromDb(void) kjChildAdd(apiSubP, nodeP); } - orionldSubCounters(apiSubP, NULL); + orionldSubCounters(apiSubP, NULL, NULL); kjChildAdd(apiSubV, apiSubP); } @@ -158,12 +160,7 @@ bool orionldGetSubscriptions(void) return legacyGetSubscriptions(); if (orionldState.uriParamOptions.fromDb == true) - { - // - // GET Subscriptions with mongoc - // return orionldGetSubscriptionsFromDb(); - } // // Not Legacy, not "From DB" - Getting the subscriptions from the subscription cache diff --git a/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp b/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp index d61ca31ee3..f60b45b395 100644 --- a/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp +++ b/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp @@ -818,6 +818,7 @@ bool orionldPatchSubscription(void) KjNode* notifierInfoP = NULL; KjNode* showChangesP = NULL; KjNode* sysAttrsP = NULL; + double timeInterval = 0; RenderFormat renderFormat = RF_NORMALIZED; bool r; @@ -838,6 +839,7 @@ bool orionldPatchSubscription(void) &mqttChange, &showChangesP, &sysAttrsP, + &timeInterval, &renderFormat); if (r == false) { @@ -860,6 +862,17 @@ bool orionldPatchSubscription(void) return false; } + // + // If the subscription used to be "on-change", timeInterval cannot be set + // + if ((timeInterval != 0) && (kjLookup(dbSubscriptionP, "timeInterval") == NULL)) + { + if (qNodeP != NULL) + qRelease(qNodeP); + orionldError(OrionldResourceNotFound, "Invalid modification (on-change to timeInterval subscription)", subscriptionId, 400); + return false; + } + // // If the subscription used to be an MQTT subscription, the MQTT connection might need closing // Only if MQTT data is modified though (or it stops being an MQTT subscription) @@ -908,7 +921,15 @@ bool orionldPatchSubscription(void) // modified. // ngsildSubscriptionPatch() performs that modification. // - CachedSubscription* cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subscriptionId); + CachedSubscription* cSubP; + + if (timeInterval == 0) + cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subscriptionId); + else + { + // What do I do here? + // Create a copy of CachedSubscription with only the fields needed by ngsildSubscriptionPatch? + } if (ngsildSubscriptionPatch(dbSubscriptionP, cSubP, orionldState.requestTree, qP, geoqP, qRenderedForDb) == false) { @@ -964,8 +985,16 @@ bool orionldPatchSubscription(void) } // Modify the subscription in the subscription cache - if (subCacheItemUpdate(orionldState.tenantP, subscriptionId, patchBody, geoCoordinatesP, qNodeP, qRenderedForDb, showChangesP) == false) - LM_E(("Internal Error (unable to update the cached subscription '%s' after a PATCH)", subscriptionId)); + if (timeInterval == 0) + { + if (subCacheItemUpdate(orionldState.tenantP, subscriptionId, patchBody, geoCoordinatesP, qNodeP, qRenderedForDb, showChangesP) == false) + LM_E(("Internal Error (unable to update the cached subscription '%s' after a PATCH)", subscriptionId)); + } + else + { + // Update the subscription in the pernot-cache + LM_X(1, ("Implement PATCH for pernot subscriptions!")); + } // All OK? 204 No Content orionldState.httpStatusCode = 204; diff --git a/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp b/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp index 2342d97820..faa82cef14 100644 --- a/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp +++ b/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp @@ -22,8 +22,6 @@ * * Author: Ken Zangelin */ -#include "logMsg/logMsg.h" // LM_* - extern "C" { #include "kbase/kMacros.h" // K_FT @@ -31,8 +29,11 @@ extern "C" #include "kjson/KjNode.h" // KjNode #include "kjson/kjLookup.h" // kjLookup #include "kjson/kjBuilder.h" // kjString, kjChildAdd, ... +#include "kjson/kjClone.h" // kjClone } +#include "logMsg/logMsg.h" // LM_* + #include "rest/httpHeaderAdd.h" // httpHeaderLocationAdd #include "cache/subCache.h" // subCacheItemLookup, CachedSubscription @@ -45,6 +46,11 @@ extern "C" #include "orionld/dbModel/dbModelFromApiSubscription.h" // dbModelFromApiSubscription #include "orionld/mongoc/mongocSubscriptionExists.h" // mongocSubscriptionExists #include "orionld/mongoc/mongocSubscriptionInsert.h" // mongocSubscriptionInsert +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/PernotSubCache.h" // PernotSubCache +#include "orionld/pernot/pernotSubCacheAdd.h" // pernotSubCacheAdd +#include "orionld/pernot/pernotSubCacheRemove.h" // pernotSubCacheRemove +#include "orionld/pernot/pernotSubCacheLookup.h" // pernotSubCacheLookup #include "orionld/mqtt/mqttParse.h" // mqttParse #include "orionld/mqtt/mqttConnectionEstablish.h" // mqttConnectionEstablish #include "orionld/mqtt/mqttDisconnect.h" // mqttDisconnect @@ -83,6 +89,7 @@ bool orionldPostSubscriptions(void) bool qIsMq = false; KjNode* showChangesP = NULL; KjNode* sysAttrsP = NULL; + double timeInterval = 0; RenderFormat renderFormat = RF_NORMALIZED; b = pCheckSubscription(subP, @@ -102,6 +109,7 @@ bool orionldPostSubscriptions(void) &mqtt, &showChangesP, &sysAttrsP, + &timeInterval, &renderFormat); if (qRenderedForDb != NULL) @@ -116,18 +124,18 @@ bool orionldPostSubscriptions(void) } // Subscription id special treats + char subscriptionId[80]; if (subIdP != NULL) { subId = subIdP->value.s; - // 'id' needs to be '_id' - mongo stuff ... - subIdP->name = (char*) "_id"; // dbModel ... - // // If the subscription already exists, a "409 Conflict" is returned // char* detail = NULL; - if ((subCacheItemLookup(orionldState.tenantP->tenant, subId) != NULL) || (mongocSubscriptionExists(subId, &detail) == true)) + if ((subCacheItemLookup(orionldState.tenantP->tenant, subId) != NULL) || + (pernotSubCacheLookup(orionldState.tenantP->tenant, subId) != NULL) || + (mongocSubscriptionExists(subId, &detail) == true)) { if (detail == NULL) orionldError(OrionldAlreadyExists, "Subscription already exists", subId, 409); @@ -139,14 +147,15 @@ bool orionldPostSubscriptions(void) return false; } + + strncpy(subscriptionId, subId, sizeof(subscriptionId) - 1); } else { - char subscriptionId[80]; strncpy(subscriptionId, "urn:ngsi-ld:subscription:", sizeof(subscriptionId) - 1); uuidGenerate(&subscriptionId[25], sizeof(subscriptionId) - 25, false); - subIdP = kjString(orionldState.kjsonP, "_id", subscriptionId); + subIdP = kjString(orionldState.kjsonP, "id", subscriptionId); } // Add subId to the tree @@ -206,6 +215,12 @@ bool orionldPostSubscriptions(void) if (mqtt == true) { + if (timeInterval != 0) + { + orionldError(OrionldBadRequestData, "Not Implemented", "Notifications in MQTT for Periodic Notification Subscription", 501); + return false; + } + char* detail = NULL; char* uri = kaStrdup(&orionldState.kalloc, uriP->value.s); // Can't destroy uriP->value.s ... mqttParse is destructive! @@ -250,17 +265,39 @@ bool orionldPostSubscriptions(void) } // sub to cache - BEFORE we change the tree to be according to the DB Model (as the DB model might change some day ...) - CachedSubscription* cSubP = subCacheApiSubscriptionInsert(subP, - qTree, - geoCoordinatesP, - orionldState.contextP, - orionldState.tenantP->tenant, - showChangesP, - sysAttrsP, - renderFormat); + CachedSubscription* cSubP = NULL; + PernotSubscription* pSubP = NULL; + + if (timeInterval == 0) + { + cSubP = subCacheApiSubscriptionInsert(subP, + qTree, + geoCoordinatesP, + orionldState.contextP, + orionldState.tenantP->tenant, + showChangesP, + sysAttrsP, + renderFormat); + } + else + { + // Add subscription to the pernot-cache + pSubP = pernotSubCacheAdd(subscriptionId, + subP, + endpointP, + qTree, + geoCoordinatesP, + orionldState.contextP, + orionldState.tenantP->tenant, + showChangesP, + sysAttrsP, + renderFormat, + timeInterval); + } // dbModel KjNode* dbSubscriptionP = subP; + subIdP->name = (char*) "_id"; // 'id' needs to be '_id' - mongo stuff ... dbModelFromApiSubscription(dbSubscriptionP, false); // sub to db - mongocSubscriptionInsert(subP); @@ -271,7 +308,10 @@ bool orionldPostSubscriptions(void) if (mqttSubscription == true) mqttDisconnect(mqttHost, mqttPort, mqttUser, mqttPassword, mqttVersion); - subCacheItemRemove(cSubP); + if (cSubP != NULL) + subCacheItemRemove(cSubP); + else + pernotSubCacheRemove(pSubP); if (qTree != NULL) qRelease(qTree); @@ -281,19 +321,27 @@ bool orionldPostSubscriptions(void) // // MQTT details of the cached subscription + // - For now, timeInterval cannot be done via MQTT // - bzero(&cSubP->httpInfo.mqtt, sizeof(cSubP->httpInfo.mqtt)); - if (mqttSubscription == true) + if (timeInterval == 0) + { + bzero(&cSubP->httpInfo.mqtt, sizeof(cSubP->httpInfo.mqtt)); + if (mqttSubscription == true) + { + cSubP->httpInfo.mqtt.mqtts = mqtts; + cSubP->httpInfo.mqtt.port = mqttPort; + cSubP->httpInfo.mqtt.qos = mqttQoS; + + if (mqttHost != NULL) strncpy(cSubP->httpInfo.mqtt.host, mqttHost, sizeof(cSubP->httpInfo.mqtt.host) - 1); + if (mqttUser != NULL) strncpy(cSubP->httpInfo.mqtt.username, mqttUser, sizeof(cSubP->httpInfo.mqtt.username) - 1); + if (mqttPassword != NULL) strncpy(cSubP->httpInfo.mqtt.password, mqttPassword, sizeof(cSubP->httpInfo.mqtt.password) - 1); + if (mqttVersion != NULL) strncpy(cSubP->httpInfo.mqtt.version, mqttVersion, sizeof(cSubP->httpInfo.mqtt.version) - 1); + if (mqttTopic != NULL) strncpy(cSubP->httpInfo.mqtt.topic, mqttTopic, sizeof(cSubP->httpInfo.mqtt.topic) - 1); + } + } + else if (mqttSubscription == true) { - cSubP->httpInfo.mqtt.mqtts = mqtts; - cSubP->httpInfo.mqtt.port = mqttPort; - cSubP->httpInfo.mqtt.qos = mqttQoS; - - if (mqttHost != NULL) strncpy(cSubP->httpInfo.mqtt.host, mqttHost, sizeof(cSubP->httpInfo.mqtt.host) - 1); - if (mqttUser != NULL) strncpy(cSubP->httpInfo.mqtt.username, mqttUser, sizeof(cSubP->httpInfo.mqtt.username) - 1); - if (mqttPassword != NULL) strncpy(cSubP->httpInfo.mqtt.password, mqttPassword, sizeof(cSubP->httpInfo.mqtt.password) - 1); - if (mqttVersion != NULL) strncpy(cSubP->httpInfo.mqtt.version, mqttVersion, sizeof(cSubP->httpInfo.mqtt.version) - 1); - if (mqttTopic != NULL) strncpy(cSubP->httpInfo.mqtt.topic, mqttTopic, sizeof(cSubP->httpInfo.mqtt.topic) - 1); + // This is already taken care of. } orionldState.httpStatusCode = 201; diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-new.test b/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-new.test index a062ef892f..10d94d4b49 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-new.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-new.test @@ -1,4 +1,4 @@ -# Copyright 2022 FIWARE Foundation e.V. +1# Copyright 2022 FIWARE Foundation e.V. # # This file is part of Orion-LD Context Broker. # @@ -226,7 +226,7 @@ bye 04. GET subscription S2 - see lang=en ===================================== HTTP/1.1 200 OK -Content-Length: 340 +Content-Length: 418 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -240,6 +240,7 @@ Link: ; rel="http:/ ], "id": "urn:subs:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "lang": "en", "notification": { "attributes": [ diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-with-arrays.test b/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-with-arrays.test index f78f944265..1223f0e394 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-with-arrays.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-with-arrays.test @@ -225,7 +225,7 @@ bye 04. GET subscription S2 - see lang=en ===================================== HTTP/1.1 200 OK -Content-Length: 340 +Content-Length: 418 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -239,6 +239,7 @@ Link: ; rel="http:/ ], "id": "urn:subs:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "lang": "en", "notification": { "attributes": [ diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications.test b/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications.test index 2d9774fcf0..ab76bef833 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications.test @@ -335,7 +335,7 @@ bye 04. GET subscription S2 - see lang=en ===================================== HTTP/1.1 200 OK -Content-Length: 340 +Content-Length: 418 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -349,6 +349,7 @@ Link: ; rel="http:/ ], "id": "urn:subs:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "lang": "en", "notification": { "attributes": [ @@ -373,7 +374,7 @@ Link: ; rel="http:/ 04. GET ALL subscriptions see lang=en in S2 =========================================== HTTP/1.1 200 OK -Content-Length: 671 +Content-Length: 827 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -388,6 +389,7 @@ Link: ; rel="http:/ ], "id": "urn:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "attributes": [ "L1" @@ -415,6 +417,7 @@ Link: ; rel="http:/ ], "id": "urn:subs:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "lang": "en", "notification": { "attributes": [ diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new-false-error-for-optional-field-not-present.test b/test/functionalTest/cases/0000_ngsild/ngsild_new-false-error-for-optional-field-not-present.test index dcdff39f47..4fe5206cd5 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new-false-error-for-optional-field-not-present.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new-false-error-for-optional-field-not-present.test @@ -176,7 +176,7 @@ Location: /ngsi-ld/v1/entities/REGEX(.*) 04. GET subscriptions ===================== HTTP/1.1 200 OK -Content-Length: 319 +Content-Length: 397 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -190,6 +190,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscription:REGEX(.*)", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -252,7 +253,7 @@ Link: ; rel="http:/ 07. GET the subscription ======================== HTTP/1.1 200 OK -Content-Length: 317 +Content-Length: 395 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -265,6 +266,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscription:REGEX(.*)", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_first_notification.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_first_notification.test index f4ed342c3a..f1984d27d9 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_first_notification.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_first_notification.test @@ -80,7 +80,6 @@ payload='{ } }, "expires": "2028-12-31T10:00:00", - "throttling": 0, "@context": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld" }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" -H "Content-Type: application/ld+json" diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_geojson-format-in-notifications-with-keyValues.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_geojson-format-in-notifications-with-keyValues.test index ca9aacc09a..31d0d7a78b 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_geojson-format-in-notifications-with-keyValues.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_geojson-format-in-notifications-with-keyValues.test @@ -59,8 +59,7 @@ payload='{ "accept": "application/geo+json" } }, - "expires": "2028-12-31T10:00:00", - "throttling": 0 + "expires": "2028-12-31T10:00:00" }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" echo diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_geojson-format-in-notifications.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_geojson-format-in-notifications.test index ccd417f179..b6fe1d9c4e 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_geojson-format-in-notifications.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_geojson-format-in-notifications.test @@ -58,8 +58,7 @@ payload='{ "accept": "application/geo+json" } }, - "expires": "2028-12-31T10:00:00", - "throttling": 0 + "expires": "2028-12-31T10:00:00" }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" echo diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_issue_1248_q_in_subscription.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_issue_1248_q_in_subscription.test index 18dd781b6f..e25b9b8c25 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_issue_1248_q_in_subscription.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_issue_1248_q_in_subscription.test @@ -235,7 +235,7 @@ bye 03. GET the subscription and see the Q-filter ============================================= HTTP/1.1 200 OK -Content-Length: 404 +Content-Length: 486 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -259,7 +259,8 @@ Link: ; rel="http:/ }, "status": "ok" }, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -365,7 +366,7 @@ bye 09. GET the subscription and see the Q-filter ============================================= HTTP/1.1 200 OK -Content-Length: 414 +Content-Length: 496 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -389,7 +390,8 @@ Link: ; rel="http:/ }, "status": "ok" }, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification-in-geojson-format.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification-in-geojson-format.test index 2b5869d87d..e585445d54 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification-in-geojson-format.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification-in-geojson-format.test @@ -57,8 +57,7 @@ payload='{ "accept": "application/geo+json" } }, - "expires": "2028-12-31T10:00:00", - "throttling": 0 + "expires": "2028-12-31T10:00:00" }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" echo diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with-q-and-subcache-refresh.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with-q-and-subcache-refresh.test index a6ce35b33d..fba19067ce 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with-q-and-subcache-refresh.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with-q-and-subcache-refresh.test @@ -192,7 +192,7 @@ bye 03. GET the subscription - from the sub-cache ============================================= HTTP/1.1 200 OK -Content-Length: 273 +Content-Length: 351 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -205,6 +205,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -228,7 +229,7 @@ Slept 2.1 seconds 05. GET the subscription - from the sub-cache ============================================= HTTP/1.1 200 OK -Content-Length: 273 +Content-Length: 351 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -241,6 +242,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with-q-commalist-attr.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with-q-commalist-attr.test index 611cde1d0b..d0df3c93e9 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with-q-commalist-attr.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with-q-commalist-attr.test @@ -567,7 +567,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:entities:T:E8 21. GET the subscription S4 to see the 'q' ========================================== HTTP/1.1 200 OK -Content-Length: 473 +Content-Length: 551 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -580,6 +580,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S4", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_concise.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_concise.test index 1c54cb8f0a..b33100bd23 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_concise.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_concise.test @@ -59,7 +59,6 @@ payload='{ } }, "expires": "2028-12-31T10:00:00", - "throttling": 0, "@context": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld" }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" -H "Content-Type: application/ld+json" diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_https.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_https.test index 2c0296c7bb..13df469984 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_https.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_https.test @@ -60,7 +60,6 @@ payload='{ } }, "expires": "2028-12-31T10:00:00", - "throttling": 0, "@context": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld" }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" -H "Content-Type: application/ld+json" diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_keyValues.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_keyValues.test index ed8d28a342..5d431f9478 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_keyValues.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_keyValues.test @@ -60,7 +60,6 @@ payload='{ } }, "expires": "2028-12-31T10:00:00", - "throttling": 0, "@context": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld" }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" -H "Content-Type: application/ld+json" diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_simplified.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_simplified.test index 3586ce407f..9a96d005a7 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_simplified.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_notification_with_simplified.test @@ -59,7 +59,6 @@ payload='{ } }, "expires": "2028-12-31T10:00:00", - "throttling": 0, "@context": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld" }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" -H "Content-Type: application/ld+json" diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_notifications_issue-1244.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_notifications_issue-1244.test index f3cf407e2a..88309a3980 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_notifications_issue-1244.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_notifications_issue-1244.test @@ -50,7 +50,6 @@ payload='{ "type": "Subscription", "entities": [{"type": "Vehicle"}], "watchedAttributes": ["speed"], - "throttling": 0, "notification": { "format": "normalized", "attributes": ["speed", "brandName"], @@ -72,7 +71,6 @@ payload='{ "type": "Subscription", "entities": [{"type": "Vehicle"}], "watchedAttributes": ["speed"], - "throttling": 0, "notification": { "format": "normalized", "endpoint": { diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subCache.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subCache.test index b4e423e33b..dd0261669c 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subCache.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subCache.test @@ -90,7 +90,7 @@ Location: /ngsi-ld/v1/subscriptions/urn:ngsi-ld:subs:S1 03. GET S1 ========== HTTP/1.1 200 OK -Content-Length: 279 +Content-Length: 357 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -103,6 +103,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-cache-propagation.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-cache-propagation.test index cd4a4723f5..789c33c490 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-cache-propagation.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-cache-propagation.test @@ -140,7 +140,7 @@ Location: /ngsi-ld/v1/subscriptions/urn:ngsi-ld:subscriptions:S2 03. GET all subscriptions from CB - see S1 ========================================== HTTP/1.1 200 OK -Content-Length: 270 +Content-Length: 348 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -154,6 +154,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -172,7 +173,7 @@ Link: ; rel="http:/ 04. GET all subscriptions from CB2 - see S2 =========================================== HTTP/1.1 200 OK -Content-Length: 270 +Content-Length: 348 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -186,6 +187,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -209,7 +211,7 @@ Slept 2 seconds 06. GET all subscriptions from CB - see S1 and S2 ================================================= HTTP/1.1 200 OK -Content-Length: 539 +Content-Length: 695 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -223,6 +225,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -243,6 +246,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -261,7 +265,7 @@ Link: ; rel="http:/ 07. GET all subscriptions from CB2 - see S2 and S1 ================================================== HTTP/1.1 200 OK -Content-Length: 539 +Content-Length: 695 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -275,6 +279,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -295,6 +300,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-create-error-handling.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-create-error-handling.test index 907caffb54..098cbe1304 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-create-error-handling.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-create-error-handling.test @@ -81,6 +81,7 @@ orionldStart CB -experimental # # 15.1. Attempt to create a subscription with a THROTTLING that is not a Number # 15.2. Attempt to create a subscription with two THROTTLINGs +# 15.3. Attempt to create a subscription with a THROTTLING == 0 # # 16.1. Creation of a subscription with a STATUS field, just to make sure it is ignored # 16.2. Creation of a subscription with a CREATEDAT field, just to make sure it is ignored @@ -544,15 +545,15 @@ Date: REGEX(.*) 19.2. Attempt to create a subscription with both TIMEINTERVAL and WATCHEDATTRIBUTES present =========================================================================================== -HTTP/1.1 501 Not Implemented -Content-Length: 137 +HTTP/1.1 400 Bad Request +Content-Length: 164 Content-Type: application/json Date: REGEX(.*) { - "detail": "Time-Interval for Subscriptions", - "title": "Not Implemented", - "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" + "detail": "Both 'timeInterval' and 'watchedAttributes' are present", + "title": "Inconsistent subscription", + "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-get.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-get.test index 6e325f62aa..77cc307f13 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-get.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-get.test @@ -179,7 +179,7 @@ bye 03. GET the subscription - from DB (options=fromDb) =================================================== HTTP/1.1 200 OK -Content-Length: 1040 +Content-Length: 1122 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -235,7 +235,8 @@ Link: ; rel="http:/ "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, "createdAt": "202REGEX(.*)Z", - "modifiedAt": "202REGEX(.*)Z" + "modifiedAt": "202REGEX(.*)Z", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -243,7 +244,7 @@ Link: ; rel="http:/ 04. GET the subscription - from sub-cache (default since Summer '22) ==================================================================== HTTP/1.1 200 OK -Content-Length: 950 +Content-Length: 1032 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -297,7 +298,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-list.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-list.test index 7796cc7246..df46ccf8b1 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-list.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-list.test @@ -780,7 +780,7 @@ NGSILD-Tenant: t1 03. GET subscriptions - see the first 20 ======================================== HTTP/1.1 200 OK -Content-Length: 13321 +Content-Length: 15121 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -813,6 +813,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0001", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -863,6 +864,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0002", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -913,6 +915,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0003", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -963,6 +966,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0004", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1013,6 +1017,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0005", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1063,6 +1068,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0006", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1113,6 +1119,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0007", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1163,6 +1170,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0008", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1213,6 +1221,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0009", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1263,6 +1272,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0010", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1313,6 +1323,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0011", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1363,6 +1374,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0012", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1413,6 +1425,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0013", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1463,6 +1476,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0014", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1513,6 +1527,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0015", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1563,6 +1578,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0016", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1613,6 +1629,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0017", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1663,6 +1680,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0018", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1713,6 +1731,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0019", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1763,6 +1782,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0020", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1792,7 +1812,7 @@ Link: ; 04. GET subscriptions with limit 5 - see the first 5 ==================================================== HTTP/1.1 200 OK -Content-Length: 3331 +Content-Length: 3781 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1825,6 +1845,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0001", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1875,6 +1896,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0002", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1925,6 +1947,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0003", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -1975,6 +1998,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0004", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2025,6 +2049,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0005", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2054,7 +2079,7 @@ Link: ; 05. GET subscriptions with limit 5 and offset 25, see subs 25-29 ================================================================ HTTP/1.1 200 OK -Content-Length: 3331 +Content-Length: 3781 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -2087,6 +2112,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0026", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2137,6 +2163,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0027", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2187,6 +2214,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0028", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2237,6 +2265,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0029", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2287,6 +2316,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0030", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2327,7 +2357,7 @@ NGSILD-Results-Count: 50 07. GET subscriptions count with limit 1 - see HTTP header (NGSILD-Results-Count) and ONE (sub0001) subscription in the array ============================================================================================================================= HTTP/1.1 200 OK -Content-Length: 860 +Content-Length: 950 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -2362,6 +2392,7 @@ NGSILD-Results-Count: 50 }, "id": "urn:ngsi-ld:Subscription:sub0001", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "modifiedAt": "202REGEX(.*)", "notification": { "attributes": [ @@ -2392,7 +2423,7 @@ NGSILD-Results-Count: 50 08. GET subscriptions from tenant t1, with offset 36 - see 4 subs ================================================================= HTTP/1.1 200 OK -Content-Length: 2665 +Content-Length: 3025 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -2426,6 +2457,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0037", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2476,6 +2508,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0038", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2526,6 +2559,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0039", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2576,6 +2610,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0040", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2605,7 +2640,7 @@ NGSILD-Results-Count: 40 09. GET subscriptions from tenant t1, with offset 36 AND with Accept: application/ld+json - see 4 subs ====================================================================================================== HTTP/1.1 200 OK -Content-Length: 3005 +Content-Length: 3365 Content-Type: application/ld+json Date: REGEX(.*) NGSILD-Results-Count: 40 @@ -2639,6 +2674,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0037", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2690,6 +2726,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0038", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2741,6 +2778,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0039", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2792,6 +2830,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0040", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2821,7 +2860,7 @@ NGSILD-Results-Count: 40 10. GET subscriptions from DB - see the first 20 ================================================ HTTP/1.1 200 OK -Content-Length: 13381 +Content-Length: 15181 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -2854,6 +2893,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0001", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2904,6 +2944,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0002", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -2954,6 +2995,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0003", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3004,6 +3046,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0004", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3054,6 +3097,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0005", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3104,6 +3148,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0006", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3154,6 +3199,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0007", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3204,6 +3250,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0008", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3254,6 +3301,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0009", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3304,6 +3352,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0010", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3354,6 +3403,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0011", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3404,6 +3454,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0012", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3454,6 +3505,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0013", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3504,6 +3556,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0014", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3554,6 +3607,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0015", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3604,6 +3658,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0016", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3654,6 +3709,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0017", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3704,6 +3760,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0018", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3754,6 +3811,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0019", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3804,6 +3862,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0020", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3833,7 +3892,7 @@ Link: ; 11. GET subscriptions from DB - with limit 5 - see the first 5 ============================================================== HTTP/1.1 200 OK -Content-Length: 3346 +Content-Length: 3796 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -3866,6 +3925,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0001", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3916,6 +3976,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0002", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -3966,6 +4027,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0003", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4016,6 +4078,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0004", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4066,6 +4129,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0005", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4095,7 +4159,7 @@ Link: ; 12. GET subscriptions from DB - with limit 5 and offset 25, see subs 25-29 ========================================================================== HTTP/1.1 200 OK -Content-Length: 3346 +Content-Length: 3796 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -4128,6 +4192,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0026", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4178,6 +4243,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0027", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4228,6 +4294,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0028", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4278,6 +4345,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0029", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4328,6 +4396,7 @@ Link: ; }, "id": "urn:ngsi-ld:Subscription:sub0030", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4368,7 +4437,7 @@ NGSILD-Results-Count: 50 14. GET subscriptions from DB - count with limit 1 - see HTTP header (NGSILD-Results-Count) and ONE (sub0001) subscription in the array ======================================================================================================================================= HTTP/1.1 200 OK -Content-Length: 863 +Content-Length: 953 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -4403,6 +4472,7 @@ NGSILD-Results-Count: 50 }, "id": "urn:ngsi-ld:Subscription:sub0001", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "modifiedAt": "202REGEX(.*)", "notification": { "attributes": [ @@ -4433,7 +4503,7 @@ NGSILD-Results-Count: 50 15. GET subscriptions from DB - from tenant t1, with offset 36 - see 4 subs =========================================================================== HTTP/1.1 200 OK -Content-Length: 2677 +Content-Length: 3037 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -4467,6 +4537,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0037", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4517,6 +4588,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0038", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4567,6 +4639,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0039", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4617,6 +4690,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0040", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4646,7 +4720,7 @@ NGSILD-Results-Count: 40 16. GET subscriptions from DB - from tenant t1, with offset 36 AND with Accept: application/ld+json - see 4 subs ================================================================================================================ HTTP/1.1 200 OK -Content-Length: 3017 +Content-Length: 3377 Content-Type: application/ld+json Date: REGEX(.*) NGSILD-Results-Count: 40 @@ -4680,6 +4754,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0037", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4731,6 +4806,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0038", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4782,6 +4858,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0039", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", @@ -4833,6 +4910,7 @@ NGSILD-Results-Count: 40 }, "id": "urn:ngsi-ld:Subscription:sub0040", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "notification": { "attributes": [ "P1", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-with-mqtt-notification.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-with-mqtt-notification.test index c3b60ad54e..1610230166 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-with-mqtt-notification.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-with-mqtt-notification.test @@ -259,7 +259,7 @@ bye 03. GET the subscription ======================== HTTP/1.1 200 OK -Content-Length: 487 +Content-Length: 584 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -275,6 +275,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:Subscription:mqttNotification", "isActive": true, + "jsonldContext": "http://REGEX(.*)", "notification": { "endpoint": { "accept": "application/json", @@ -361,7 +362,7 @@ Notifications: 1 07. GET the subscription, without context => AirQualityObserved as longname =========================================================================== HTTP/1.1 200 OK -Content-Length: 487 +Content-Length: 557 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -377,6 +378,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:Subscription:mqttNotification", "isActive": true, + "jsonldContext": "https://fiware.github.io/data-models/context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -410,7 +412,7 @@ Link: ; rel="http:/ 07. GET the subscription, with context => AirQualityObserved as shortname ========================================================================= HTTP/1.1 200 OK -Content-Length: 449 +Content-Length: 519 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -426,6 +428,7 @@ Link: ; rel="http://www.w3. ], "id": "urn:ngsi-ld:Subscription:mqttNotification", "isActive": true, + "jsonldContext": "https://fiware.github.io/data-models/context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_and_q_issue_1180.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_and_q_issue_1180.test index 7d2dd5a908..87a6874968 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_and_q_issue_1180.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_and_q_issue_1180.test @@ -156,7 +156,7 @@ bye 03. GET S1 - see q with %22 =========================== HTTP/1.1 200 OK -Content-Length: 290 +Content-Length: 368 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -169,6 +169,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -187,7 +188,7 @@ Link: ; rel="http:/ 04. GET all subscriptions - see S1:q with %22 ============================================= HTTP/1.1 200 OK -Content-Length: 292 +Content-Length: 370 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -201,6 +202,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test index 6c2145890d..d6d0489b90 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test @@ -252,7 +252,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E1 03. GET S1, see lastSuccess, lastNotification, and timesSent==1 =============================================================== HTTP/1.1 200 OK -Content-Length: 368 +Content-Length: 446 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -265,6 +265,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -352,7 +353,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E4 08. GET S1, see lastSuccess, lastNotification, and timesSent==4 =============================================================== HTTP/1.1 200 OK -Content-Length: 368 +Content-Length: 446 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -365,6 +366,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -398,7 +400,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E5 11. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==5 ============================================================================ HTTP/1.1 200 OK -Content-Length: 498 +Content-Length: 576 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -411,6 +413,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "consecutiveErrors": 1, "endpoint": { @@ -482,7 +485,7 @@ bye 14. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==5 ============================================================================ HTTP/1.1 200 OK -Content-Length: 409 +Content-Length: 487 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -495,6 +498,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -539,7 +543,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E7 18. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==7 ============================================================================ HTTP/1.1 200 OK -Content-Length: 472 +Content-Length: 550 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -552,6 +556,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-https-notifications.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-https-notifications.test index c693bffade..26d3ab25cc 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-https-notifications.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-https-notifications.test @@ -260,7 +260,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E1 03. GET S1, see lastSuccess, lastNotification, and timesSent==1 =============================================================== HTTP/1.1 200 OK -Content-Length: 369 +Content-Length: 447 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -273,6 +273,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -360,7 +361,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E4 08. GET S1, see lastSuccess, lastNotification, and timesSent==4 =============================================================== HTTP/1.1 200 OK -Content-Length: 369 +Content-Length: 447 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -373,6 +374,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -406,7 +408,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E5 11. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==5 ============================================================================ HTTP/1.1 200 OK -Content-Length: 497 +Content-Length: 575 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -419,6 +421,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "consecutiveErrors": 1, "endpoint": { @@ -490,7 +493,7 @@ bye 14. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==5 ============================================================================ HTTP/1.1 200 OK -Content-Length: 410 +Content-Length: 488 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -503,6 +506,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -547,7 +551,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E7 18. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==7 ============================================================================ HTTP/1.1 200 OK -Content-Length: 471 +Content-Length: 549 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -560,6 +564,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-mqtt-notifications.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-mqtt-notifications.test index 066b06bf48..b2d030c31a 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-mqtt-notifications.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-mqtt-notifications.test @@ -223,7 +223,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E1 03. GET S1, see lastSuccess, lastNotification, and timesSent==1 =============================================================== HTTP/1.1 200 OK -Content-Length: 370 +Content-Length: 448 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -236,6 +236,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -323,7 +324,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E4 08. GET S1, see lastSuccess, lastNotification, and timesSent==4 =============================================================== HTTP/1.1 200 OK -Content-Length: 370 +Content-Length: 448 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -336,6 +337,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -365,7 +367,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E5 11. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==5 ============================================================================ HTTP/1.1 200 OK -Content-Length: 370 +Content-Length: 448 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -378,6 +380,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -445,7 +448,7 @@ bye 14. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==5 ============================================================================ HTTP/1.1 200 OK -Content-Length: 370 +Content-Length: 448 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -458,6 +461,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -487,7 +491,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E6 18. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==7 ============================================================================ HTTP/1.1 200 OK -Content-Length: 370 +Content-Length: 448 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -500,6 +504,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscriptions:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_create.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_create.test index f3c757b20a..71f55e40ae 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_create.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_create.test @@ -239,13 +239,6 @@ payload='{ "watchedAttributes": [ "P2" ], "timeInterval": 12, "q": "P2>10", - "geoQ": { - "geometry": "circle", - "coordinates": "[1,2]", - "georel": "near", - "geoproperty": "not supported" - }, - "csf": "not implemented", "isActive": false, "notification": { "attributes": [ "P1", "P2", "A3" ], @@ -458,7 +451,7 @@ bye 03. GET /ngsi-ld/v1/subscription/http://a.b.c/subs/sub01, see @context in HTTP header ===================================================================================== HTTP/1.1 200 OK -Content-Length: 1260 +Content-Length: 1354 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -526,7 +519,8 @@ Link: ; "throttling": 5, "createdAt": "202REGEX(.*)", "modifiedAt": "202REGEX(.*)", - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld" } @@ -534,7 +528,7 @@ Link: ; 04. GET /ngsi-ld/v1/subscription/http://a.b.c/subs/sub01, accepting jsonld - see @context in payload ==================================================================================================== HTTP/1.1 200 OK -Content-Length: 1262 +Content-Length: 1356 Content-Type: application/ld+json Date: REGEX(.*) @@ -600,6 +594,7 @@ Date: REGEX(.*) "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, "origin": "cache", + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", "@context": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld" } @@ -635,15 +630,15 @@ Date: REGEX(.*) 07. Attempt to create a subscription with both 'watchedAttributes' and 'timeInterval' - see failure =================================================================================================== -HTTP/1.1 501 Not Implemented -Content-Length: 137 +HTTP/1.1 400 Bad Request +Content-Length: 164 Content-Type: application/json Date: REGEX(.*) { - "detail": "Time-Interval for Subscriptions", - "title": "Not Implemented", - "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" + "detail": "Both 'timeInterval' and 'watchedAttributes' are present", + "title": "Inconsistent subscription", + "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" } @@ -668,7 +663,7 @@ Location: /ngsi-ld/v1/subscriptions/http://a.b.c/subs/sub09 10. GET sub09 with sysAttrs and make sure no 'expires' nor 'throttling' is returned =================================================================================== HTTP/1.1 200 OK -Content-Length: 485 +Content-Length: 563 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -684,6 +679,7 @@ Link: ; ], "id": "http://a.b.c/subs/sub09", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "modifiedAt": REGEX(.*), "notification": { "attributes": [ diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_delete.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_delete.test index 9692cb51c8..bdd63b3be3 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_delete.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_delete.test @@ -151,7 +151,7 @@ Location: /ngsi-ld/v1/subscriptions/http://a.b.c/subs/sub01 02. GET subscription 'http://a.b.c/subs/sub01' and see it ========================================================= HTTP/1.1 200 OK -Content-Length: 789 +Content-Length: 883 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -196,7 +196,8 @@ Link: ; }, "expiresAt": "2028-12-31T10:00:00.123Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_geoq.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_geoq.test index 51a1551a0d..d6876ff28d 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_geoq.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_geoq.test @@ -173,7 +173,7 @@ Location: /ngsi-ld/v1/subscriptions/urn:S1 02. GET S1 from cache, see the "geoq" fields ============================================ HTTP/1.1 200 OK -Content-Length: 358 +Content-Length: 436 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -195,6 +195,7 @@ Link: ; rel="http:/ }, "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -261,7 +262,7 @@ Date: REGEX(.*) 05. GET S1 from cache, see the new "geoq" ========================================= HTTP/1.1 200 OK -Content-Length: 358 +Content-Length: 436 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -280,6 +281,7 @@ Link: ; rel="http:/ }, "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -298,7 +300,7 @@ Link: ; rel="http:/ 06. GET S1 from DB, see the new "geoq" ====================================== HTTP/1.1 200 OK -Content-Length: 361 +Content-Length: 439 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -320,6 +322,7 @@ Link: ; rel="http:/ }, "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -386,7 +389,7 @@ Date: REGEX(.*) 09. GET S1 from cache, see the new "geoq" AND "q" ================================================= HTTP/1.1 200 OK -Content-Length: 365 +Content-Length: 443 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -405,6 +408,7 @@ Link: ; rel="http:/ }, "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -423,7 +427,7 @@ Link: ; rel="http:/ 10. GET S1 from DB, see the new "geoq" AND "q" ============================================== HTTP/1.1 200 OK -Content-Length: 368 +Content-Length: 446 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -445,6 +449,7 @@ Link: ; rel="http:/ }, "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_issue_1374.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_issue_1374.test index 4a1f4c90cf..8d7ed3e5b4 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_issue_1374.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_issue_1374.test @@ -241,7 +241,7 @@ Date: REGEX(.*) 05. GET S1 from cache, see "description" == "Sub S1" ==================================================== HTTP/1.1 200 OK -Content-Length: 312 +Content-Length: 390 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -255,6 +255,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -273,7 +274,7 @@ Link: ; rel="http:/ 06. GET S2 from cache, see "description" == "Sub S2" and the new value for "entities" ===================================================================================== HTTP/1.1 200 OK -Content-Length: 354 +Content-Length: 432 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -292,6 +293,7 @@ Link: ; rel="http:/ ], "id": "urn:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -310,7 +312,7 @@ Link: ; rel="http:/ 07. GET S1 from DB, see "description" == "Sub S1" ================================================= HTTP/1.1 200 OK -Content-Length: 315 +Content-Length: 393 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -324,6 +326,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -342,7 +345,7 @@ Link: ; rel="http:/ 08. GET S2 from DV, see "description" == "Sub S2"and the new value for "entities" ================================================================================= HTTP/1.1 200 OK -Content-Length: 357 +Content-Length: 435 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -361,6 +364,7 @@ Link: ; rel="http:/ ], "id": "urn:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -386,7 +390,7 @@ Date: REGEX(.*) 10. GET S1 to see the geoQ ========================== HTTP/1.1 200 OK -Content-Length: 413 +Content-Length: 491 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -406,6 +410,7 @@ Link: ; rel="http:/ }, "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -431,7 +436,7 @@ Date: REGEX(.*) 12. GET GET S2 from cache, see the new "notification" ===================================================== HTTP/1.1 200 OK -Content-Length: 380 +Content-Length: 458 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -450,6 +455,7 @@ Link: ; rel="http:/ ], "id": "urn:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "attributes": [ "R1", @@ -473,7 +479,7 @@ Link: ; rel="http:/ 13. GET GET S2 from DB, see the new "notification" ================================================== HTTP/1.1 200 OK -Content-Length: 383 +Content-Length: 461 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -492,6 +498,7 @@ Link: ; rel="http:/ ], "id": "urn:S2", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "attributes": [ "R1", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_q.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_q.test index bb75321868..f29863630b 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_q.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_q.test @@ -156,7 +156,7 @@ Location: /ngsi-ld/v1/subscriptions/urn:S1 02. GET S1 from cache, see the "q" ================================== HTTP/1.1 200 OK -Content-Length: 258 +Content-Length: 336 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -169,6 +169,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -235,7 +236,7 @@ Date: REGEX(.*) 05. GET S1 from cache, see the new "q" ====================================== HTTP/1.1 200 OK -Content-Length: 258 +Content-Length: 336 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -248,6 +249,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -266,7 +268,7 @@ Link: ; rel="http:/ 06. GET S1 from DB, see the new "q" =================================== HTTP/1.1 200 OK -Content-Length: 261 +Content-Length: 339 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -279,6 +281,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -345,7 +348,7 @@ Date: REGEX(.*) 09. GET S1 from cache, see the new "q" ====================================== HTTP/1.1 200 OK -Content-Length: 265 +Content-Length: 343 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -358,6 +361,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -376,7 +380,7 @@ Link: ; rel="http:/ 10. GET S1 from DB, see the new "q" =================================== HTTP/1.1 200 OK -Content-Length: 268 +Content-Length: 346 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -389,6 +393,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_two_subs-issue_1368.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_two_subs-issue_1368.test index 2c5ff159be..ddb760083b 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_two_subs-issue_1368.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_two_subs-issue_1368.test @@ -432,7 +432,7 @@ bye 08. GET all subscriptions, see updates ====================================== HTTP/1.1 200 OK -Content-Length: 820 +Content-Length: 976 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -447,6 +447,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscription:REGEX(.*)", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "attributes": [ "humidity" @@ -474,6 +475,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscription:REGEX(.*)", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "attributes": [ "temperature" diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_update_all_individual_fields.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_update_all_individual_fields.test index fcfa4761a8..785208b2c0 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_update_all_individual_fields.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_update_all_individual_fields.test @@ -578,7 +578,7 @@ bye 03. GET subscription from the broker ==================================== HTTP/1.1 200 OK -Content-Length: 831 +Content-Length: 913 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -624,7 +624,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -639,7 +640,7 @@ Date: REGEX(.*) 05. GET subscription and see the new 'subscriptionName' ======================================================= HTTP/1.1 200 OK -Content-Length: 818 +Content-Length: 900 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -685,7 +686,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -700,7 +702,7 @@ Date: REGEX(.*) 07. GET subscription and see the new 'description' ================================================== HTTP/1.1 200 OK -Content-Length: 797 +Content-Length: 879 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -746,7 +748,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -817,7 +820,7 @@ bye 10. GET subscription and see the new 'entities' =============================================== HTTP/1.1 200 OK -Content-Length: 798 +Content-Length: 880 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -865,7 +868,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -936,7 +940,7 @@ bye 13. GET subscription and see the new 'watchedAttributes' ======================================================== HTTP/1.1 200 OK -Content-Length: 798 +Content-Length: 880 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -984,22 +988,23 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } 14. PATCH 'timeInterval' and see error ====================================== -HTTP/1.1 501 Not Implemented -Content-Length: 137 +HTTP/1.1 400 Bad Request +Content-Length: 176 Content-Type: application/json Date: REGEX(.*) { - "detail": "Time-Interval for Subscriptions", - "title": "Not Implemented", - "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" + "detail": "urn:ngsi-ld:subscriptions:S01", + "title": "Invalid modification (on-change to timeInterval subscription)", + "type": "https://uri.etsi.org/ngsi-ld/errors/ResourceNotFound" } @@ -1069,7 +1074,7 @@ bye 17. GET subscription and see the new 'q' ======================================== HTTP/1.1 200 OK -Content-Length: 807 +Content-Length: 889 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1117,7 +1122,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -1188,7 +1194,7 @@ bye 20. GET subscription and see the new 'geoQ' =========================================== HTTP/1.1 200 OK -Content-Length: 786 +Content-Length: 868 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1233,7 +1239,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -1304,7 +1311,7 @@ bye 25. GET subscription and see the new 'isActive' =============================================== HTTP/1.1 200 OK -Content-Length: 787 +Content-Length: 869 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1349,7 +1356,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -1420,7 +1428,7 @@ bye 28. GET subscription and see the new 'notification' =================================================== HTTP/1.1 200 OK -Content-Length: 786 +Content-Length: 868 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1465,7 +1473,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:00:00.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -1480,7 +1489,7 @@ Date: REGEX(.*) 30. GET subscription and see the new 'expiresAt' ================================================ HTTP/1.1 200 OK -Content-Length: 786 +Content-Length: 868 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1525,7 +1534,8 @@ Link: ; rel="http:/ }, "expiresAt": "2026-04-01T12:34:56.000Z", "throttling": 5, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -1540,7 +1550,7 @@ Date: REGEX(.*) 32. GET subscription and see the new 'throttling' ================================================= HTTP/1.1 200 OK -Content-Length: 787 +Content-Length: 869 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1585,7 +1595,8 @@ Link: ; rel="http:/ }, "expiresAt": "2026-04-01T12:34:56.000Z", "throttling": 12, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -1656,7 +1667,7 @@ Date: REGEX(.*) 35. GET subscription and see all the fields =========================================== HTTP/1.1 200 OK -Content-Length: 852 +Content-Length: 934 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1703,7 +1714,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:34:34.000Z", "throttling": 34, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -1833,7 +1845,7 @@ bye 39. GET subscription and see A=1 in receiverInfo ================================================ HTTP/1.1 200 OK -Content-Length: 950 +Content-Length: 1032 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1886,7 +1898,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:34:34.000Z", "throttling": 34, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } @@ -1901,7 +1914,7 @@ Date: REGEX(.*) 41. GET subscription and see A=2 in receiverInfo ================================================ HTTP/1.1 200 OK -Content-Length: 950 +Content-Length: 1032 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -1954,7 +1967,8 @@ Link: ; rel="http:/ }, "expiresAt": "2028-12-31T10:34:34.000Z", "throttling": 34, - "origin": "cache" + "origin": "cache", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_timestamps.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_timestamps.test index 0c62f3e592..1c183035f7 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_timestamps.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_timestamps.test @@ -341,7 +341,7 @@ bye 03. GET the subscription via REST - make sure all timestamps have milliseconds, save timestamps =============================================================================================== HTTP/1.1 200 OK -Content-Length: 498 +Content-Length: 576 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -357,6 +357,7 @@ Link: ; rel="http:/ "expiresAt": "2028-12-31T11:12:13.456Z", "id": "urn:ngsi-ld:subscriptions:s1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "modifiedAt": "REGEX(.*\.\d\d\dZ)", "notification": { "endpoint": { @@ -377,7 +378,7 @@ Link: ; rel="http:/ 03b. GET the subscription via REST from DB - make sure all timestamps have milliseconds, save timestamps ======================================================================================================== HTTP/1.1 200 OK -Content-Length: 501 +Content-Length: 579 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -393,6 +394,7 @@ Link: ; rel="http:/ "expiresAt": "2028-12-31T11:12:13.456Z", "id": "urn:ngsi-ld:subscriptions:s1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "modifiedAt": "REGEX(.*\.\d\d\dZ)", "notification": { "endpoint": { @@ -459,7 +461,7 @@ Date: REGEX(.*) 07. GET the subscription via REST - see 'timesSent==2' and timestamps for lastSuccess, lastNotification ======================================================================================================= HTTP/1.1 200 OK -Content-Length: 599 +Content-Length: 677 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -475,6 +477,7 @@ Link: ; rel="http:/ "expiresAt": "2028-12-31T11:12:13.456Z", "id": "urn:ngsi-ld:subscriptions:s1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "modifiedAt": "REGEX(.*\.\d\d\dZ)", "notification": { "endpoint": { @@ -501,7 +504,7 @@ timesSent == 2 - good! 07b. GET the subscription via REST from DB - see 'timesSent==2' and timestamps for lastSuccess, lastNotification ================================================================================================================ HTTP/1.1 200 OK -Content-Length: 602 +Content-Length: 680 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -517,6 +520,7 @@ Link: ; rel="http:/ "expiresAt": "2028-12-31T11:12:13.456Z", "id": "urn:ngsi-ld:subscriptions:s1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "modifiedAt": "202REGEX(.*)Z", "notification": { "endpoint": { @@ -550,7 +554,7 @@ Date: REGEX(.*) 09. GET the subscription via REST - make sure 'modifiedAt' has changed and 'createdAt' has not ============================================================================================== HTTP/1.1 200 OK -Content-Length: 612 +Content-Length: 690 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -566,6 +570,7 @@ Link: ; rel="http:/ "expiresAt": "2028-12-31T11:12:13.456Z", "id": "urn:ngsi-ld:subscriptions:s1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "modifiedAt": "REGEX(.*\.\d\d\dZ)", "notification": { "endpoint": { @@ -604,7 +609,7 @@ Date: REGEX(.*) 12. GET the subscription via REST - make sure lastFailure is present and has milliseconds ========================================================================================= HTTP/1.1 200 OK -Content-Length: 742 +Content-Length: 820 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -620,6 +625,7 @@ Link: ; rel="http:/ "expiresAt": "2028-12-31T11:12:13.456Z", "id": "urn:ngsi-ld:subscriptions:s1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "modifiedAt": "REGEX(.*\.\d\d\dZ)", "notification": { "consecutiveErrors": 1, diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_with_mqtt_issue-1178-II.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_with_mqtt_issue-1178-II.test index 47c71a0242..bf355c2903 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_with_mqtt_issue-1178-II.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_with_mqtt_issue-1178-II.test @@ -198,7 +198,7 @@ bye 03. GET the subscription ======================== HTTP/1.1 200 OK -Content-Length: 539 +Content-Length: 617 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -213,6 +213,7 @@ Link: ; rel="http://www.w3. ], "id": "urn:ngsi-ld:subscription:REGEX(.*)", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "attributes": [ "Engine_Oxigen", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_notification-stats.test b/test/functionalTest/cases/0000_ngsild/ngsild_notification-stats.test index 37f3c8bdc1..30894af311 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_notification-stats.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_notification-stats.test @@ -297,7 +297,7 @@ Link: ; rel="http:/ 08. GET S1, see one success (lastSuccess + timesSent: 1) ======================================================== HTTP/1.1 200 OK -Content-Length: 465 +Content-Length: 543 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -312,6 +312,7 @@ Link: ; rel="http:/ ], "id": "urn:S1:https:ok", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -333,7 +334,7 @@ Link: ; rel="http:/ 09. GET S2, see one success (lastSuccess + timesSent: 1) ======================================================== HTTP/1.1 200 OK -Content-Length: 462 +Content-Length: 540 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -348,6 +349,7 @@ Link: ; rel="http:/ ], "id": "urn:S2:http:ok", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -369,7 +371,7 @@ Link: ; rel="http:/ 10. GET S3, see one failure (lastFailure + timesSent: 1, consecutiveErrors: 1) ============================================================================== HTTP/1.1 200 OK -Content-Length: 547 +Content-Length: 625 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -384,6 +386,7 @@ Link: ; rel="http:/ ], "id": "urn:S3:https:fails", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "consecutiveErrors": 1, "endpoint": { @@ -407,7 +410,7 @@ Link: ; rel="http:/ 11. GET S4, see one failure (lastFailure + timesSent: 1, consecutiveErrors: 1) ============================================================================== HTTP/1.1 200 OK -Content-Length: 553 +Content-Length: 631 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -422,6 +425,7 @@ Link: ; rel="http:/ ], "id": "urn:S4:http:fails", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "consecutiveErrors": 1, "endpoint": { diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_notification_issue-1322.test b/test/functionalTest/cases/0000_ngsild/ngsild_notification_issue-1322.test index 001da2c6f1..4222adefbb 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_notification_issue-1322.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_notification_issue-1322.test @@ -124,7 +124,7 @@ NGSILD-Tenant: captn 03. GET subscription - see failure ================================== HTTP/1.1 200 OK -Content-Length: 748 +Content-Length: 826 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -139,6 +139,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscription:8f6f80a4-a7a4-11ed-9cbf-16804aa0063b", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "attributes": [ "location", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_patch_entity2-notification-counters.test b/test/functionalTest/cases/0000_ngsild/ngsild_patch_entity2-notification-counters.test index e2f55e4865..e6ced61623 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_patch_entity2-notification-counters.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_patch_entity2-notification-counters.test @@ -222,7 +222,7 @@ HTTP/1.1 204 No Content 07. GET S1 - see lastSuccess==now, lastFailure==never, errors==0, timesSent==10, status==active =============================================================================================== HTTP/1.1 200 OK -Content-Length: 361 +Content-Length: 439 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -236,6 +236,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -256,7 +257,7 @@ Link: ; rel="http:/ 08. GET S2 - see lastSuccess==never, lastFailure==now, timesSent==4, status==paused =================================================================================== HTTP/1.1 200 OK -Content-Length: 450 +Content-Length: 528 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -270,6 +271,7 @@ Link: ; rel="http:/ ], "id": "urn:S2", "isActive": false, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "consecutiveErrors": 3, "endpoint": { diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test index 1ab6bff6b2..f5b97cac6d 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test @@ -25,22 +25,94 @@ Pernot Subscription - periodic notifications --SHELL-INIT-- dbInit CB -orionldStart CB +orionldStart CB -experimental --SHELL-- # -# 01. Create a subscription urn:S1 with timeInterval -# 02. See urn:S1 in the database -# 03. GET urn:S1 from cache -# 04. GET urn:S1 from database -# 05. Restart broker -# 06. GET urn:S1 from cache -# 07. GET urn:S1 from database +# 01. Attempt to create a subscription with timeInterval AND watchedAttributes - see 400 +# 02. Attempt to create a subscription with timeInterval AND notificationTrigger - see 400 +# 03. Attempt to create a subscription with timeInterval as a String - see 400 +# 05. Attempt to create a subscription with duplicated timeInterval - see 400 +# 06. Create a subscription urn:S1 with timeInterval +# 07. See urn:S1 in the database +# 08. GET urn:S1 from cache +# 09. GET urn:S1 from database +# 10. Restart broker +# 11. GET urn:S1 from cache +# 12. GET urn:S1 from database # -echo "01. Create a subscription urn:S1 with timeInterval" -echo "==================================================" +echo "01. Attempt to create a subscription with timeInterval AND watchedAttributes - see 400" +echo "======================================================================================" +payload='{ + "id": "urn:S1", + "type": "Subscription", + "entities": [ + { + "type": "T" + } + ], + "watchedAttributes": [ "a", "b" ], + "timeInterval": 2, + "notification": { + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "02. Attempt to create a subscription with timeInterval AND notificationTrigger - see 400" +echo "========================================================================================" +payload='{ + "id": "urn:S1", + "type": "Subscription", + "entities": [ + { + "type": "T" + } + ], + "notificationTrigger": [ "entityCreated,", "attributeDeleted" ], + "timeInterval": 2, + "notification": { + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "03. Attempt to create a subscription with timeInterval as a String - see 400" +echo "============================================================================" +payload='{ + "id": "urn:S1", + "type": "Subscription", + "entities": [ + { + "type": "T" + } + ], + "timeInterval": "2", + "notification": { + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "05. Attempt to create a subscription with duplicated timeInterval - see 400" +echo "===========================================================================" payload='{ "id": "urn:S1", "type": "Subscription", @@ -54,6 +126,38 @@ payload='{ "endpoint": { "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" } + }, + "timeInterval": 5 +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "06. Create a subscription urn:S1 with timeInterval" +echo "==================================================" +payload='{ + "id": "urn:S1", + "type": "Subscription", + "description": "Test Subscription No 1", + "subscriptionName": "S1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + }, + { + "idPattern": "urn:E.*", + "type": "T" + } + ], + "q": "(a>5&b<9)|a<4", + "timeInterval": 2, + "notification": { + "attributes": [ "P1", "P2" ], + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } } }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" @@ -61,14 +165,234 @@ echo echo -echo "02. See urn:S1 in the database" +echo "07. See urn:S1 in the database" echo "==============================" mongoCmd2 ftest "db.csubs.findOne()" echo echo +echo "08. GET urn:S1 from cache" +echo "=========================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1 +echo +echo + + +echo "09. GET urn:S1 from database" +echo "============================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1?options=fromDb +echo +echo + + --REGEXPECT-- +01. Attempt to create a subscription with timeInterval AND watchedAttributes - see 400 +====================================================================================== +HTTP/1.1 400 Bad Request +Content-Length: 164 +Content-Type: application/json +Date: REGEX(.*) + +{ + "detail": "Both 'timeInterval' and 'watchedAttributes' are present", + "title": "Inconsistent subscription", + "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" +} + + +02. Attempt to create a subscription with timeInterval AND notificationTrigger - see 400 +======================================================================================== +HTTP/1.1 501 Not Implemented +Content-Length: 125 +Content-Type: application/json +Date: REGEX(.*) + +{ + "detail": "notificationTrigger", + "title": "Not Implemented", + "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" +} + + +03. Attempt to create a subscription with timeInterval as a String - see 400 +============================================================================ +HTTP/1.1 400 Bad Request +Content-Length: 127 +Content-Type: application/json +Date: REGEX(.*) + +{ + "detail": "Subscription::timeInterval", + "title": "Not a JSON Number", + "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" +} + + +05. Attempt to create a subscription with duplicated timeInterval - see 400 +=========================================================================== +HTTP/1.1 400 Bad Request +Content-Length: 126 +Content-Type: application/json +Date: REGEX(.*) + +{ + "detail": "Subscription::timeInterval", + "title": "Duplicated field", + "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" +} + + +06. Create a subscription urn:S1 with timeInterval +================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:S1 + + + +07. See urn:S1 in the database +============================== +MongoDB shell version REGEX(.*) +connecting to: REGEX(.*) +MongoDB server version: REGEX(.*) +{ + "_id" : "urn:S1", + "description" : "Test Subscription No 1", + "name" : "S1", + "entities" : [ + { + "id" : "urn:E1", + "type" : "https://uri.etsi.org/ngsi-ld/default-context/T", + "isPattern" : "false", + "isTypePattern" : false + }, + { + "id" : "urn:E.*", + "type" : "https://uri.etsi.org/ngsi-ld/default-context/T", + "isPattern" : "true", + "isTypePattern" : false + } + ], + "ldQ" : "(https://uri=etsi=org/ngsi-ld/default-context/a.value>5;https://uri=etsi=org/ngsi-ld/default-context/b.value<9)|https://uri=etsi=org/ngsi-ld/default-context/a.value<4", + "timeInterval" : 2, + "createdAt" : REGEX(.*), + "modifiedAt" : REGEX(.*), + "throttling" : 0, + "expression" : { + "geometry" : "", + "coords" : "", + "georel" : "", + "geoproperty" : "", + "q" : "P;!P", + "mq" : "P.P;!P.P" + }, + "attrs" : [ + "https://uri.etsi.org/ngsi-ld/default-context/P1", + "https://uri.etsi.org/ngsi-ld/default-context/P2" + ], + "reference" : "http://REGEX(.*)/notify", + "mimeType" : "application/json", + "format" : "normalized", + "conditions" : [ ], + "status" : "active", + "custom" : false, + "servicePath" : "/#", + "blacklist" : false, + "ldContext" : "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" +} +bye + + +08. GET urn:S1 from cache +========================= +HTTP/1.1 200 OK +Content-Length: 497 +Content-Type: application/json +Date: REGEX(.*) +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + +{ + "description": "Test Subscription No 1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + }, + { + "idPattern": "urn:E.*", + "type": "T" + } + ], + "id": "urn:S1", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "status": "ok" + }, + "origin": "cache", + "q": "(a>5;b<9)|a<4", + "status": "active", + "subscriptionName": "S1", + "timeInterval": 2, + "type": "Subscription" +} + + +09. GET urn:S1 from database +============================ +HTTP/1.1 200 OK +Content-Length: 500 +Content-Type: application/json +Date: REGEX(.*) +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + +{ + "description": "Test Subscription No 1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + }, + { + "idPattern": "urn:E.*", + "type": "T" + } + ], + "id": "urn:S1", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "status": "ok" + }, + "origin": "database", + "q": "(a>5;b<9)|a<4", + "status": "active", + "subscriptionName": "S1", + "timeInterval": 2, + "type": "Subscription" +} + + --TEARDOWN-- brokerStop CB dbDrop CB diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test b/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test index 910c3fc053..ecbaa07574 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test @@ -334,7 +334,7 @@ Ngsild-Attribute-Format: Normalized 06. GET the subscription, from cache, to see showChanges=true ============================================================= HTTP/1.1 200 OK -Content-Length: 397 +Content-Length: 475 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -348,6 +348,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -369,7 +370,7 @@ Link: ; rel="http:/ 07. GET the subscription, from DB, to see showChanges=true ========================================================== HTTP/1.1 200 OK -Content-Length: 400 +Content-Length: 478 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -383,6 +384,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -408,7 +410,7 @@ Link: ; rel="http:/ 09. GET the subscription, from cache, to see showChanges=true ============================================================= HTTP/1.1 200 OK -Content-Length: 296 +Content-Length: 374 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -422,6 +424,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -440,7 +443,7 @@ Link: ; rel="http:/ 10. GET the subscription, from DB, to see showChanges=true ========================================================== HTTP/1.1 200 OK -Content-Length: 299 +Content-Length: 377 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -454,6 +457,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -558,7 +562,7 @@ bye 15. GET the subscription, from cache, to see showChanges=false ============================================================== HTTP/1.1 200 OK -Content-Length: 378 +Content-Length: 456 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -572,6 +576,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -592,7 +597,7 @@ Link: ; rel="http:/ 16. GET the subscription, from DB, to see showChanges=false =========================================================== HTTP/1.1 200 OK -Content-Length: 381 +Content-Length: 459 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -606,6 +611,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -674,7 +680,7 @@ bye 19. GET the subscription, from cache, to see showChanges=true ============================================================= HTTP/1.1 200 OK -Content-Length: 397 +Content-Length: 475 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -688,6 +694,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -709,7 +716,7 @@ Link: ; rel="http:/ 20. GET the subscription, from DB, to see showChanges=true ========================================================== HTTP/1.1 200 OK -Content-Length: 400 +Content-Length: 478 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -723,6 +730,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subCache-counters-II.test b/test/functionalTest/cases/0000_ngsild/ngsild_subCache-counters-II.test index cecca2b6fb..484d37d35f 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subCache-counters-II.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subCache-counters-II.test @@ -175,7 +175,7 @@ Date: REGEX(.*) 04. GET S1 - see timesSent == 6 =============================== HTTP/1.1 200 OK -Content-Length: 359 +Content-Length: 437 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -188,6 +188,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -213,7 +214,7 @@ slept 5 06. GET S1 - see timesSent == 6 =============================== HTTP/1.1 200 OK -Content-Length: 359 +Content-Length: 437 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -226,6 +227,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -265,7 +267,7 @@ Date: REGEX(.*) 08. GET S1 - see timesSent == 11 ================================ HTTP/1.1 200 OK -Content-Length: 360 +Content-Length: 438 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -278,6 +280,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subs:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subCache-with-mongoc-on-startup.test b/test/functionalTest/cases/0000_ngsild/ngsild_subCache-with-mongoc-on-startup.test index b5bdf65a6a..b38fe148db 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subCache-with-mongoc-on-startup.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subCache-with-mongoc-on-startup.test @@ -237,7 +237,7 @@ bye 03. GET the subscription from cache, without context - see long names ===================================================================== HTTP/1.1 200 OK -Content-Length: 933 +Content-Length: 937 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -261,6 +261,7 @@ Link: ; rel="http:/ }, "id": "urn:ngsi-ld:subs:S1", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testFullContext.jsonld", "notification": { "attributes": [ "http://example.org/P1", @@ -295,7 +296,7 @@ Link: ; rel="http:/ "status": "ok" }, "origin": "cache", - "q": "(https://uri.etsi.org/ngsi-ld/default-context/temp<18|temp>25);(https://uri.etsi.org/ngsi-ld/default-context/humidity<20|humidity>80)", + "q": "(temp<18|temp>25);(humidity<20|humidity>80)", "status": "paused", "subscriptionName": "S1", "throttling": 5, @@ -309,7 +310,7 @@ Link: ; rel="http:/ 04. GET the subscription from DB, without context - see long names ================================================================== HTTP/1.1 200 OK -Content-Length: 937 +Content-Length: 941 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -333,6 +334,7 @@ Link: ; rel="http:/ }, "id": "urn:ngsi-ld:subs:S1", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testFullContext.jsonld", "notification": { "attributes": [ "http://example.org/P1", @@ -367,7 +369,7 @@ Link: ; rel="http:/ "status": "ok" }, "origin": "database", - "q": "(https://uri.etsi.org/ngsi-ld/default-context/temp<18|temp>25);(https://uri.etsi.org/ngsi-ld/default-context/humidity<20|humidity>80)", + "q": "(temp<18|temp>25);(humidity<20|humidity>80)", "status": "paused", "subscriptionName": "S1", "throttling": 5, @@ -381,7 +383,7 @@ Link: ; rel="http:/ 05. GET the subscription from cache, with testFullContext - see short names =========================================================================== HTTP/1.1 200 OK -Content-Length: 857 +Content-Length: 861 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -405,6 +407,7 @@ Link: 25);(https://uri.etsi.org/ngsi-ld/default-context/humidity<20|humidity>80)", + "q": "(temp<18|temp>25);(humidity<20|humidity>80)", "status": "paused", "subscriptionName": "S1", "throttling": 5, @@ -453,7 +456,7 @@ Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -477,6 +480,7 @@ Link: 25);(https://uri.etsi.org/ngsi-ld/default-context/humidity<20|humidity>80)", + "q": "(temp<18|temp>25);(humidity<20|humidity>80)", "status": "paused", "subscriptionName": "S1", "throttling": 5, @@ -588,7 +592,7 @@ bye 09. GET the subscription from cache =================================== HTTP/1.1 200 OK -Content-Length: 933 +Content-Length: 937 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -612,6 +616,7 @@ Link: ; rel="http:/ }, "id": "urn:ngsi-ld:subs:S1", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testFullContext.jsonld", "notification": { "attributes": [ "http://example.org/P1", @@ -646,7 +651,7 @@ Link: ; rel="http:/ "status": "ok" }, "origin": "cache", - "q": "(https://uri.etsi.org/ngsi-ld/default-context/temp<18|temp>25);(https://uri.etsi.org/ngsi-ld/default-context/humidity<20|humidity>80)", + "q": "(temp<18|temp>25);(humidity<20|humidity>80)", "status": "paused", "subscriptionName": "S1", "throttling": 5, @@ -660,7 +665,7 @@ Link: ; rel="http:/ 10. GET the subscription from DB ================================ HTTP/1.1 200 OK -Content-Length: 937 +Content-Length: 941 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -684,6 +689,7 @@ Link: ; rel="http:/ }, "id": "urn:ngsi-ld:subs:S1", "isActive": false, + "jsonldContext": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testFullContext.jsonld", "notification": { "attributes": [ "http://example.org/P1", @@ -718,7 +724,7 @@ Link: ; rel="http:/ "status": "ok" }, "origin": "database", - "q": "(https://uri.etsi.org/ngsi-ld/default-context/temp<18|temp>25);(https://uri.etsi.org/ngsi-ld/default-context/humidity<20|humidity>80)", + "q": "(temp<18|temp>25);(humidity<20|humidity>80)", "status": "paused", "subscriptionName": "S1", "throttling": 5, diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue-1316.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue-1316.test index e9387e1058..802bb92aa2 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue-1316.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue-1316.test @@ -103,7 +103,7 @@ NGSILD-Tenant: t_02 02. Retrieve S1 =============== HTTP/1.1 200 OK -Content-Length: 395 +Content-Length: 473 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -117,6 +117,7 @@ Link: ; rel="http:/ ], "id": "urn:subscr:123457", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "attributes": [ "status" @@ -145,7 +146,7 @@ Link: ; rel="http:/ 04. Retrieve S1 (using tenant t_02) =================================== HTTP/1.1 200 OK -Content-Length: 395 +Content-Length: 473 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -159,6 +160,7 @@ Link: ; rel="http:/ ], "id": "urn:subscr:123457", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "attributes": [ "status" diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_create.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_create.test index 39b500ac2f..f6856767eb 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_create.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_create.test @@ -628,14 +628,14 @@ Date: REGEX(.*) 07. Attempt to create a subscription with both 'watchedAttributes' and 'timeInterval' - see failure =================================================================================================== -HTTP/1.1 501 Not Implemented -Content-Length: 144 +HTTP/1.1 400 Bad Request +Content-Length: 160 Content-Type: application/json Date: REGEX(.*) { - "detail": "Subscription::timeInterval is not implemented", - "title": "Not Implemented", + "detail": "Both 'timeInterval' and 'watchedAttributes' present", + "title": "Inconsistent subscription", "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_creation_errors.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_creation_errors.test index 09d547832e..797a4e4378 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_creation_errors.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_creation_errors.test @@ -553,14 +553,14 @@ Date: REGEX(.*) 19.2. Attempt to create a subscription with both TIMEINTERVAL and WATCHEDATTRIBUTES present =========================================================================================== -HTTP/1.1 501 Not Implemented -Content-Length: 144 +HTTP/1.1 400 Bad Request +Content-Length: 160 Content-Type: application/json Date: REGEX(.*) { - "detail": "Subscription::timeInterval is not implemented", - "title": "Not Implemented", + "detail": "Both 'timeInterval' and 'watchedAttributes' present", + "title": "Inconsistent subscription", "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_issue_1179-with-experimental.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_issue_1179-with-experimental.test index 7763c4be5e..dc9e90bf8b 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_issue_1179-with-experimental.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_issue_1179-with-experimental.test @@ -346,7 +346,7 @@ NGSILD-Tenant: captn 07. GET subscriptions ===================== HTTP/1.1 200 OK -Content-Length: 2123 +Content-Length: 2372 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -362,6 +362,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscription:REGEX(.*)", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.5.jsonld", "notification": { "attributes": [ "location", @@ -405,6 +406,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscription:REGEX(.*)", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.5.jsonld", "notification": { "attributes": [ "location", @@ -445,6 +447,7 @@ Link: ; rel="http:/ ], "id": "urn:ngsi-ld:subscription:REGEX(.*)", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.5.jsonld", "notification": { "attributes": [ "location", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_patch_update_all_individual_fields.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_patch_update_all_individual_fields.test index b94a5e7ebf..962aa25298 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_patch_update_all_individual_fields.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_patch_update_all_individual_fields.test @@ -995,15 +995,15 @@ Link: ; rel="http:/ 14. PATCH 'timeInterval' and see error ====================================== -HTTP/1.1 501 Not Implemented -Content-Length: 137 +HTTP/1.1 400 Bad Request +Content-Length: 176 Content-Type: application/json Date: REGEX(.*) { - "detail": "Time-Interval for Subscriptions", - "title": "Not Implemented", - "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" + "detail": "urn:ngsi-ld:subscriptions:S01", + "title": "Invalid modification (on-change to timeInterval subscription)", + "type": "https://uri.etsi.org/ngsi-ld/errors/ResourceNotFound" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_with_sysAttrs.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_with_sysAttrs.test index 48aae74e57..a817e8d048 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_with_sysAttrs.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_with_sysAttrs.test @@ -313,7 +313,7 @@ Location: /ngsi-ld/v1/subscriptions/urn:S1 04. GET the subscription and see the sysAttrs=true ================================================== HTTP/1.1 200 OK -Content-Length: 261 +Content-Length: 339 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -326,6 +326,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", @@ -344,7 +345,7 @@ Link: ; rel="http:/ 05. GET the subscription from DB and see the sysAttrs=true ========================================================== HTTP/1.1 200 OK -Content-Length: 264 +Content-Length: 342 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -357,6 +358,7 @@ Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", "notification": { "endpoint": { "accept": "application/json", From 93994c0017f818eaaf797dc9f4dba1bebbfe0523 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sat, 22 Jul 2023 18:57:04 +0200 Subject: [PATCH 04/29] PATCH of a Pernot Subscription: 501 --- .../orionldPatchSubscription.cpp | 44 ++++++++++++++++--- ...on_patch_update_all_individual_fields.test | 4 +- .../cases/0000_ngsild/ngsild_pernot.test | 38 +++++++++++++--- ...on_patch_update_all_individual_fields.test | 4 +- 4 files changed, 74 insertions(+), 16 deletions(-) diff --git a/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp b/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp index f60b45b395..b8288ffdea 100644 --- a/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp +++ b/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp @@ -862,17 +862,50 @@ bool orionldPatchSubscription(void) return false; } + KjNode* dbTimeIntervalP = kjLookup(dbSubscriptionP, "timeInterval"); + double dbTimeInterval = 0; + + if (dbTimeIntervalP != NULL) + dbTimeInterval = (dbTimeIntervalP->type == KjInt)? dbTimeIntervalP->value.i : dbTimeIntervalP->value.f; + + bool subWasPernot = (dbTimeInterval != 0); + // // If the subscription used to be "on-change", timeInterval cannot be set // - if ((timeInterval != 0) && (kjLookup(dbSubscriptionP, "timeInterval") == NULL)) + if (subWasPernot == false) { - if (qNodeP != NULL) - qRelease(qNodeP); - orionldError(OrionldResourceNotFound, "Invalid modification (on-change to timeInterval subscription)", subscriptionId, 400); + if (timeInterval != 0) + { + if (qNodeP != NULL) + qRelease(qNodeP); + orionldError(OrionldBadRequestData, "Invalid modification (on-change to timeInterval subscription)", subscriptionId, 400); + return false; + } + } + else // If the subscription used to be "pernot", watchedAttributes+throtttling cannot be set + { +#if 0 + // + // These checks really belong to pCheckSubscription() - just need to pass it the dbTimeInterval value + // + KjNode* watchedAttributesP = kjLookup(subTree, "watchedAttributes"); + KjNode* throttlingP = kjLookup(subTree, "throttling"); + + if (watchedAttributesP != NULL) + orionldError(OrionldBadRequestData, "Invalid modification (pernot subscription cannot have watchedAttributes", subscriptionId, 400); + if (throttlingP != NULL) + orionldError(OrionldBadRequestData, "Invalid modification (pernot subscription cannot have throttlingP", subscriptionId, 400); + + if ((watchedAttributesP != NULL) || (throttlingP != NULL)) + return false; +#else + orionldError(OrionldOperationNotSupported, "Not Implemented", "Patching of periodic notification subscriptions", 501); return false; +#endif } + // // If the subscription used to be an MQTT subscription, the MQTT connection might need closing // Only if MQTT data is modified though (or it stops being an MQTT subscription) @@ -927,8 +960,7 @@ bool orionldPatchSubscription(void) cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subscriptionId); else { - // What do I do here? - // Create a copy of CachedSubscription with only the fields needed by ngsildSubscriptionPatch? + // Can't get here right now } if (ngsildSubscriptionPatch(dbSubscriptionP, cSubP, orionldState.requestTree, qP, geoqP, qRenderedForDb) == false) diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_update_all_individual_fields.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_update_all_individual_fields.test index 785208b2c0..a36a94d418 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_update_all_individual_fields.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_update_all_individual_fields.test @@ -997,14 +997,14 @@ Link: ; rel="http:/ 14. PATCH 'timeInterval' and see error ====================================== HTTP/1.1 400 Bad Request -Content-Length: 176 +Content-Length: 174 Content-Type: application/json Date: REGEX(.*) { "detail": "urn:ngsi-ld:subscriptions:S01", "title": "Invalid modification (on-change to timeInterval subscription)", - "type": "https://uri.etsi.org/ngsi-ld/errors/ResourceNotFound" + "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test index f5b97cac6d..caf88a8917 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test @@ -33,8 +33,9 @@ orionldStart CB -experimental # 01. Attempt to create a subscription with timeInterval AND watchedAttributes - see 400 # 02. Attempt to create a subscription with timeInterval AND notificationTrigger - see 400 # 03. Attempt to create a subscription with timeInterval as a String - see 400 -# 05. Attempt to create a subscription with duplicated timeInterval - see 400 -# 06. Create a subscription urn:S1 with timeInterval +# 04. Attempt to create a subscription with duplicated timeInterval - see 400 +# 05. Create a subscription urn:S1 with timeInterval +# 06. Attempt to PATCH a pernot subscription # 07. See urn:S1 in the database # 08. GET urn:S1 from cache # 09. GET urn:S1 from database @@ -111,7 +112,7 @@ echo echo -echo "05. Attempt to create a subscription with duplicated timeInterval - see 400" +echo "04. Attempt to create a subscription with duplicated timeInterval - see 400" echo "===========================================================================" payload='{ "id": "urn:S1", @@ -134,7 +135,7 @@ echo echo -echo "06. Create a subscription urn:S1 with timeInterval" +echo "05. Create a subscription urn:S1 with timeInterval" echo "==================================================" payload='{ "id": "urn:S1", @@ -165,6 +166,17 @@ echo echo + +echo "06. Attempt to PATCH a pernot subscription" +echo "==========================================" +payload='{ + "description": "Test Subscription No 2" +}' +orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1 --payload "$payload" -X PATCH +echo +echo + + echo "07. See urn:S1 in the database" echo "==============================" mongoCmd2 ftest "db.csubs.findOne()" @@ -229,7 +241,7 @@ Date: REGEX(.*) } -05. Attempt to create a subscription with duplicated timeInterval - see 400 +04. Attempt to create a subscription with duplicated timeInterval - see 400 =========================================================================== HTTP/1.1 400 Bad Request Content-Length: 126 @@ -243,7 +255,7 @@ Date: REGEX(.*) } -06. Create a subscription urn:S1 with timeInterval +05. Create a subscription urn:S1 with timeInterval ================================================== HTTP/1.1 201 Created Content-Length: 0 @@ -252,6 +264,20 @@ Location: /ngsi-ld/v1/subscriptions/urn:S1 +06. Attempt to PATCH a pernot subscription +========================================== +HTTP/1.1 501 Not Implemented +Content-Length: 153 +Content-Type: application/json +Date: REGEX(.*) + +{ + "detail": "Patching of periodic notification subscriptions", + "title": "Not Implemented", + "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" +} + + 07. See urn:S1 in the database ============================== MongoDB shell version REGEX(.*) diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_patch_update_all_individual_fields.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_patch_update_all_individual_fields.test index 962aa25298..5daf2c6e42 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_patch_update_all_individual_fields.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_patch_update_all_individual_fields.test @@ -996,14 +996,14 @@ Link: ; rel="http:/ 14. PATCH 'timeInterval' and see error ====================================== HTTP/1.1 400 Bad Request -Content-Length: 176 +Content-Length: 174 Content-Type: application/json Date: REGEX(.*) { "detail": "urn:ngsi-ld:subscriptions:S01", "title": "Invalid modification (on-change to timeInterval subscription)", - "type": "https://uri.etsi.org/ngsi-ld/errors/ResourceNotFound" + "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" } From 406745e57709757c4e56cd72834ce378a5f7d70a Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Mon, 24 Jul 2023 14:56:36 +0200 Subject: [PATCH 05/29] Fixed the pernot loop and made pernot optional (off by default) --- src/app/orionld/orionld.cpp | 10 ++- src/lib/orionld/pernot/pernotLoop.cpp | 59 +++++++++++------ src/lib/orionld/pernot/pernotTreat.cpp | 63 ++++++++++++++++++- src/lib/orionld/pernot/pernotTreat.h | 4 +- .../cases/0000_ngsild/ngsild_pernot.test | 19 ++++-- 5 files changed, 125 insertions(+), 30 deletions(-) diff --git a/src/app/orionld/orionld.cpp b/src/app/orionld/orionld.cpp index 519a6496c8..16094cfff4 100644 --- a/src/app/orionld/orionld.cpp +++ b/src/app/orionld/orionld.cpp @@ -237,6 +237,7 @@ bool ngsiv1Autocast; int contextDownloadAttempts; int contextDownloadTimeout; bool troe; +bool pernot; bool disableFileLog; bool lmtmp; char troeHost[256]; @@ -317,6 +318,7 @@ int cSubCounters; #define INSECURE_NOTIF "allow HTTPS notifications to peers which certificate cannot be authenticated with known CA certificates" #define NGSIV1_AUTOCAST "automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations" #define TROE_DESC "enable TRoE - temporal representation of entities" +#define PERNOT_DESC "enable Pernot - Periodic Notifications" #define DISABLE_FILE_LOG "disable logging into file" #define TMPTRACES_DESC "disable traces" #define TROE_HOST_DESC "host for troe database db server" @@ -420,6 +422,7 @@ PaArgument paArgs[] = { "-ctxTimeout", &contextDownloadTimeout, "CONTEXT_DOWNLOAD_TIMEOUT", PaInt, PaOpt, 5000, 0, 20000, CTX_TMO_DESC }, { "-ctxAttempts", &contextDownloadAttempts, "CONTEXT_DOWNLOAD_ATTEMPTS", PaInt, PaOpt, 3, 0, 100, CTX_ATT_DESC }, { "-troe", &troe, "TROE", PaBool, PaOpt, false, false, true, TROE_DESC }, + { "-pernot", &pernot, "PERNOT", PaBool, PaOpt, false, false, true, PERNOT_DESC }, { "-lmtmp", &lmtmp, "TMP_TRACES", PaBool, PaHid, true, false, true, TMPTRACES_DESC }, { "-socketService", &socketService, "SOCKET_SERVICE", PaBool, PaHid, false, false, true, SOCKET_SERVICE_DESC }, { "-troeHost", troeHost, "TROE_HOST", PaString, PaOpt, _i "localhost", PaNL, PaNL, TROE_HOST_DESC }, @@ -1166,7 +1169,9 @@ int main(int argC, char* argV[]) // Note that regCacheInit uses the tenantList, so orionldTenantInit must be called before regCacheInit // regCacheInit(); - pernotSubCacheInit(); + + if (pernot == true) + pernotSubCacheInit(); orionldServiceInit(restServiceVV, 9); @@ -1319,7 +1324,8 @@ int main(int argC, char* argV[]) // Start the thread for periodic notifications - pernotLoopStart(); + if (pernot == true) + pernotLoopStart(); if (socketService == true) { diff --git a/src/lib/orionld/pernot/pernotLoop.cpp b/src/lib/orionld/pernot/pernotLoop.cpp index fb7bdffa69..73f59e6fc7 100644 --- a/src/lib/orionld/pernot/pernotLoop.cpp +++ b/src/lib/orionld/pernot/pernotLoop.cpp @@ -25,6 +25,12 @@ #include // strerror #include // gettimeofday #include // pthread_create +#include // usleep + +extern "C" +{ +#include "kbase/kMacros.h" // K_MIN +} #include "logMsg/logMsg.h" // LM_x @@ -51,7 +57,7 @@ bool pernotSubInsert(void) // // currentTime - // -static double currentTime(void) +double currentTime(void) { // int gettimeofday(struct timeval *tv, struct timezone *tz); struct timeval tv; @@ -59,7 +65,7 @@ static double currentTime(void) if (gettimeofday(&tv, NULL) != 0) LM_RE(0, ("gettimeofday error: %s", strerror(errno))); - return tv.tv_sec + tv.tv_usec / 1000000; + return tv.tv_sec + tv.tv_usec / 1000000.0; } @@ -72,18 +78,28 @@ static void* pernotLoop(void* vP) { while (1) { + double minDiff; + + minDiff = 1; // Initialize with 1 second, which is the MAX sleep period + for (PernotSubscription* subP = pernotSubCache.head; subP != NULL; subP = subP->next) { double now = currentTime(); if (subP->state == SubPaused) + { + LM_T(LmtPernot, ("%s: Paused", subP->subscriptionId)); continue; + } - if (subP->expiresAt <= now) + if ((subP->expiresAt > 0) && (subP->expiresAt <= now)) subP->state = SubExpired; // Should it be removed? if (subP->state == SubExpired) + { + LM_T(LmtPernot, ("%s: Expired", subP->subscriptionId)); continue; + } if (subP->state == SubErroneous) { @@ -91,30 +107,36 @@ static void* pernotLoop(void* vP) if (subP->lastFailureTime + subP->cooldown < now) subP->state = SubActive; else + { + LM_T(LmtPernot, ("%s: Erroneous", subP->subscriptionId)); continue; + } } - if (subP->lastNotificationAttempt + subP->timeInterval < now) + double diffTime = subP->lastNotificationAttempt + subP->timeInterval - now; + // LM_T(LmtPernot, ("%s: lastNotificationAttempt: %f", subP->subscriptionId, subP->lastNotificationAttempt)); + // LM_T(LmtPernot, ("%s: timeInterval: %f", subP->subscriptionId, subP->timeInterval)); + // LM_T(LmtPernot, ("%s: now: %f", subP->subscriptionId, now)); + // LM_T(LmtPernot, ("%s: diffTime: %f", subP->subscriptionId, diffTime)); + + if (diffTime <= 0) { subP->lastNotificationAttempt = now; // Either it works or fails, the timestamp needs to be updated - if (pernotTreat(subP) == true) // Just send the notification, don't await any response - { - subP->lastSuccessTime = now; - subP->consecutiveErrors = 0; - } - else - { - subP->lastFailureTime = now; - subP->consecutiveErrors += 1; + pernotTreatStart(subP); // Query runs in a new thread - if (subP->consecutiveErrors >= 3) - { - subP->state = SubErroneous; - } - } + // Restart the min diff time + minDiff = 0; + break; } + else + minDiff = K_MIN(minDiff, diffTime); } + + if (minDiff != 0) + usleep(minDiff * 1000000); } + + return NULL; } @@ -126,5 +148,6 @@ pthread_t pernotThreadID; // void pernotLoopStart(void) { + LM_T(LmtPernot, ("Starting thread for the Periodic Notification Loop")); pthread_create(&pernotThreadID, NULL, pernotLoop, NULL); } diff --git a/src/lib/orionld/pernot/pernotTreat.cpp b/src/lib/orionld/pernot/pernotTreat.cpp index a126923dab..f584e57591 100644 --- a/src/lib/orionld/pernot/pernotTreat.cpp +++ b/src/lib/orionld/pernot/pernotTreat.cpp @@ -22,16 +22,73 @@ * * Author: Ken Zangelin */ +#include // pthread_exit + +extern "C" +{ +#include "kjson/KjNode.h" // KjNode +} + +#include "logMsg/logMsg.h" // LM_x + +#include "orionld/common/orionldState.h" // orionldState, pernotSubCache #include "orionld/pernot/PernotSubscription.h" // PernotSubscription #include "orionld/pernot/pernotTreat.h" // Own interface - +extern double currentTime(void); // FIXME: Move to orionld/common // ----------------------------------------------------------------------------- // // pernotTreat - // -bool pernotTreat(PernotSubscription* subP) +static void* pernotTreat(void* vP) { - return false; + PernotSubscription* subP = (PernotSubscription*) vP; + bool ok = true; + double now = currentTime(); + + // Build the query + LM_T(LmtPernot, ("Creating the query for pernot-subscription %s", subP->subscriptionId)); + + // + // Timestamps and Status + // + if (ok == true) + { + subP->lastSuccessTime = now; + subP->consecutiveErrors = 0; + } + else + { + subP->lastFailureTime = now; + subP->notificationErrors += 1; + subP->consecutiveErrors += 1; + + if (subP->consecutiveErrors >= 3) + { + subP->state = SubErroneous; + LM_W(("%s: 3 consecutive errors - setting the subscription in Error state", subP->subscriptionId)); + } + } + + subP->notificationAttempts += 1; + + pthread_exit(0); + return NULL; +} + + + +// ----------------------------------------------------------------------------- +// +// pernotTreatStart - +// +void pernotTreatStart(PernotSubscription* subP) +{ + pthread_t tid; + + LM_T(LmtPernot, ("Starting thread for one Periodic Notification Subscription (%s)", subP->subscriptionId)); + pthread_create(&tid, NULL, pernotTreat, subP); + + // It's OK to lose the thread ID ... I think ... } diff --git a/src/lib/orionld/pernot/pernotTreat.h b/src/lib/orionld/pernot/pernotTreat.h index dc43fbe4f1..984c5d1c97 100644 --- a/src/lib/orionld/pernot/pernotTreat.h +++ b/src/lib/orionld/pernot/pernotTreat.h @@ -31,8 +31,8 @@ // ----------------------------------------------------------------------------- // -// pernotTreat - +// pernotTreatStart - // -extern bool pernotTreat(PernotSubscription* subP); +extern void pernotTreatStart(PernotSubscription* subP); #endif // SRC_LIB_ORIONLD_PERNOT_PERNOTTREAT_H_ diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test index caf88a8917..237818e3a6 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test @@ -25,7 +25,7 @@ Pernot Subscription - periodic notifications --SHELL-INIT-- dbInit CB -orionldStart CB -experimental +orionldStart CB -pernot -experimental --SHELL-- @@ -197,6 +197,9 @@ orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1?options=fromDb echo echo +# To see traces from the pernot loop in t he logfile +#sleep 4 + --REGEXPECT-- 01. Attempt to create a subscription with timeInterval AND watchedAttributes - see 400 @@ -334,7 +337,7 @@ bye 08. GET urn:S1 from cache ========================= HTTP/1.1 200 OK -Content-Length: 497 +Content-Length: 598 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -364,7 +367,10 @@ Link: ; rel="http:/ "uri": "http://127.0.0.1:9997/notify" }, "format": "normalized", - "status": "ok" + "lastNotification": "202REGEX(.*)Z", + "lastSuccess": "202REGEX(.*)Z", + "status": "ok", + "timesSent": 1 }, "origin": "cache", "q": "(a>5;b<9)|a<4", @@ -378,7 +384,7 @@ Link: ; rel="http:/ 09. GET urn:S1 from database ============================ HTTP/1.1 200 OK -Content-Length: 500 +Content-Length: 601 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -408,7 +414,10 @@ Link: ; rel="http:/ "uri": "http://127.0.0.1:9997/notify" }, "format": "normalized", - "status": "ok" + "lastNotification": "202REGEX(.*)Z", + "lastSuccess": "202REGEX(.*)Z", + "status": "ok", + "timesSent": 1 }, "origin": "database", "q": "(a>5;b<9)|a<4", From 21eeae5f95e964bfa6043b4c41987585d2a653a1 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sat, 26 Aug 2023 18:49:18 +0200 Subject: [PATCH 06/29] Merging with develop (v1.4.0) --- src/app/orionld/orionld.cpp | 5 +- src/lib/cache/CachedSubscription.h | 9 +- src/lib/cache/subCache.cpp | 14 +- src/lib/logMsg/traceLevels.h | 4 +- src/lib/mongoBackend/MongoCommonUpdate.cpp | 12 +- .../mongoBackend/mongoGetSubscriptions.cpp | 14 +- src/lib/mongoBackend/mongoRegistrationAux.cpp | 2 +- src/lib/mongoBackend/mongoSubCache.cpp | 20 +- .../mongoBackend/mongoUpdateSubscription.cpp | 12 +- src/lib/mongoBackend/safeMongo.cpp | 5 +- src/lib/mongoBackend/safeMongo.h | 3 +- src/lib/ngsi/ContextElementResponse.cpp | 8 +- .../orionld/common/orionldServerConnect.cpp | 2 +- src/lib/orionld/common/orionldServerConnect.h | 2 +- src/lib/orionld/common/orionldState.cpp | 1 - src/lib/orionld/common/orionldState.h | 4 +- .../dbModel/dbModelFromApiSubscription.cpp | 1 + .../orionld/dbModel/dbModelToApiEntity.cpp | 2 +- src/lib/orionld/dbModel/dbModelToApiEntity.h | 2 +- .../dbModel/dbModelToApiSubscription.cpp | 49 ++- .../kjTree/kjTreeFromPernotSubscription.cpp | 10 +- .../mongoBackend/mongoLdRegistrationAux.cpp | 12 +- .../orionld/mongoc/mongocEntitiesQuery.cpp | 2 + .../orionld/mongoc/mongocEntitiesQuery2.cpp | 3 + .../orionld/mongoc/mongocKjTreeFromBson.cpp | 2 +- .../mongoc/mongocSubCachePopulateByTenant.cpp | 6 +- .../mongoc/mongocSubCountersUpdate.cpp | 56 ++- .../orionld/mongoc/mongocSubCountersUpdate.h | 15 +- src/lib/orionld/notifications/httpNotify.cpp | 33 +- src/lib/orionld/notifications/httpNotify.h | 14 +- .../notifications/notificationFailure.cpp | 97 ++++- .../notifications/notificationFailure.h | 5 +- .../notifications/notificationSend.cpp | 50 ++- .../notifications/notificationSuccess.cpp | 10 +- src/lib/orionld/pernot/CMakeLists.txt | 1 + src/lib/orionld/pernot/PernotSubCache.cpp | 24 +- src/lib/orionld/pernot/PernotSubCache.h | 3 + src/lib/orionld/pernot/PernotSubscription.h | 44 +- src/lib/orionld/pernot/pernotLoop.cpp | 123 ++++-- src/lib/orionld/pernot/pernotSend.cpp | 237 +++++++++++ src/lib/orionld/pernot/pernotSend.h | 43 ++ src/lib/orionld/pernot/pernotSubCacheAdd.cpp | 162 +++++++- src/lib/orionld/pernot/pernotSubCacheAdd.h | 6 +- .../orionld/pernot/pernotSubCacheLookup.cpp | 2 +- src/lib/orionld/pernot/pernotTreat.cpp | 140 ++++++- src/lib/orionld/q/qAliasCompact.cpp | 4 +- .../orionldGetSubscription.cpp | 98 ++++- .../orionldPostSubscriptions.cpp | 8 +- ...ption_counters-for-http-notifications.test | 4 +- .../ngsild_new_subscription_timestamps.test | 3 +- .../ngsild_notification-stats.test | 6 +- .../ngsild_notification_issue-1322.test | 3 +- .../cases/0000_ngsild/ngsild_pernot.test | 215 +++++++++- ...ngsild_pernot_counters_and_timestamps.test | 392 ++++++++++++++++++ .../0000_ngsild/ngsild_pernot_noMatch.test | 146 +++++++ .../0000_ngsild/ngsild_subCache-counters.test | 2 +- .../1952_sub_notification_wo_attrs.test | 1 - 57 files changed, 1881 insertions(+), 272 deletions(-) create mode 100644 src/lib/orionld/pernot/pernotSend.cpp create mode 100644 src/lib/orionld/pernot/pernotSend.h create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_pernot_counters_and_timestamps.test create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_pernot_noMatch.test diff --git a/src/app/orionld/orionld.cpp b/src/app/orionld/orionld.cpp index 16094cfff4..34486199f1 100644 --- a/src/app/orionld/orionld.cpp +++ b/src/app/orionld/orionld.cpp @@ -209,6 +209,7 @@ char reqMutexPolicy[16]; int writeConcern; unsigned int cprForwardLimit; int subCacheInterval; +int subCacheFlushInterval; char notificationMode[64]; int notificationQueueSize; int notificationThreadNum; @@ -254,7 +255,7 @@ bool noswap; bool experimental = false; bool mongocOnly = false; bool debugCurl = false; -int cSubCounters; +uint32_t cSubCounters; @@ -295,6 +296,7 @@ int cSubCounters; #define WRITE_CONCERN_DESC "db write concern (0:unacknowledged, 1:acknowledged)" #define CPR_FORWARD_LIMIT_DESC "maximum number of distributed requests to Context Providers for a single client request" #define SUB_CACHE_IVAL_DESC "interval in seconds between calls to Subscription Cache refresh (0: no refresh)" +#define SUB_CACHE_FLUSH_IVAL_DESC "interval in seconds between calls to Pernot Subscription Cache Flush to DB (0: no flush)" #define NOTIFICATION_MODE_DESC "notification mode (persistent|transient|threadpool:q:n)" #define NO_CACHE "disable subscription cache for lookups" #define CONN_MEMORY_DESC "maximum memory size per connection (in kilobytes)" @@ -397,6 +399,7 @@ PaArgument paArgs[] = { "-corsMaxAge", &maxAge, "CORS_MAX_AGE", PaInt, PaOpt, 86400, -1, 86400, CORS_MAX_AGE_DESC }, { "-cprForwardLimit", &cprForwardLimit, "CPR_FORWARD_LIMIT", PaUInt, PaOpt, 1000, 0, UINT_MAX, CPR_FORWARD_LIMIT_DESC }, { "-subCacheIval", &subCacheInterval, "SUBCACHE_IVAL", PaInt, PaOpt, 0, 0, 3600, SUB_CACHE_IVAL_DESC }, + { "-subCacheFlushIval", &subCacheFlushInterval, "SUBCACHE_FLUSH_IVAL", PaInt, PaOpt, 10, 0, 3600, SUB_CACHE_FLUSH_IVAL_DESC }, { "-noCache", &noCache, "NOCACHE", PaBool, PaOpt, false, false, true, NO_CACHE }, { "-connectionMemory", &connectionMemory, "CONN_MEMORY", PaUInt, PaOpt, 64, 0, 1024, CONN_MEMORY_DESC }, { "-maxConnections", &maxConnections, "MAX_CONN", PaUInt, PaOpt, 1020, 1, PaNL, MAX_CONN_DESC }, diff --git a/src/lib/cache/CachedSubscription.h b/src/lib/cache/CachedSubscription.h index e0bc402f87..51d2053812 100644 --- a/src/lib/cache/CachedSubscription.h +++ b/src/lib/cache/CachedSubscription.h @@ -36,6 +36,7 @@ #include "orionld/context/OrionldContext.h" // OrionldContext #include "orionld/types/Protocol.h" // Protocol #include "orionld/types/OrionldAlteration.h" // OrionldAlterationTypes +#include "orionld/types/OrionldTenant.h" // OrionldTenant @@ -95,6 +96,7 @@ struct CachedSubscription std::vector metadata; std::vector notifyConditionV; char* tenant; + OrionldTenant* tenantP; char* servicePath; bool triggers[OrionldAlterationTypes]; double throttling; @@ -116,12 +118,15 @@ struct CachedSubscription std::string status; int64_t count; // delta count - since last sub cache refresh int64_t dbCount; // count taken from the database + int64_t failures; + int64_t dbFailures; + int consecutiveErrors; // Not in DB, no need + double lastNotificationTime; // timestamp of last notification attempt double lastFailure; // timestamp of last notification failure double lastSuccess; // timestamp of last successful notification - int consecutiveErrors; // Not in DB char lastErrorReason[128]; // Not in DB - int dirty; + uint32_t dirty; double createdAt; double modifiedAt; diff --git a/src/lib/cache/subCache.cpp b/src/lib/cache/subCache.cpp index 490ae9c77c..6655188703 100644 --- a/src/lib/cache/subCache.cpp +++ b/src/lib/cache/subCache.cpp @@ -51,6 +51,7 @@ extern "C" #include "orionld/types/OrionldTenant.h" // OrionldTenant #include "orionld/common/orionldState.h" // orionldState #include "orionld/common/tenantList.h" // tenantList +#include "orionld/common/orionldTenantLookup.h" // orionldTenantLookup #include "orionld/common/urlParse.h" // urlParse #include "orionld/q/qBuild.h" // qBuild #include "orionld/q/qRelease.h" // qRelease @@ -872,6 +873,7 @@ bool subCacheItemInsert // First the non-complex values // cSubP->tenant = (tenant[0] == 0)? NULL : strdup(tenant); + cSubP->tenantP = orionldTenantLookup(tenant); cSubP->expirationTime = expirationTime; cSubP->throttling = throttling; cSubP->lastNotificationTime = lastNotificationTime; @@ -1396,7 +1398,16 @@ void subCacheSync(void) if (cssP != NULL) { if (experimental == true) - mongocSubCountersUpdate(cSubP, cssP->count, cssP->lastNotificationTime, cssP->lastFailure, cssP->lastSuccess, false, cssP->ngsild); + mongocSubCountersUpdate(cSubP->tenantP, + cSubP->subscriptionId, + cssP->ngsild, + cssP->count, + 0, // failures - fix! + 0, // noMatch - only for PerNot + cssP->lastNotificationTime, + cssP->lastSuccess, + cssP->lastFailure, + false); else { std::string tenant = (cSubP->tenant == NULL)? "" : cSubP->tenant; // Use char* !!! @@ -1480,6 +1491,7 @@ void subCacheStart(void) LM_E(("Runtime Error (error creating thread: %d)", ret)); return; } + pthread_detach(tid); } diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index 6898d988e1..3113899cd5 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -62,6 +62,7 @@ typedef enum TraceLevels LmtSubCache = 50, // Subscription Cache LmtSubCacheMatch, // Subscription Cache Matches LmtSubCacheDebug, // Subscription Cache Debug + LmtSubCacheStats, // Subscription Cache Counters and Timestamps // // Registration Cache @@ -94,7 +95,8 @@ typedef enum TraceLevels // Pernot sub-cache // LmtPernot = 100, // Periodic Notification Subscription cache - + LmtPernotLoop, // Pernot loop, when each sub is triggered in time + LmtPernotFlush = 102, // Pernot flush to DB // // Misc // diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 50fbcbe85e..b7225eb744 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -688,7 +688,7 @@ static bool mergeAttrInfo(const BSONObj& attr, ContextAttribute* caP, BSONObj* m /* 4. Add creation date */ if (attr.hasField(ENT_ATTRS_CREATION_DATE)) { - ab.append(ENT_ATTRS_CREATION_DATE, getNumberFieldAsDoubleF(&attr, ENT_ATTRS_CREATION_DATE)); + ab.append(ENT_ATTRS_CREATION_DATE, getNumberFieldAsDoubleF(&attr, ENT_ATTRS_CREATION_DATE, false)); } /* Was it an actual update? */ @@ -774,7 +774,7 @@ static bool mergeAttrInfo(const BSONObj& attr, ContextAttribute* caP, BSONObj* m * in database by a CB instance previous to the support of creation and modification dates */ if (attr.hasField(ENT_ATTRS_MODIFICATION_DATE)) { - ab.append(ENT_ATTRS_MODIFICATION_DATE, getNumberFieldAsDoubleF(&attr, ENT_ATTRS_MODIFICATION_DATE)); + ab.append(ENT_ATTRS_MODIFICATION_DATE, getNumberFieldAsDoubleF(&attr, ENT_ATTRS_MODIFICATION_DATE, false)); } } @@ -1676,8 +1676,8 @@ static bool addTriggeredSubscriptions_noCache // // NOTE: renderFormatString: NGSIv1 JSON is 'default' (for old db-content) // - double throttling = getNumberFieldAsDoubleF(&sub, CSUB_THROTTLING); - double lastNotification = getNumberFieldAsDoubleF(&sub, CSUB_LASTNOTIFICATION); + double throttling = getNumberFieldAsDoubleF(&sub, CSUB_THROTTLING, true); + double lastNotification = getNumberFieldAsDoubleF(&sub, CSUB_LASTNOTIFICATION, true); const char* renderFmt = getStringFieldF(&sub, CSUB_FORMAT); const char* renderFormatString = (renderFmt[0] == 0)? "legacy" : renderFmt; RenderFormat renderFormat = stringToRenderFormat(renderFormatString); @@ -3538,8 +3538,8 @@ static void updateEntity // The hasField() check is needed as the entity could have been created with very old Orion version not // supporting modification/creation dates - notifyCerP->contextElement.entityId.creDate = getNumberFieldAsDoubleF(bobP, ENT_CREATION_DATE); - notifyCerP->contextElement.entityId.modDate = getNumberFieldAsDoubleF(bobP, ENT_MODIFICATION_DATE); + notifyCerP->contextElement.entityId.creDate = getNumberFieldAsDoubleF(bobP, ENT_CREATION_DATE, false); + notifyCerP->contextElement.entityId.modDate = getNumberFieldAsDoubleF(bobP, ENT_MODIFICATION_DATE, false); // The logic to detect notification loops is to check that the correlator in the request differs from the last one seen for the entity and, // in addition, the request was sent due to a custom notification diff --git a/src/lib/mongoBackend/mongoGetSubscriptions.cpp b/src/lib/mongoBackend/mongoGetSubscriptions.cpp index b14a0e1bd6..3e4920ede8 100644 --- a/src/lib/mongoBackend/mongoGetSubscriptions.cpp +++ b/src/lib/mongoBackend/mongoGetSubscriptions.cpp @@ -177,12 +177,12 @@ static void extractNotification(Subscription* subP, const BSONObj* rP, OrionldTe ngsiv2::Notification* nP = &subP->notification; - subP->throttling = rP->hasField(CSUB_THROTTLING)? getNumberFieldAsDoubleF(rP, CSUB_THROTTLING) : -1; - nP->lastNotification = rP->hasField(CSUB_LASTNOTIFICATION)? getNumberFieldAsDoubleF(rP, CSUB_LASTNOTIFICATION) : -1; - nP->timesSent = rP->hasField(CSUB_COUNT)? getIntOrLongFieldAsLongF(rP, CSUB_COUNT) : 0; - nP->blacklist = rP->hasField(CSUB_BLACKLIST)? getBoolFieldF(rP, CSUB_BLACKLIST) : false; - nP->lastFailure = rP->hasField(CSUB_LASTFAILURE)? getNumberFieldAsDoubleF(rP, CSUB_LASTFAILURE) : -1; - nP->lastSuccess = rP->hasField(CSUB_LASTSUCCESS)? getNumberFieldAsDoubleF(rP, CSUB_LASTSUCCESS) : -1; + subP->throttling = rP->hasField(CSUB_THROTTLING)? getNumberFieldAsDoubleF(rP, CSUB_THROTTLING, true) : -1; + nP->lastNotification = rP->hasField(CSUB_LASTNOTIFICATION)? getNumberFieldAsDoubleF(rP, CSUB_LASTNOTIFICATION, true) : -1; + nP->timesSent = rP->hasField(CSUB_COUNT)? getIntOrLongFieldAsLongF(rP, CSUB_COUNT) : 0; + nP->blacklist = rP->hasField(CSUB_BLACKLIST)? getBoolFieldF(rP, CSUB_BLACKLIST) : false; + nP->lastFailure = rP->hasField(CSUB_LASTFAILURE)? getNumberFieldAsDoubleF(rP, CSUB_LASTFAILURE, true) : -1; + nP->lastSuccess = rP->hasField(CSUB_LASTSUCCESS)? getNumberFieldAsDoubleF(rP, CSUB_LASTSUCCESS, true) : -1; // Attributes format subP->attrsFormat = rP->hasField(CSUB_FORMAT)? stringToRenderFormat(getStringFieldF(rP, CSUB_FORMAT)) : RF_LEGACY; @@ -222,7 +222,7 @@ static void extractNotification(Subscription* subP, const BSONObj* rP, OrionldTe */ static void extractStatus(Subscription* s, const BSONObj& r) { - s->expires = r.hasField(CSUB_EXPIRATION)? getNumberFieldAsDoubleF(&r, CSUB_EXPIRATION) : -1; + s->expires = r.hasField(CSUB_EXPIRATION)? getNumberFieldAsDoubleF(&r, CSUB_EXPIRATION, true) : -1; // // Status diff --git a/src/lib/mongoBackend/mongoRegistrationAux.cpp b/src/lib/mongoBackend/mongoRegistrationAux.cpp index 59c209a2cd..5769355e84 100644 --- a/src/lib/mongoBackend/mongoRegistrationAux.cpp +++ b/src/lib/mongoBackend/mongoRegistrationAux.cpp @@ -253,7 +253,7 @@ bool mongoSetDataProvided(ngsiv2::Registration* regP, const mongo::BSONObj* rP, */ void mongoSetExpires(ngsiv2::Registration* regP, const mongo::BSONObj& r) { - regP->expires = (r.hasField(REG_EXPIRATION))? getNumberFieldAsDoubleF(&r, REG_EXPIRATION) : -1; + regP->expires = (r.hasField(REG_EXPIRATION))? getNumberFieldAsDoubleF(&r, REG_EXPIRATION, true) : -1; } diff --git a/src/lib/mongoBackend/mongoSubCache.cpp b/src/lib/mongoBackend/mongoSubCache.cpp index b7f903d8e7..4e708fcb80 100644 --- a/src/lib/mongoBackend/mongoSubCache.cpp +++ b/src/lib/mongoBackend/mongoSubCache.cpp @@ -150,13 +150,13 @@ int mongoSubCacheItemInsert(const char* tenant, const BSONObj& sub) cSubP->tenant = (tenant == NULL || tenant[0] == 0)? strdup("") : strdup(tenant); // FIXME: strdup("") ... really? cSubP->servicePath = strdup(sub.hasField(CSUB_SERVICE_PATH)? getStringFieldF(&sub, CSUB_SERVICE_PATH) : "/"); cSubP->renderFormat = renderFormat; - cSubP->throttling = sub.hasField(CSUB_THROTTLING)? getNumberFieldAsDoubleF(&sub, CSUB_THROTTLING) : -1; - cSubP->expirationTime = sub.hasField(CSUB_EXPIRATION)? getNumberFieldAsDoubleF(&sub, CSUB_EXPIRATION) : 0; - cSubP->lastNotificationTime = sub.hasField(CSUB_LASTNOTIFICATION)? getNumberFieldAsDoubleF(&sub, CSUB_LASTNOTIFICATION) : -1; - cSubP->status = sub.hasField(CSUB_STATUS)? getStringFieldF(&sub, CSUB_STATUS) : "active"; - cSubP->blacklist = sub.hasField(CSUB_BLACKLIST)? getBoolFieldF(&sub, CSUB_BLACKLIST) : false; - cSubP->lastFailure = sub.hasField(CSUB_LASTFAILURE)? getNumberFieldAsDoubleF(&sub, CSUB_LASTFAILURE) : -1; - cSubP->lastSuccess = sub.hasField(CSUB_LASTSUCCESS)? getNumberFieldAsDoubleF(&sub, CSUB_LASTSUCCESS) : -1; + cSubP->throttling = sub.hasField(CSUB_THROTTLING)? getNumberFieldAsDoubleF(&sub, CSUB_THROTTLING, true) : -1; + cSubP->expirationTime = sub.hasField(CSUB_EXPIRATION)? getNumberFieldAsDoubleF(&sub, CSUB_EXPIRATION, true) : 0; + cSubP->lastNotificationTime = sub.hasField(CSUB_LASTNOTIFICATION)? getNumberFieldAsDoubleF(&sub, CSUB_LASTNOTIFICATION, true) : -1; + cSubP->status = sub.hasField(CSUB_STATUS)? getStringFieldF(&sub, CSUB_STATUS) : "active"; + cSubP->blacklist = sub.hasField(CSUB_BLACKLIST)? getBoolFieldF(&sub, CSUB_BLACKLIST) : false; + cSubP->lastFailure = sub.hasField(CSUB_LASTFAILURE)? getNumberFieldAsDoubleF(&sub, CSUB_LASTFAILURE, true) : -1; + cSubP->lastSuccess = sub.hasField(CSUB_LASTSUCCESS)? getNumberFieldAsDoubleF(&sub, CSUB_LASTSUCCESS, true) : -1; cSubP->count = 0; cSubP->next = NULL; @@ -452,14 +452,14 @@ int mongoSubCacheItemInsert // if the database objuect contains lastNotificationTime, // then use the value from the database // - lastNotificationTime = getNumberFieldAsDoubleF(&sub, CSUB_LASTNOTIFICATION); + lastNotificationTime = getNumberFieldAsDoubleF(&sub, CSUB_LASTNOTIFICATION, true); } cSubP->tenant = (tenant == NULL || tenant[0] == 0)? NULL : strdup(tenant); cSubP->subscriptionId = strdup(subscriptionId); cSubP->servicePath = strdup(servicePath); cSubP->renderFormat = renderFormat; - cSubP->throttling = sub.hasField(CSUB_THROTTLING)? getNumberFieldAsDoubleF(&sub, CSUB_THROTTLING) : -1; + cSubP->throttling = sub.hasField(CSUB_THROTTLING)? getNumberFieldAsDoubleF(&sub, CSUB_THROTTLING, true) : -1; cSubP->expirationTime = expirationTime; cSubP->lastNotificationTime = lastNotificationTime; cSubP->count = 0; @@ -781,6 +781,8 @@ void mongoSubCountersUpdate else snprintf(collectionPath, sizeof(collectionPath), "%s.csubs", dbName); + LM_T(LmtSubCacheStats, ("Updating sub::count to %ll", count)); + if (count > 0) mongoSubCountersUpdateCount(collectionPath, subId, count, ngsild); if (lastNotificationTime > 0) mongoSubCountersUpdateLastNotificationTime(collectionPath, subId, lastNotificationTime, ngsild); if (lastFailure > 0) mongoSubCountersUpdateLastFailure(collectionPath, subId, lastFailure, ngsild); diff --git a/src/lib/mongoBackend/mongoUpdateSubscription.cpp b/src/lib/mongoBackend/mongoUpdateSubscription.cpp index 247fb7640d..8e1653a0ab 100644 --- a/src/lib/mongoBackend/mongoUpdateSubscription.cpp +++ b/src/lib/mongoBackend/mongoUpdateSubscription.cpp @@ -78,7 +78,7 @@ static void setExpiration(const SubscriptionUpdate& subUp, const BSONObj& subOri { if (subOrig.hasField(CSUB_EXPIRATION)) { - double expires = getNumberFieldAsDoubleF(&subOrig, CSUB_EXPIRATION); + double expires = getNumberFieldAsDoubleF(&subOrig, CSUB_EXPIRATION, true); b->append(CSUB_EXPIRATION, expires); LM_T(LmtLegacy, ("Subscription expiration: %f", expires)); @@ -162,7 +162,7 @@ static void setThrottling(const SubscriptionUpdate& subUp, const BSONObj& subOri { if (subOrig.hasField(CSUB_THROTTLING)) { - double throttling = getNumberFieldAsDoubleF(&subOrig, CSUB_THROTTLING); + double throttling = getNumberFieldAsDoubleF(&subOrig, CSUB_THROTTLING, true); b->append(CSUB_THROTTLING, throttling); LM_T(LmtLegacy, ("Subscription throttling: %f", throttling)); @@ -528,7 +528,7 @@ static void setLastNotification(const BSONObj& subOrig, CachedSubscription* subC return; } - double lastNotification = getNumberFieldAsDoubleF(&subOrig, CSUB_LASTNOTIFICATION); + double lastNotification = getNumberFieldAsDoubleF(&subOrig, CSUB_LASTNOTIFICATION, true); // // Compare with 'lastNotificationTime', that might come from the sub-cache. @@ -550,7 +550,7 @@ static void setLastNotification(const BSONObj& subOrig, CachedSubscription* subC */ static double setLastFailure(const BSONObj& subOrig, CachedSubscription* subCacheP, BSONObjBuilder* b) { - double lastFailure = subOrig.hasField(CSUB_LASTFAILURE)? getNumberFieldAsDoubleF(&subOrig, CSUB_LASTFAILURE) : 0; + double lastFailure = subOrig.hasField(CSUB_LASTFAILURE)? getNumberFieldAsDoubleF(&subOrig, CSUB_LASTFAILURE, true) : 0; // // Compare with 'lastFailure' from the sub-cache. @@ -574,7 +574,7 @@ static double setLastFailure(const BSONObj& subOrig, CachedSubscription* subCach */ static double setLastSuccess(const BSONObj& subOrig, CachedSubscription* subCacheP, BSONObjBuilder* b) { - double lastSuccess = getNumberFieldAsDoubleF(&subOrig, CSUB_LASTSUCCESS); + double lastSuccess = getNumberFieldAsDoubleF(&subOrig, CSUB_LASTSUCCESS, true); // // Compare with 'lastSuccess' from the sub-cache. @@ -805,7 +805,7 @@ void updateInCache lastNotification, lastFailure, lastSuccess, - doc.hasField(CSUB_EXPIRATION)? getNumberFieldAsDoubleF(&doc, CSUB_EXPIRATION) : 0, + doc.hasField(CSUB_EXPIRATION)? getNumberFieldAsDoubleF(&doc, CSUB_EXPIRATION, true) : 0, doc.hasField(CSUB_STATUS)? getStringFieldF(&doc, CSUB_STATUS) : STATUS_ACTIVE, q, mq, diff --git a/src/lib/mongoBackend/safeMongo.cpp b/src/lib/mongoBackend/safeMongo.cpp index 1d2af7f9d9..0863012a20 100644 --- a/src/lib/mongoBackend/safeMongo.cpp +++ b/src/lib/mongoBackend/safeMongo.cpp @@ -280,7 +280,7 @@ bool getBoolField(const BSONObj* bP, const char* field, const char* caller, int * * getNumberFieldAsDouble - */ -double getNumberFieldAsDouble(const BSONObj* bP, const char* field, const char* caller, int line) +double getNumberFieldAsDouble(const BSONObj* bP, const char* field, bool okToNotExist, const char* caller, int line) { mongo::BSONType type = mongo::EOO; @@ -296,6 +296,9 @@ double getNumberFieldAsDouble(const BSONObj* bP, const char* field, const char* } else { + if (okToNotExist == true) + return 0; + LM_E(("Runtime Error (double/int/long field '%s' is missing in BSONObj <%s> from caller %s:%d)", field, bP->toString().c_str(), diff --git a/src/lib/mongoBackend/safeMongo.h b/src/lib/mongoBackend/safeMongo.h index 1e0799f67c..0813774832 100644 --- a/src/lib/mongoBackend/safeMongo.h +++ b/src/lib/mongoBackend/safeMongo.h @@ -46,7 +46,7 @@ #define getNumberFieldF(b, field) getNumberField(b, field, __FUNCTION__, __LINE__) #define getIntFieldF(b, field) getIntField(b, field, __FUNCTION__, __LINE__) #define getIntOrLongFieldAsLongF(b, field) getIntOrLongFieldAsLong(b, field, __FUNCTION__, __LINE__) -#define getNumberFieldAsDoubleF(b, field) getNumberFieldAsDouble(b, field, __FUNCTION__, __LINE__) +#define getNumberFieldAsDoubleF(b, field, okToNotExist) getNumberFieldAsDouble(b, field, okToNotExist, __FUNCTION__, __LINE__) #define getBoolFieldF(b, field) getBoolField(b, field, __FUNCTION__, __LINE__) #define getFieldF(b, field) getField(b, field, __FUNCTION__, __LINE__) #define setStringVectorF(b, field, v) setStringVector(b, field, v, __FUNCTION__, __LINE__) @@ -148,6 +148,7 @@ extern double getNumberFieldAsDouble ( const mongo::BSONObj* bP, const char* field, + bool okToNotExist, const char* caller = "", int line = 0 ); diff --git a/src/lib/ngsi/ContextElementResponse.cpp b/src/lib/ngsi/ContextElementResponse.cpp index 6e66206bd2..73a167ce11 100644 --- a/src/lib/ngsi/ContextElementResponse.cpp +++ b/src/lib/ngsi/ContextElementResponse.cpp @@ -306,12 +306,12 @@ ContextElementResponse::ContextElementResponse /* Set creDate and modDate at attribute level */ if (attr.hasField(ENT_ATTRS_CREATION_DATE)) { - caP->creDate = getNumberFieldAsDoubleF(&attr, ENT_ATTRS_CREATION_DATE); + caP->creDate = getNumberFieldAsDoubleF(&attr, ENT_ATTRS_CREATION_DATE, false); } if (attr.hasField(ENT_ATTRS_MODIFICATION_DATE)) { - caP->modDate = getNumberFieldAsDoubleF(&attr, ENT_ATTRS_MODIFICATION_DATE); + caP->modDate = getNumberFieldAsDoubleF(&attr, ENT_ATTRS_MODIFICATION_DATE, false); } contextElement.contextAttributeVector.push_back(caP); @@ -320,12 +320,12 @@ ContextElementResponse::ContextElementResponse /* Set creDate and modDate at entity level */ if (entityDocP->hasField(ENT_CREATION_DATE)) { - contextElement.entityId.creDate = getNumberFieldAsDoubleF(entityDocP, ENT_CREATION_DATE); + contextElement.entityId.creDate = getNumberFieldAsDoubleF(entityDocP, ENT_CREATION_DATE, false); } if (entityDocP->hasField(ENT_MODIFICATION_DATE)) { - contextElement.entityId.modDate = getNumberFieldAsDoubleF(entityDocP, ENT_MODIFICATION_DATE); + contextElement.entityId.modDate = getNumberFieldAsDoubleF(entityDocP, ENT_MODIFICATION_DATE, false); } } diff --git a/src/lib/orionld/common/orionldServerConnect.cpp b/src/lib/orionld/common/orionldServerConnect.cpp index b0625af25a..12f314595f 100644 --- a/src/lib/orionld/common/orionldServerConnect.cpp +++ b/src/lib/orionld/common/orionldServerConnect.cpp @@ -39,7 +39,7 @@ // // orionldServerConnect - // -int orionldServerConnect(char* ip, uint16_t portNo) +int orionldServerConnect(const char* ip, uint16_t portNo) { int fd; struct hostent* heP; diff --git a/src/lib/orionld/common/orionldServerConnect.h b/src/lib/orionld/common/orionldServerConnect.h index fa1d092f81..1d9bf8aadd 100644 --- a/src/lib/orionld/common/orionldServerConnect.h +++ b/src/lib/orionld/common/orionldServerConnect.h @@ -32,6 +32,6 @@ // // orionldServerConnect - // -extern int orionldServerConnect(char* ip, uint16_t portNo); +extern int orionldServerConnect(const char* ip, uint16_t portNo); #endif // SRC_LIB_ORIONLD_COMMON_ORIONLDSERVERCONNECT_H_ diff --git a/src/lib/orionld/common/orionldState.cpp b/src/lib/orionld/common/orionldState.cpp index d32f274d50..f8bb296af1 100644 --- a/src/lib/orionld/common/orionldState.cpp +++ b/src/lib/orionld/common/orionldState.cpp @@ -110,7 +110,6 @@ size_t hostHeaderLen; PernotSubCache pernotSubCache; - // // Variables for Mongo C Driver // diff --git a/src/lib/orionld/common/orionldState.h b/src/lib/orionld/common/orionldState.h index 379edbb432..5341a670c6 100644 --- a/src/lib/orionld/common/orionldState.h +++ b/src/lib/orionld/common/orionldState.h @@ -566,6 +566,8 @@ extern char dbURI[]; // From orionld.cpp extern bool multitenancy; // From orionld.cpp extern int contextDownloadAttempts; // From orionld.cpp extern int contextDownloadTimeout; // From orionld.cpp +extern int subCacheInterval; // From orionld.cpp +extern int subCacheFlushInterval; // From orionld.cpp extern bool troe; // From orionld.cpp extern char troeHost[256]; // From orionld.cpp extern unsigned short troePort; // From orionld.cpp @@ -593,7 +595,7 @@ extern char hostHeaderNoLF[128]; // move to orionld.cpp? extern size_t hostHeaderLen; // move to orionld.cpp? extern bool debugCurl; // From orionld.cpp extern bool noCache; // From orionld.cpp -extern int cSubCounters; // Number of subscription counter updates before flush from sub-cache to DB +extern uint32_t cSubCounters; // Number of subscription counter updates before flush from sub-cache to DB extern PernotSubCache pernotSubCache; extern char localIpAndPort[135]; // Local address for X-Forwarded-For (from orionld.cpp) extern unsigned long long inReqPayloadMaxSize; diff --git a/src/lib/orionld/dbModel/dbModelFromApiSubscription.cpp b/src/lib/orionld/dbModel/dbModelFromApiSubscription.cpp index 8ec86f340c..55947b8abf 100644 --- a/src/lib/orionld/dbModel/dbModelFromApiSubscription.cpp +++ b/src/lib/orionld/dbModel/dbModelFromApiSubscription.cpp @@ -348,6 +348,7 @@ bool dbModelFromApiSubscription(KjNode* apiSubscriptionP, bool patch) // it's much better to not do this inside the loop. Especially as the tree must be modified and a for-loop // would no longer be possible // + kjTreeLog(notificationP, "notificationP", LmtSR); if (notificationP != NULL) { KjNode* nItemP = notificationP->value.firstChildP; diff --git a/src/lib/orionld/dbModel/dbModelToApiEntity.cpp b/src/lib/orionld/dbModel/dbModelToApiEntity.cpp index cafdd01330..166347a077 100644 --- a/src/lib/orionld/dbModel/dbModelToApiEntity.cpp +++ b/src/lib/orionld/dbModel/dbModelToApiEntity.cpp @@ -57,7 +57,7 @@ extern "C" // orionldAlterationsTreat() ? // orionldPostBatchUpsert() // -KjNode* dbModelToApiEntity(KjNode* dbEntityP, bool sysAttrs, const char* entityId) +KjNode* dbModelToApiEntity(KjNode* dbEntityP, bool sysAttrs, char* entityId) { KjNode* apiEntityP = kjObject(orionldState.kjsonP, NULL); KjNode* dbIdObjectP = kjLookup(dbEntityP, "_id"); diff --git a/src/lib/orionld/dbModel/dbModelToApiEntity.h b/src/lib/orionld/dbModel/dbModelToApiEntity.h index 6dc15549b5..a7cd3e07bf 100644 --- a/src/lib/orionld/dbModel/dbModelToApiEntity.h +++ b/src/lib/orionld/dbModel/dbModelToApiEntity.h @@ -49,7 +49,7 @@ extern "C" // USED BY // - orionldAlterationsTreat (for notifications) // -extern KjNode* dbModelToApiEntity(KjNode* attrP, bool sysAttrs, const char* entityId); +extern KjNode* dbModelToApiEntity(KjNode* attrP, bool sysAttrs, char* entityId); diff --git a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp index acb3112316..1dfbb3241b 100644 --- a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp +++ b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp @@ -185,7 +185,6 @@ KjNode* dbModelToApiSubscription double* timeIntervalP ) { - LM_T(LmtSR, ("Here")); KjNode* dbSubIdP = kjLookup(dbSubP, "_id"); DB_ITEM_NOT_FOUND(dbSubIdP, "id", tenant); KjNode* dbNameP = kjLookup(dbSubP, "name"); KjNode* dbDescriptionP = kjLookup(dbSubP, "description"); @@ -205,6 +204,8 @@ KjNode* dbModelToApiSubscription KjNode* dbExpirationP = kjLookup(dbSubP, "expiration"); KjNode* dbLdContextP = kjLookup(dbSubP, "ldContext"); KjNode* dbCountP = kjLookup(dbSubP, "count"); + KjNode* timesFailedP = kjLookup(dbSubP, "timesFailed"); + KjNode* noMatchP = kjLookup(dbSubP, "noMatch"); KjNode* dbLastNotificationP = kjLookup(dbSubP, "lastNotification"); KjNode* dbLastSuccessP = kjLookup(dbSubP, "lastSuccess"); KjNode* dbLastFailureP = kjLookup(dbSubP, "lastFailure"); @@ -489,7 +490,37 @@ KjNode* dbModelToApiSubscription kjChildAdd(notificationP, nStatusNodeP); + // lastNotification + if (dbLastNotificationP != NULL) + kjChildAdd(notificationP, dbLastNotificationP); + + // lastSuccess + if (dbLastSuccessP != NULL) + kjChildAdd(notificationP, dbLastSuccessP); + + // lastFailure + if (dbLastFailureP != NULL) + kjChildAdd(notificationP, dbLastFailureP); + + // timesSent + if (dbCountP != NULL) + { + dbCountP->name = (char*) "timesSent"; + kjChildAdd(notificationP, dbCountP); + } + + // timesFailed + if (timesFailedP != NULL) + kjChildAdd(notificationP, timesFailedP); + + // noMatch + if (noMatchP != NULL) + kjChildAdd(notificationP, noMatchP); + // Add "notification" to top level + kjChildAdd(apiSubP, notificationP); + + if (dbReferenceP != NULL) { kjChildAdd(endpointP, dbReferenceP); @@ -502,7 +533,6 @@ KjNode* dbModelToApiSubscription dbMimeTypeP->name = (char*) "accept"; } - kjChildAdd(apiSubP, notificationP); // @@ -647,21 +677,6 @@ KjNode* dbModelToApiSubscription dbLdContextP->name = (char*) "jsonldContext"; } - // count - take sub-cache delta into consideration - if (dbCountP != NULL) - kjChildAdd(apiSubP, dbCountP); - - // lastNotification - take from sub-cache - if (dbLastNotificationP != NULL) - kjChildAdd(apiSubP, dbLastNotificationP); - - // lastSuccess - take from sub-cache - if (dbLastSuccessP != NULL) - kjChildAdd(apiSubP, dbLastSuccessP); - - // lastFailure - take from sub-cache - if (dbLastFailureP != NULL) - kjChildAdd(apiSubP, dbLastFailureP); if (qNodePP != NULL) // FIXME: This is more than a bit weird ... *qNodePP = NULL; diff --git a/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.cpp b/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.cpp index 14e1199cef..dc584c2164 100644 --- a/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.cpp +++ b/src/lib/orionld/kjTree/kjTreeFromPernotSubscription.cpp @@ -167,9 +167,9 @@ KjNode* kjTreeFromPernotSubscription(PernotSubscription* pSubP, bool sysAttrs, b kjChildAdd(notificationP, notificationStatusP); // 4. counters and timestamps - counterAdd(notificationP, "timesSent", pSubP->dbNotificationAttempts + pSubP->notificationAttempts); - counterAdd(notificationP, "timesFailed", pSubP->dbNotificationErrors + pSubP->notificationErrors); - timestampAdd(notificationP, "lastNotification", pSubP->lastNotificationAttempt); + counterAdd(notificationP, "timesSent", pSubP->notificationAttemptsDb + pSubP->notificationAttempts); + counterAdd(notificationP, "timesFailed", pSubP->notificationErrorsDb + pSubP->notificationErrors); + timestampAdd(notificationP, "lastNotification", pSubP->lastNotificationTime); timestampAdd(notificationP, "lastSuccess", pSubP->lastSuccessTime); timestampAdd(notificationP, "lastFailure", pSubP->lastFailureTime); @@ -202,10 +202,12 @@ KjNode* kjTreeFromPernotSubscription(PernotSubscription* pSubP, bool sysAttrs, b KjNode* qP = kjLookup(sP, "q"); if (qP != NULL) { - LM_T(LmtPernot, ("Found 'q' - fixing it", qP->value.s)); + LM_T(LmtPernot, ("Found 'q' (%s)- fixing it", qP->value.s)); dbModelValueStrip(qP); + LM_T(LmtPernot, ("'q' after dbModelValueStrip: '%s'", qP->value.s)); qAliasCompact(qP, true); // qAliasCompact uses orionldState.contextP - which is what we want qP->name = (char*) "q"; + LM_T(LmtPernot, ("'q' final: '%s'", qP->value.s)); } else LM_T(LmtPernot, ("No 'q'")); diff --git a/src/lib/orionld/mongoBackend/mongoLdRegistrationAux.cpp b/src/lib/orionld/mongoBackend/mongoLdRegistrationAux.cpp index 5d7855d250..896cac6c92 100644 --- a/src/lib/orionld/mongoBackend/mongoLdRegistrationAux.cpp +++ b/src/lib/orionld/mongoBackend/mongoLdRegistrationAux.cpp @@ -132,7 +132,7 @@ void mongoSetLdName(ngsiv2::Registration* reg, mongo::BSONObj* bobjP) */ void mongoSetExpiration(ngsiv2::Registration* regP, const mongo::BSONObj& r) { - regP->expires = (r.hasField(REG_EXPIRATION))? getNumberFieldAsDoubleF(&r, REG_EXPIRATION) : -1; + regP->expires = (r.hasField(REG_EXPIRATION))? getNumberFieldAsDoubleF(&r, REG_EXPIRATION, true) : -1; } @@ -148,8 +148,8 @@ void mongoSetLdObservationInterval(ngsiv2::Registration* reg, const mongo::BSONO mongo::BSONObj obj; getObjectFieldF(&obj, &r, REG_OBSERVATION_INTERVAL); - reg->observationInterval.start = getNumberFieldAsDoubleF(&obj, "startAt"); - reg->observationInterval.end = obj.hasField("endAt") ? getNumberFieldAsDoubleF(&obj, "endAt") : -1; + reg->observationInterval.start = getNumberFieldAsDoubleF(&obj, "startAt", true); + reg->observationInterval.end = obj.hasField("endAt") ? getNumberFieldAsDoubleF(&obj, "endAt", true) : -1; } else { @@ -171,8 +171,8 @@ void mongoSetLdManagementInterval(ngsiv2::Registration* reg, const mongo::BSONOb mongo::BSONObj obj; getObjectFieldF(&obj, &r, REG_MANAGEMENT_INTERVAL); - reg->managementInterval.start = getNumberFieldAsDoubleF(&obj, "startAt"); - reg->managementInterval.end = obj.hasField("endAt") ? getNumberFieldAsDoubleF(&obj, "endAt") : -1; + reg->managementInterval.start = getNumberFieldAsDoubleF(&obj, "startAt", true); + reg->managementInterval.end = obj.hasField("endAt") ? getNumberFieldAsDoubleF(&obj, "endAt", true) : -1; } else { @@ -190,7 +190,7 @@ void mongoSetLdManagementInterval(ngsiv2::Registration* reg, const mongo::BSONOb void mongoSetLdTimestamp(double* timestampP, const char* name, const mongo::BSONObj& bobj) { if (bobj.hasField(name)) - *timestampP = getNumberFieldAsDoubleF(&bobj, name); + *timestampP = getNumberFieldAsDoubleF(&bobj, name, true); else *timestampP = -1; } diff --git a/src/lib/orionld/mongoc/mongocEntitiesQuery.cpp b/src/lib/orionld/mongoc/mongocEntitiesQuery.cpp index 9b05ccb0ec..d786ff79c9 100644 --- a/src/lib/orionld/mongoc/mongocEntitiesQuery.cpp +++ b/src/lib/orionld/mongoc/mongocEntitiesQuery.cpp @@ -40,6 +40,7 @@ extern "C" #include "orionld/types/OrionldGeoInfo.h" // OrionldGeoInfo #include "orionld/q/QNode.h" // QNode #include "orionld/q/qTreeToBson.h" // qTreeToBson +#include "orionld/mongoc/mongocWriteLog.h" // MONGOC_RLOG - FIXME: change name to mongocLog.h #include "orionld/mongoc/mongocConnectionGet.h" // mongocConnectionGet #include "orionld/mongoc/mongocKjTreeToBson.h" // mongocKjTreeToBson #include "orionld/mongoc/mongocKjTreeFromBson.h" // mongocKjTreeFromBson @@ -788,6 +789,7 @@ KjNode* mongocEntitiesQuery if (limit != 0) { + MONGOC_RLOG("Lookup Entities", orionldState.tenantP->mongoDbName, "entities", &mongoFilter, LmtMongoc); mongoCursorP = mongoc_collection_find_with_opts(orionldState.mongoc.entitiesP, &mongoFilter, &options, readPrefs); bson_destroy(&options); diff --git a/src/lib/orionld/mongoc/mongocEntitiesQuery2.cpp b/src/lib/orionld/mongoc/mongocEntitiesQuery2.cpp index 27961b29f5..c5b9d072fb 100644 --- a/src/lib/orionld/mongoc/mongocEntitiesQuery2.cpp +++ b/src/lib/orionld/mongoc/mongocEntitiesQuery2.cpp @@ -39,6 +39,7 @@ extern "C" #include "orionld/types/OrionldGeoInfo.h" // OrionldGeoInfo #include "orionld/types/OrionldGeometry.h" // orionldGeometryFromString #include "orionld/q/QNode.h" // QNode +#include "orionld/mongoc/mongocWriteLog.h" // MONGOC_RLOG - FIXME: change name to mongocLog.h #include "orionld/mongoc/mongocConnectionGet.h" // mongocConnectionGet #include "orionld/mongoc/mongocKjTreeToBson.h" // mongocKjTreeToBson #include "orionld/mongoc/mongocKjTreeFromBson.h" // mongocKjTreeFromBson @@ -379,6 +380,7 @@ KjNode* mongocEntitiesQuery2 bson_free(optionsString); #endif + MONGOC_RLOG("Lookup Entities", orionldState.tenantP->mongoDbName, "entities", &mongoFilter, LmtMongoc); mongoCursorP = mongoc_collection_find_with_opts(orionldState.mongoc.entitiesP, &mongoFilter, &options, readPrefs); bson_destroy(&options); @@ -400,6 +402,7 @@ KjNode* mongocEntitiesQuery2 #endif int hits = 0; + mongoDocP = NULL; while (mongoc_cursor_next(mongoCursorP, &mongoDocP)) { entityNodeP = mongocKjTreeFromBson(mongoDocP, &title, &detail); diff --git a/src/lib/orionld/mongoc/mongocKjTreeFromBson.cpp b/src/lib/orionld/mongoc/mongocKjTreeFromBson.cpp index 995ca6430a..4395d17d47 100644 --- a/src/lib/orionld/mongoc/mongocKjTreeFromBson.cpp +++ b/src/lib/orionld/mongoc/mongocKjTreeFromBson.cpp @@ -59,7 +59,7 @@ KjNode* mongocKjTreeFromBson(const void* dataP, char** titleP, char** detailsP) else { treeP = kjParse(orionldState.kjsonP, json); - + LM_T(LmtPernot, ("")); if (treeP == NULL) { *titleP = (char*) "Internal Error"; diff --git a/src/lib/orionld/mongoc/mongocSubCachePopulateByTenant.cpp b/src/lib/orionld/mongoc/mongocSubCachePopulateByTenant.cpp index e3b7fbbc38..bc0cadf685 100644 --- a/src/lib/orionld/mongoc/mongocSubCachePopulateByTenant.cpp +++ b/src/lib/orionld/mongoc/mongocSubCachePopulateByTenant.cpp @@ -106,6 +106,8 @@ bool mongocSubCachePopulateByTenant(OrionldTenant* tenantP) KjNode* sysAttrsP = NULL; RenderFormat renderFormat = RF_NORMALIZED; double timeInterval = 0; + + kjTreeLog(dbSubP, "dbSubP", LmtPernot); KjNode* apiSubP = dbModelToApiSubscription(dbSubP, tenantP->tenant, true, @@ -120,6 +122,8 @@ bool mongocSubCachePopulateByTenant(OrionldTenant* tenantP) if (apiSubP == NULL) continue; + kjTreeLog(apiSubP, "apiSubP", LmtPernot); + OrionldContext* contextP = NULL; if (contextNodeP != NULL) contextP = orionldContextFromUrl(contextNodeP->value.s, NULL); @@ -127,7 +131,7 @@ bool mongocSubCachePopulateByTenant(OrionldTenant* tenantP) if (timeInterval == 0) subCacheApiSubscriptionInsert(apiSubP, qTree, coordinatesP, contextP, tenantP->tenant, showChangesP, sysAttrsP, renderFormat); else - pernotSubCacheAdd(NULL, apiSubP, NULL, qTree, coordinatesP, contextP, tenantP->tenant, showChangesP, sysAttrsP, renderFormat, timeInterval); + pernotSubCacheAdd(NULL, apiSubP, NULL, qTree, coordinatesP, contextP, tenantP, showChangesP, sysAttrsP, renderFormat, timeInterval); } mongoc_client_pool_push(mongocPool, connectionP); diff --git a/src/lib/orionld/mongoc/mongocSubCountersUpdate.cpp b/src/lib/orionld/mongoc/mongocSubCountersUpdate.cpp index dde05e309f..4132d97abe 100644 --- a/src/lib/orionld/mongoc/mongocSubCountersUpdate.cpp +++ b/src/lib/orionld/mongoc/mongocSubCountersUpdate.cpp @@ -27,9 +27,11 @@ #include "logMsg/logMsg.h" // LM_* #include "cache/subCache.h" // CachedSubscription + #include "orionld/common/orionldState.h" // orionldState #include "orionld/common/tenantList.h" // tenant0 #include "orionld/common/orionldTenantLookup.h" // orionldTenantLookup +#include "orionld/types/OrionldTenant.h" // OrionldTenant #include "orionld/mongoc/mongocSubCountersUpdate.h" // Own interface @@ -57,53 +59,67 @@ // void mongocSubCountersUpdate ( - CachedSubscription* cSubP, - int deltaCount, + OrionldTenant* tenantP, + const char* subscriptionId, + bool ngsild, + int32_t deltaAttempts, + int32_t deltaFailures, + int32_t deltaNoMatch, double lastNotificationTime, - double lastFailure, double lastSuccess, - bool forcedToPause, - bool ngsild + double lastFailure, + bool forcedToPause ) { - OrionldTenant* tenantP = (cSubP->tenant == NULL)? &tenant0 : orionldTenantLookup(cSubP->tenant); + if (tenantP == NULL) + tenantP = &tenant0; + mongoc_client_t* connectionP = mongoc_client_pool_pop(mongocPool); mongoc_collection_t* subscriptionsP = mongoc_client_get_collection(connectionP, tenantP->mongoDbName, "csubs"); bson_t request; // Entire request with count and timestamps to be updated bson_t reply; - bson_t count; - bson_t max; bson_t selector; + bson_t max; + bson_t inc; bson_init(&reply); bson_init(&selector); - bson_init(&count); + bson_init(&inc); bson_init(&max); bson_init(&request); // Selector - The _id is an OID if not NGSI-LD - if (cSubP->ldContext != "") - bson_append_utf8(&selector, "_id", 3, cSubP->subscriptionId, -1); + if (ngsild == true) + bson_append_utf8(&selector, "_id", 3, subscriptionId, -1); else { bson_oid_t oid; - bson_oid_init_from_string(&oid, cSubP->subscriptionId); + bson_oid_init_from_string(&oid, subscriptionId); bson_append_oid(&selector, "_id", 3, &oid); } - // Count - if (deltaCount > 0) - bson_append_int64(&count, "count", 5, deltaCount); + // Attempts (timesSent) + if (deltaAttempts > 0) + bson_append_int32(&inc, "count", 9, deltaAttempts); + + // deltaFailures (timesFailed) + if (deltaFailures > 0) + bson_append_int32(&inc, "timesFailed", 11, deltaFailures); + + // deltaNoMatch (noMatch) + if (deltaNoMatch > 0) + bson_append_int32(&inc, "noMatch", 7, deltaNoMatch); + + if (deltaAttempts + deltaFailures + deltaNoMatch > 0) + bson_append_document(&request, "$inc", 4, &inc); + // Timestamps if (lastNotificationTime > 0) bson_append_double(&max, "lastNotification", 16, lastNotificationTime); if (lastSuccess > 0) bson_append_double(&max, "lastSuccess", 11, lastSuccess); if (lastFailure > 0) bson_append_double(&max, "lastFailure", 11, lastFailure); - if (deltaCount > 0) - bson_append_document(&request, "$inc", 4, &count); - if (forcedToPause == true) { bson_t status; @@ -119,14 +135,14 @@ void mongocSubCountersUpdate bool b = mongoc_collection_update_one(subscriptionsP, &selector, &request, NULL, &reply, &bError); if (b == false) - LM_E(("SUBC: mongoc error updating subscription counters/timestamps for '%s': [%d.%d]: %s", cSubP->subscriptionId, bError.domain, bError.code, bError.message)); + LM_E(("SUBC: mongoc error updating subscription counters/timestamps for '%s': [%d.%d]: %s", subscriptionId, bError.domain, bError.code, bError.message)); mongoc_client_pool_push(mongocPool, connectionP); mongoc_collection_destroy(subscriptionsP); bson_destroy(&reply); bson_destroy(&selector); - bson_destroy(&count); + bson_destroy(&inc); bson_destroy(&max); bson_destroy(&request); } diff --git a/src/lib/orionld/mongoc/mongocSubCountersUpdate.h b/src/lib/orionld/mongoc/mongocSubCountersUpdate.h index bac0841c79..ef76e604db 100644 --- a/src/lib/orionld/mongoc/mongocSubCountersUpdate.h +++ b/src/lib/orionld/mongoc/mongocSubCountersUpdate.h @@ -25,7 +25,7 @@ * * Author: Ken Zangelin */ -#include "cache/subCache.h" // CachedSubscription +#include "orionld/types/OrionldTenant.h" // OrionldTenant @@ -35,13 +35,16 @@ // extern void mongocSubCountersUpdate ( - CachedSubscription* cSubP, - int deltaCount, + OrionldTenant* tenantP, + const char* subscriptionId, + bool ngsild, + int32_t deltaAttempts, + int32_t deltaFailures, + int32_t deltaNoMatch, double lastNotificationTime, - double lastFailure, double lastSuccess, - bool forcedToPause, - bool ngsild + double lastFailure, + bool forcedToPause ); #endif // SRC_LIB_ORIONLD_MONGOC_MONGOCSUBCOUNTERSUPDATE_H_ diff --git a/src/lib/orionld/notifications/httpNotify.cpp b/src/lib/orionld/notifications/httpNotify.cpp index 325a59cc9e..0d0e7f48a7 100644 --- a/src/lib/orionld/notifications/httpNotify.cpp +++ b/src/lib/orionld/notifications/httpNotify.cpp @@ -37,30 +37,41 @@ // // httpNotify - send a notification over http // -int httpNotify(CachedSubscription* cSubP, struct iovec* ioVec, int ioVecLen, double notificationTime) +int httpNotify +( + CachedSubscription* cSubP, + PernotSubscription* pSubP, + const char* subscriptionId, + const char* ip, + unsigned short port, + const char* path, + struct iovec* ioVec, + int ioVecLen, + double notificationTime +) { // Connect - LM_T(LmtNotificationSend, ("%s: Connecting to notification '%s:%d' receptor for '%s' notification", cSubP->subscriptionId, cSubP->ip, cSubP->port, cSubP->protocolString)); - int fd = orionldServerConnect(cSubP->ip, cSubP->port); + LM_T(LmtNotificationSend, ("%s: Connecting to notification '%s:%d' receptor for HTTP notification", subscriptionId, ip, port)); + int fd = orionldServerConnect(ip, port); if (fd == -1) { - LM_E(("Internal Error (unable to connect to server for notification for subscription '%s': %s)", cSubP->subscriptionId, strerror(errno))); - notificationFailure(cSubP, "Unable to connect to notification endpoint", notificationTime); + LM_E(("Internal Error (unable to connect to server for notification for subscription '%s': %s)", subscriptionId, strerror(errno))); + notificationFailure(cSubP, pSubP, "Unable to connect to notification endpoint", notificationTime); return -1; } - LM_T(LmtNotificationSend, ("%s: Connected to notification receptor '%s:%d' on fd %d", cSubP->subscriptionId, cSubP->ip, cSubP->port, fd)); + LM_T(LmtNotificationSend, ("%s: Connected to notification receptor '%s:%d' on fd %d", subscriptionId, ip, port, fd)); if (lmTraceIsSet(LmtNotificationHeaders) == true) { for (int ix = 0; ix < ioVecLen - 1; ix++) { - LM_T(LmtNotificationHeaders, ("%s: Notification Request Header: '%s'", cSubP->subscriptionId, ioVec[ix].iov_base)); + LM_T(LmtNotificationHeaders, ("%s: Notification Request Header: '%s'", subscriptionId, ioVec[ix].iov_base)); } } - LM_T(LmtNotificationBody, ("%s: Notification Request Body: %s", cSubP->subscriptionId, ioVec[ioVecLen - 1].iov_base)); + LM_T(LmtNotificationBody, ("%s: Notification Request Body: %s", subscriptionId, ioVec[ioVecLen - 1].iov_base)); // Send int nb; @@ -68,12 +79,12 @@ int httpNotify(CachedSubscription* cSubP, struct iovec* ioVec, int ioVecLen, dou { close(fd); - LM_E(("Internal Error (unable to send to server for notification for subscription '%s' (fd: %d): %s", cSubP->subscriptionId, fd, strerror(errno))); - notificationFailure(cSubP, "Unable to write to notification endpoint", notificationTime); + LM_E(("Internal Error (unable to send to server for notification for subscription '%s' (fd: %d): %s", subscriptionId, fd, strerror(errno))); + notificationFailure(cSubP, pSubP, "Unable to write to notification endpoint", notificationTime); return -1; } - LM_T(LmtNotificationSend, ("%s: Written %d bytes to fd %d of %s:%d", cSubP->subscriptionId, nb, fd, cSubP->ip, cSubP->port)); + LM_T(LmtNotificationSend, ("%s: Written %d bytes to fd %d of %s:%d", subscriptionId, nb, fd, ip, port)); return fd; } diff --git a/src/lib/orionld/notifications/httpNotify.h b/src/lib/orionld/notifications/httpNotify.h index 30b88f4d5f..1c7393d073 100644 --- a/src/lib/orionld/notifications/httpNotify.h +++ b/src/lib/orionld/notifications/httpNotify.h @@ -28,6 +28,7 @@ #include // iovec #include "cache/CachedSubscription.h" // CachedSubscription +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription @@ -35,6 +36,17 @@ // // httpNotify - send a notification over http // -extern int httpNotify(CachedSubscription* cSubP, struct iovec* ioVec, int ioVecLen, double timestamp); +extern int httpNotify +( + CachedSubscription* cSubP, + PernotSubscription* pSubP, + const char* subscriptionId, + const char* ip, + unsigned short port, + const char* path, + struct iovec* ioVec, + int ioVecLen, + double notificationTime +); #endif // SRC_LIB_ORIONLD_NOTIFICATIONS_HTTPNOTIFY_H_ diff --git a/src/lib/orionld/notifications/notificationFailure.cpp b/src/lib/orionld/notifications/notificationFailure.cpp index 8ae08d0ab3..e2804ae25b 100644 --- a/src/lib/orionld/notifications/notificationFailure.cpp +++ b/src/lib/orionld/notifications/notificationFailure.cpp @@ -48,6 +48,7 @@ void notificationFailure(CachedSubscription* subP, const char* errorReason, doub subP->lastFailure = notificationTime; subP->consecutiveErrors += 1; subP->count += 1; + subP->failures += 1; subP->dirty += 1; strncpy(subP->lastErrorReason, errorReason, sizeof(subP->lastErrorReason) - 1); @@ -75,10 +76,12 @@ void notificationFailure(CachedSubscription* subP, const char* errorReason, doub if (((cSubCounters != 0) && (subP->dirty >= cSubCounters)) || (forcedToPause == true)) { LM_T(LmtNotificationStats, ("%s: Calling mongocSubCountersUpdate", subP->subscriptionId)); - mongocSubCountersUpdate(subP, subP->count, subP->lastNotificationTime, subP->lastFailure, subP->lastSuccess, forcedToPause, true); - subP->dirty = 0; - subP->dbCount += subP->count; - subP->count = 0; + mongocSubCountersUpdate(subP->tenantP, subP->subscriptionId, (subP->ldContext != ""), subP->count, subP->failures, 0, subP->lastNotificationTime, subP->lastSuccess, subP->lastFailure, forcedToPause); + subP->dirty = 0; + subP->dbCount += subP->count; + subP->count = 0; + subP->dbFailures += subP->failures; + subP->failures = 0; } else LM_T(LmtNotificationStats, ("%s: Not calling mongocSubCountersUpdate (cSubCounters: %d, dirty: %d, forcedToPause: %s)", @@ -87,3 +90,89 @@ void notificationFailure(CachedSubscription* subP, const char* errorReason, doub subP->dirty, (forcedToPause == true)? "true" : "false")); } + + + +// ----------------------------------------------------------------------------- +// +// notificationFailure - +// +void notificationFailure(PernotSubscription* pSubP, const char* errorReason, double notificationTime) +{ + LM_T(LmtNotificationStats, ("%s: notification failure (timestamp: %f)", pSubP->subscriptionId, notificationTime)); + bool forcedToPause = false; + + pSubP->lastNotificationTime = notificationTime; + pSubP->lastFailureTime = notificationTime; + pSubP->consecutiveErrors += 1; + pSubP->notificationAttempts += 1; + pSubP->dirty += 1; + + strncpy(pSubP->lastErrorReason, errorReason, sizeof(pSubP->lastErrorReason) - 1); + + // Force the subscription into "paused" due to too many consecutive errors + if (pSubP->consecutiveErrors >= 3) + { + LM_T(LmtNotificationStats, ("%s: force the subscription into PAUSE due to 3 consecutive errors", pSubP->subscriptionId)); + pSubP->isActive = false; + pSubP->state = SubPaused; + forcedToPause = true; + } + + promCounterIncrease(promNotifications); + promCounterIncrease(promNotificationsFailed); + + LM_T(LmtNotificationStats, ("%s: dirty: %d, cSubCounters: %d", pSubP->subscriptionId, pSubP->dirty, cSubCounters)); + + // + // Flush to DB? + // - If forcedToPause + // - If pSubP->dirty (number of counter updates since last flush) >= cSubCounters + // - AND cSubCounters != 0 + // + if (((cSubCounters != 0) && (pSubP->dirty >= cSubCounters)) || (forcedToPause == true)) + { + LM_T(LmtNotificationStats, ("%s: Calling mongocSubCountersUpdate", pSubP->subscriptionId)); + + // Save to database + mongocSubCountersUpdate(pSubP->tenantP, + pSubP->subscriptionId, + true, + pSubP->notificationAttempts, + pSubP->notificationErrors, + pSubP->noMatch, + pSubP->lastNotificationTime, + pSubP->lastSuccessTime, + pSubP->lastFailureTime, + forcedToPause); + + // Reset counters + pSubP->dirty = 0; + pSubP->notificationAttemptsDb += pSubP->notificationAttempts; + pSubP->notificationAttempts = 0; + pSubP->notificationErrorsDb += pSubP->notificationErrors; + pSubP->notificationErrors = 0; + pSubP->noMatchDb += pSubP->noMatch; + pSubP->noMatch = 0; + } + else + LM_T(LmtNotificationStats, ("%s: Not calling mongocSubCountersUpdate (cSubCounters: %d, dirty: %d, forcedToPause: %s)", + pSubP->subscriptionId, + cSubCounters, + pSubP->dirty, + (forcedToPause == true)? "true" : "false")); +} + + + +// ----------------------------------------------------------------------------- +// +// notificationFailure - +// +void notificationFailure(CachedSubscription* cSubP, PernotSubscription* pSubP, const char* errorReason, double notificationTime) +{ + if (cSubP != NULL) + notificationFailure(cSubP, errorReason, notificationTime); + else + notificationFailure(pSubP, errorReason, notificationTime); +} diff --git a/src/lib/orionld/notifications/notificationFailure.h b/src/lib/orionld/notifications/notificationFailure.h index 462707e579..2a5935634c 100644 --- a/src/lib/orionld/notifications/notificationFailure.h +++ b/src/lib/orionld/notifications/notificationFailure.h @@ -26,6 +26,7 @@ * Author: Ken Zangelin */ #include "cache/CachedSubscription.h" // CachedSubscription +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription @@ -33,6 +34,8 @@ // // notificationFailure - // -extern void notificationFailure(CachedSubscription* subP, const char* errorReason, double timestamp); +extern void notificationFailure(CachedSubscription* cSubP, const char* errorReason, double timestamp); +extern void notificationFailure(PernotSubscription* pSubP, const char* errorReason, double timestamp); +extern void notificationFailure(CachedSubscription* cSubP, PernotSubscription* pSubP, const char* errorReason, double timestamp); #endif // SRC_LIB_ORIONLD_NOTIFICATIONS_NOTIFICATIONFAILURE_H_ diff --git a/src/lib/orionld/notifications/notificationSend.cpp b/src/lib/orionld/notifications/notificationSend.cpp index 704fadd2af..98ddb43398 100644 --- a/src/lib/orionld/notifications/notificationSend.cpp +++ b/src/lib/orionld/notifications/notificationSend.cpp @@ -66,19 +66,19 @@ extern "C" // ----------------------------------------------------------------------------- // -// Fixed value headers +// Fixed value headers - mocve to separate file - also used in pernot/pernotSend.cpp // -static const char* contentTypeHeaderJson = (char*) "Content-Type: application/json\r\n"; -static const char* contentTypeHeaderJsonLd = (char*) "Content-Type: application/ld+json\r\n"; -static const char* contentTypeHeaderGeoJson = (char*) "Content-Type: application/geo+json\r\n"; -static const char* acceptHeader = (char*) "Accept: application/json\r\n"; +const char* contentTypeHeaderJson = (char*) "Content-Type: application/json\r\n"; +const char* contentTypeHeaderJsonLd = (char*) "Content-Type: application/ld+json\r\n"; +const char* contentTypeHeaderGeoJson = (char*) "Content-Type: application/geo+json\r\n"; +const char* acceptHeader = (char*) "Accept: application/json\r\n"; -static const char* normalizedHeader = (char*) "Ngsild-Attribute-Format: Normalized\r\n"; -static const char* conciseHeader = (char*) "Ngsild-Attribute-Format: Concise\r\n"; -static const char* simplifiedHeader = (char*) "Ngsild-Attribute-Format: Simplified\r\n"; +const char* normalizedHeader = (char*) "Ngsild-Attribute-Format: Normalized\r\n"; +const char* conciseHeader = (char*) "Ngsild-Attribute-Format: Concise\r\n"; +const char* simplifiedHeader = (char*) "Ngsild-Attribute-Format: Simplified\r\n"; -static const char* normalizedHeaderNgsiV2 = (char*) "Ngsiv2-Attrsformat: normalized\r\n"; -static const char* keyValuesHeaderNgsiV2 = (char*) "Ngsiv2-Attrsformat: keyValues\r\n"; +const char* normalizedHeaderNgsiV2 = (char*) "Ngsiv2-Attrsformat: normalized\r\n"; +const char* keyValuesHeaderNgsiV2 = (char*) "Ngsiv2-Attrsformat: keyValues\r\n"; char userAgentHeader[64]; // "User-Agent: orionld/" + ORIONLD_VERSION + \r\n" - initialized in orionldServiceInit() size_t userAgentHeaderLen = 0; // Set in orionldServiceInit() @@ -495,7 +495,7 @@ static KjNode* attributeFilter(KjNode* apiEntityP, OrionldAlterationMatch* mAltP // // notificationTreeForNgsiV2 - // -KjNode* notificationTreeForNgsiV2(OrionldAlterationMatch* matchP) +static KjNode* notificationTreeForNgsiV2(OrionldAlterationMatch* matchP) { CachedSubscription* subP = matchP->subP; KjNode* notificationP = kjObject(orionldState.kjsonP, NULL); @@ -509,7 +509,7 @@ KjNode* notificationTreeForNgsiV2(OrionldAlterationMatch* matchP) // KjNode* apiEntityP = matchP->altP->finalApiEntityP; // This is not correct - can be more than one entity - if (matchP->subP->attributes.size() > 0) + if (subP->attributes.size() > 0) apiEntityP = attributeFilter(apiEntityP, matchP); if ((subP->renderFormat == RF_CROSS_APIS_KEYVALUES) || (subP->renderFormat == RF_CROSS_APIS_KEYVALUES_COMPACT)) @@ -533,7 +533,7 @@ KjNode* notificationTreeForNgsiV2(OrionldAlterationMatch* matchP) // // notificationTree - // -KjNode* notificationTree(OrionldAlterationMatch* matchList) +static KjNode* notificationTree(OrionldAlterationMatch* matchList) { CachedSubscription* subP = matchList->subP; KjNode* notificationP = kjObject(orionldState.kjsonP, NULL); @@ -612,6 +612,19 @@ KjNode* notificationTree(OrionldAlterationMatch* matchList) // size_t iov_len; /* Number of bytes to transfer */ // }; // +// To adapt notificationSend to pernot, I need: +// - mAltP->subP->renderFormat (easy) +// - mAltP->subP->subscriptionId (notificationTreeForNgsiV2 - easy) +// - finalApiEntityP (notificationTreeForNgsiV2 - easy - that's the output of the query for Pernot) +// - subP->attributes (notificationTreeForNgsiV2 - easy) +// - subP->contextP (notificationTreeForNgsiV2 - easy) +// - finalApiEntityWithSysAttrsP (notificationTree - no probs, must add sysAttrs to the query if Pern ot sub has sysAttrs set) +// - subP->httpInfo.mimeType (notificationTree - easy) +// - subP->ldContext (notificationSend) +// - subP->httpInfo.notifierInfo (notificationSend) +// - mAltP->subP->rest +// +// int notificationSend(OrionldAlterationMatch* mAltP, double timestamp, CURL** curlHandlePP) { bool ngsiv2 = (mAltP->subP->renderFormat >= RF_CROSS_APIS_NORMALIZED); @@ -879,7 +892,16 @@ int notificationSend(OrionldAlterationMatch* mAltP, double timestamp, CURL** cur // // The message is ready - just need to be sent // - if (mAltP->subP->protocol == HTTP) return httpNotify(mAltP->subP, ioVec, ioVecLen, timestamp); + if (mAltP->subP->protocol == HTTP) + return httpNotify(mAltP->subP, + NULL, + mAltP->subP->subscriptionId, + mAltP->subP->ip, + mAltP->subP->port, + mAltP->subP->rest, + ioVec, + ioVecLen, + timestamp); else if (mAltP->subP->protocol == HTTPS) return httpsNotify(mAltP->subP, ioVec, ioVecLen, timestamp, curlHandlePP); else if (mAltP->subP->protocol == MQTT) return mqttNotify(mAltP->subP, ioVec, ioVecLen, timestamp); diff --git a/src/lib/orionld/notifications/notificationSuccess.cpp b/src/lib/orionld/notifications/notificationSuccess.cpp index 9f66830f15..b4793fec9e 100644 --- a/src/lib/orionld/notifications/notificationSuccess.cpp +++ b/src/lib/orionld/notifications/notificationSuccess.cpp @@ -58,10 +58,12 @@ void notificationSuccess(CachedSubscription* subP, const double timestamp) if ((cSubCounters != 0) && (subP->dirty >= cSubCounters)) { LM_T(LmtNotificationStats, ("%s: Calling mongocSubCountersUpdate", subP->subscriptionId)); - mongocSubCountersUpdate(subP, subP->count, subP->lastNotificationTime, subP->lastFailure, subP->lastSuccess, false, true); - subP->dirty = 0; - subP->dbCount += subP->count; - subP->count = 0; + mongocSubCountersUpdate(subP->tenantP, subP->subscriptionId, (subP->ldContext != ""), subP->count, subP->failures, 0, subP->lastNotificationTime, subP->lastSuccess, subP->lastFailure, false); + subP->dirty = 0; + subP->dbCount += subP->count; + subP->count = 0; + subP->dbFailures += subP->failures; + subP->failures = 0; } else LM_T(LmtNotificationStats, ("%s: Not calling mongocSubCountersUpdate (cSubCounters: %d, dirty: %d)", diff --git a/src/lib/orionld/pernot/CMakeLists.txt b/src/lib/orionld/pernot/CMakeLists.txt index 4c31454352..f27a44126a 100644 --- a/src/lib/orionld/pernot/CMakeLists.txt +++ b/src/lib/orionld/pernot/CMakeLists.txt @@ -27,6 +27,7 @@ SET (SOURCES pernotSubCacheLookup.cpp pernotLoop.cpp pernotTreat.cpp + pernotSend.cpp ) # Include directories diff --git a/src/lib/orionld/pernot/PernotSubCache.cpp b/src/lib/orionld/pernot/PernotSubCache.cpp index 60f1c34265..6305c02276 100644 --- a/src/lib/orionld/pernot/PernotSubCache.cpp +++ b/src/lib/orionld/pernot/PernotSubCache.cpp @@ -30,33 +30,11 @@ -// ----------------------------------------------------------------------------- -// -// pernotSubCacheAdd - -// -PernotSubscription* pernotSubCacheAdd -( - const char* subscriptionId, - KjNode* subP, - QNode* qTree, - KjNode* geoCoordinatesP, - OrionldContext* contextP, - const char* tenant, - KjNode* showChangesP, - KjNode* sysAttrsP, - RenderFormat renderFormat -) -{ - return NULL; -} - - - // ----------------------------------------------------------------------------- // // pernotSubCacheRemove - // -bool pernotSubCacheRemove(PernotSubCache* pSubP) +bool pernotSubCacheRemove(PernotSubscription* pSubP) { return true; } diff --git a/src/lib/orionld/pernot/PernotSubCache.h b/src/lib/orionld/pernot/PernotSubCache.h index 8ff6440f44..ac359ca959 100644 --- a/src/lib/orionld/pernot/PernotSubCache.h +++ b/src/lib/orionld/pernot/PernotSubCache.h @@ -25,6 +25,8 @@ * * Author: Ken Zangelin */ +#include // sem_t + #include "orionld/pernot/PernotSubscription.h" // PernotSubscription @@ -37,6 +39,7 @@ typedef struct PernotSubCache { PernotSubscription* head; PernotSubscription* tail; + int newSubs; } PernotSubCache; #endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSUBCACHE_H_ diff --git a/src/lib/orionld/pernot/PernotSubscription.h b/src/lib/orionld/pernot/PernotSubscription.h index d808e9f80e..c36ac5d5ef 100644 --- a/src/lib/orionld/pernot/PernotSubscription.h +++ b/src/lib/orionld/pernot/PernotSubscription.h @@ -33,6 +33,12 @@ extern "C" #include "kjson/KjNode.h" // KjNode } +#include "common/RenderFormat.h" // RenderFormat +#include "common/MimeType.h" // MimeType + +#include "orionld/types/Protocol.h" // Protocol +#include "orionld/types/StringArray.h" // StringArray +#include "orionld/types/OrionldTenant.h" // OrionldTenant @@ -58,25 +64,51 @@ typedef struct PernotSubscription { char* subscriptionId; PernotState state; - char tenant[64]; + double timeInterval; // In seconds KjNode* kjSubP; // OR, I split the entire tree into binary fields inside PernotSubscription ... + OrionldTenant* tenantP; + + // Cached from KjSubP + char* lang; + char* context; + bool isActive; // Timestamps - double lastNotificationAttempt; // In seconds + double lastNotificationTime; // In seconds double lastSuccessTime; double lastFailureTime; double expiresAt; // Counters - uint32_t dbNotificationAttempts; // Total number of notification attempts, in DB + uint32_t notificationAttemptsDb; // Total number of notification attempts, in DB uint32_t notificationAttempts; // Total number of notification attempts, in cache (to be added to dbCount) - uint32_t dbNotificationErrors; // Total number of FAILED notification attempts, in DB + uint32_t notificationErrorsDb; // Total number of FAILED notification attempts, in DB uint32_t notificationErrors; // Total number of FAILED notification attempts, in cache (to be added to dbNotificationErrors) + uint32_t noMatchDb; // Total number of attempts without any matching results of the entities query, in DB + uint32_t noMatch; // Total number of attempts without any matching results of the entities query, in cache (added to dbNoMatch) + uint32_t dirty; // Counter of number of cache counters/timestamps updates since last flush to DB - // Error handling + // URL + char url[512]; // parsed and destroyed - can't be used (protocolString, ip, and rest points inside this buffer) + char* protocolString; // pointing to 'protocol' part of 'url' + char* ip; // pointing to 'ip' part of 'url' + unsigned short port; // port, as parsed from 'url' + char* rest; // pointing to 'rest' part of 'url' + Protocol protocol; + + // HTTP headers for the notification + StringArray headers; + + // Notification Format/Details + bool ngsiv2; + bool sysAttrs; + RenderFormat renderFormat; + MimeType mimeType; + + // Errors uint32_t consecutiveErrors; - double timeInterval; // In seconds uint32_t cooldown; + char* lastErrorReason; CURL* curlHandle; diff --git a/src/lib/orionld/pernot/pernotLoop.cpp b/src/lib/orionld/pernot/pernotLoop.cpp index 73f59e6fc7..7f9c903faf 100644 --- a/src/lib/orionld/pernot/pernotLoop.cpp +++ b/src/lib/orionld/pernot/pernotLoop.cpp @@ -35,6 +35,7 @@ extern "C" #include "logMsg/logMsg.h" // LM_x #include "orionld/common/orionldState.h" // orionldState, pernotSubCache +#include "orionld/mongoc/mongocSubCountersUpdate.h" // mongocSubCountersUpdate #include "orionld/pernot/PernotSubscription.h" // PernotSubscription #include "orionld/pernot/PernotSubCache.h" // PernotSubCache #include "orionld/pernot/pernotTreat.h" // pernotTreat @@ -44,28 +45,63 @@ extern "C" // ----------------------------------------------------------------------------- // -// pernotSubInsert - move to pernot/pernotSubInsert.h/cpp +// currentTime - // -bool pernotSubInsert(void) +double currentTime(void) { - return true; + // int gettimeofday(struct timeval *tv, struct timezone *tz); + struct timeval tv; + + if (gettimeofday(&tv, NULL) != 0) + LM_RE(0, ("gettimeofday error: %s", strerror(errno))); + + return tv.tv_sec + tv.tv_usec / 1000000.0; } // ----------------------------------------------------------------------------- // -// currentTime - +// pernotSubCacheFlushToDb - // -double currentTime(void) +void pernotSubCacheFlushToDb(void) { - // int gettimeofday(struct timeval *tv, struct timezone *tz); - struct timeval tv; + for (PernotSubscription* subP = pernotSubCache.head; subP != NULL; subP = subP->next) + { + if (subP->dirty == false) + continue; + + LM_T(LmtPernotFlush, ("%s: flushing to db (noMatch=%d, notificationAttempts=%d)", subP->subscriptionId, subP->noMatch, subP->notificationAttempts)); + mongocSubCountersUpdate(subP->tenantP, + subP->subscriptionId, + true, + subP->notificationAttempts, + subP->notificationErrors, + subP->noMatch, + subP->lastNotificationTime, + subP->lastSuccessTime, + subP->lastFailureTime, + false); + + // Reset counters + subP->dirty = 0; + subP->notificationAttemptsDb += subP->notificationAttempts; + subP->notificationAttempts = 0; + subP->notificationErrorsDb += subP->notificationErrors; + subP->notificationErrors = 0; + subP->noMatchDb += subP->noMatch; + subP->noMatch = 0; + } +} - if (gettimeofday(&tv, NULL) != 0) - LM_RE(0, ("gettimeofday error: %s", strerror(errno))); - return tv.tv_sec + tv.tv_usec / 1000000.0; + +// ----------------------------------------------------------------------------- +// +// pernotSubCacheRefresh - +// +void pernotSubCacheRefresh(void) +{ } @@ -76,19 +112,25 @@ double currentTime(void) // static void* pernotLoop(void* vP) { + double nextFlushAt = currentTime() + subCacheFlushInterval; + double nextCacheRefreshAt = currentTime() + subCacheInterval; + double now = 0; + while (1) { - double minDiff; - - minDiff = 1; // Initialize with 1 second, which is the MAX sleep period - for (PernotSubscription* subP = pernotSubCache.head; subP != NULL; subP = subP->next) { - double now = currentTime(); + now = currentTime(); if (subP->state == SubPaused) { - LM_T(LmtPernot, ("%s: Paused", subP->subscriptionId)); + LM_T(LmtPernotLoop, ("%s: Paused", subP->subscriptionId)); + continue; + } + + if (subP->isActive == false) + { + LM_T(LmtPernotLoop, ("%s: Inactive", subP->subscriptionId)); continue; } @@ -97,7 +139,7 @@ static void* pernotLoop(void* vP) if (subP->state == SubExpired) { - LM_T(LmtPernot, ("%s: Expired", subP->subscriptionId)); + LM_T(LmtPernotLoop, ("%s: Expired", subP->subscriptionId)); continue; } @@ -108,34 +150,47 @@ static void* pernotLoop(void* vP) subP->state = SubActive; else { - LM_T(LmtPernot, ("%s: Erroneous", subP->subscriptionId)); + LM_T(LmtPernotLoop, ("%s: Erroneous", subP->subscriptionId)); continue; } } - double diffTime = subP->lastNotificationAttempt + subP->timeInterval - now; - // LM_T(LmtPernot, ("%s: lastNotificationAttempt: %f", subP->subscriptionId, subP->lastNotificationAttempt)); - // LM_T(LmtPernot, ("%s: timeInterval: %f", subP->subscriptionId, subP->timeInterval)); - // LM_T(LmtPernot, ("%s: now: %f", subP->subscriptionId, now)); - // LM_T(LmtPernot, ("%s: diffTime: %f", subP->subscriptionId, diffTime)); + double diffTime = subP->lastNotificationTime + subP->timeInterval - now; + LM_T(LmtPernotLoop, ("%s: lastNotificationTime: %f", subP->subscriptionId, subP->lastNotificationTime)); + LM_T(LmtPernotLoop, ("%s: now: %f", subP->subscriptionId, now)); + LM_T(LmtPernotLoop, ("%s: diffTime: %f", subP->subscriptionId, diffTime)); + LM_T(LmtPernotLoop, ("%s: noMatch: %d", subP->subscriptionId, subP->noMatch)); + LM_T(LmtPernotLoop, ("%s: noMatchDb: %d", subP->subscriptionId, subP->noMatchDb)); + LM_T(LmtPernotLoop, ("%s: notificationAttempts: %d", subP->subscriptionId, subP->notificationAttempts)); + LM_T(LmtPernotLoop, ("%s: notificationAttemptsDb: %d", subP->subscriptionId, subP->notificationAttemptsDb)); if (diffTime <= 0) { - subP->lastNotificationAttempt = now; // Either it works or fails, the timestamp needs to be updated - pernotTreatStart(subP); // Query runs in a new thread - - // Restart the min diff time - minDiff = 0; - break; + LM_T(LmtPernotLoop, ("%s: ---------- Sending notification at %f", subP->subscriptionId, now)); + subP->lastNotificationTime = now; // Either it works or fails, the timestamp needs to be updated (it's part of the loop) + pernotTreatStart(subP); // Query runs in a new thread, loop continues } - else - minDiff = K_MIN(minDiff, diffTime); } - if (minDiff != 0) - usleep(minDiff * 1000000); + if ((subCacheFlushInterval > 0) && (now > nextFlushAt)) + { + LM_T(LmtPernotLoop, ("Flushing Pernot SubCache contents to DB")); + pernotSubCacheFlushToDb(); + nextFlushAt += subCacheFlushInterval; + } + else if ((subCacheInterval > 0) && (now > nextCacheRefreshAt)) + { + LM_T(LmtPernotLoop, ("Refreshing Pernot SubCache contents from DB")); + pernotSubCacheRefresh(); + nextCacheRefreshAt += subCacheInterval; + } + else + { + // LM_T(LmtPernotLoop, ("Sleeping 50ms")); + usleep(50000); // We always end up here if there are no subscriptions in the cache + } } - + LM_T(LmtPernotLoop, ("End of loop")); return NULL; } diff --git a/src/lib/orionld/pernot/pernotSend.cpp b/src/lib/orionld/pernot/pernotSend.cpp new file mode 100644 index 0000000000..8b42cc8e28 --- /dev/null +++ b/src/lib/orionld/pernot/pernotSend.cpp @@ -0,0 +1,237 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +extern "C" +{ +#include "kjson/KjNode.h" // KjNode +#include "kjson/kjRenderSize.h" // kjFastRenderSize +#include "kjson/kjRender.h" // kjFastRender +#include "kjson/kjBuilder.h" // kjObject, kjArray, kjString, kjChildAdd, ... +} + +#include "logMsg/logMsg.h" // LM_* + +#include "orionld/common/orionldState.h" // orionldState +#include "orionld/common/numberToDate.h" // numberToDate +#include "orionld/common/uuidGenerate.h" // uuidGenerate +#include "orionld/common/tenantList.h" // tenant0 +#include "orionld/context/orionldCoreContext.h" // ORIONLD_CORE_CONTEXT_URL_V1_0 +#include "orionld/notifications/httpNotify.h" // httpNotify +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/pernotSend.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// Fixed value headers - from notifications/notificationSend.cpp +// +extern const char* contentTypeHeaderJson; +extern const char* contentTypeHeaderJsonLd; +extern const char* contentTypeHeaderGeoJson; +extern const char* acceptHeader; +extern const char* normalizedHeader; +extern const char* conciseHeader; +extern const char* simplifiedHeader; +extern const char* normalizedHeaderNgsiV2; +extern const char* keyValuesHeaderNgsiV2; + + + +// ----------------------------------------------------------------------------- +// +// notificationTree - +// +static KjNode* notificationTree(PernotSubscription* subP, KjNode* entityArray) +{ + KjNode* notificationP = kjObject(orionldState.kjsonP, NULL); + char notificationId[80]; + + strncpy(notificationId, "urn:ngsi-ld:Notification:", sizeof(notificationId) - 1); // notificationId, could be a thread variable ... + uuidGenerate(¬ificationId[25], sizeof(notificationId) - 25, false); + + char date[64]; + numberToDate(subP->lastNotificationTime, date, sizeof(date)); + KjNode* idNodeP = kjString(orionldState.kjsonP, "id", notificationId); + KjNode* typeNodeP = kjString(orionldState.kjsonP, "type", "Notification"); + KjNode* subscriptionIdNodeP = kjString(orionldState.kjsonP, "subscriptionId", subP->subscriptionId); + KjNode* notifiedAtNodeP = kjString(orionldState.kjsonP, "notifiedAt", date); + + entityArray->name = (char*) "data"; + + kjChildAdd(notificationP, idNodeP); + kjChildAdd(notificationP, typeNodeP); + kjChildAdd(notificationP, subscriptionIdNodeP); + kjChildAdd(notificationP, notifiedAtNodeP); + kjChildAdd(notificationP, entityArray); + + return notificationP; +} + + + +// ----------------------------------------------------------------------------- +// +// notificationTreeForNgsiV2 - +// +static KjNode* notificationTreeForNgsiV2(PernotSubscription* subP, KjNode* entityArray) +{ + LM_X(1, ("Pernot subs in NGSIv2 format is not implemented, how did we get here???")); + return NULL; +} + + + +// ----------------------------------------------------------------------------- +// +// pernotSend - +// +bool pernotSend(PernotSubscription* subP, KjNode* entityArray) +{ + // + // Outgoing Payload Body + // + char body[2 * 1024]; + KjNode* notificationP = (subP->ngsiv2 == false)? notificationTree(subP, entityArray) : notificationTreeForNgsiV2(subP, entityArray); + long unsigned int payloadBodySize = kjFastRenderSize(notificationP); + char* payloadBody = (payloadBodySize < sizeof(body))? body : kaAlloc(&orionldState.kalloc, payloadBodySize); + + kjFastRender(notificationP, payloadBody); + + // Assuming HTTP for now + + // + // Outgoing Header + // + char requestHeader[512]; + size_t requestHeaderLen = 0; + + if (subP->renderFormat < RF_CROSS_APIS_NORMALIZED) + requestHeaderLen = snprintf(requestHeader, sizeof(requestHeader), "POST /%s?subscriptionId=%s HTTP/1.1\r\n", + subP->rest, + subP->subscriptionId); + + // + // Content-Length + // + char contentLenHeader[48]; + size_t contentLenHeaderLen; + int contentLen = strlen(payloadBody); + + contentLenHeaderLen = snprintf(contentLenHeader, sizeof(contentLenHeader) - 1, "Content-Length: %d\r\n", contentLen); + + + int headers = 7; // the minimum number of request headers + + // + // Headers from Subscription::notification::endpoint::receiverInfo+headers (or custom notification in NGSIv2 ...) + // + headers += subP->headers.items; + + char hostHeader[128]; + size_t hostHeaderLen = snprintf(hostHeader, sizeof(hostHeader) - 1, "Host: %s\r\n", subP->ip); + + int ioVecLen = headers + 3; // Request line + X headers + empty line + payload body + struct iovec ioVec[53] = { + { requestHeader, requestHeaderLen }, + { contentLenHeader, contentLenHeaderLen }, + { (void*) contentTypeHeaderJson, 32 }, // Index 2 + { (void*) userAgentHeader, userAgentHeaderLen }, + { (void*) hostHeader, hostHeaderLen }, + { (void*) acceptHeader, 26 }, + { (void*) normalizedHeader, 37 } // Index 6 + }; + int headerIx = 7; + bool addLinkHeader = true; + + if (subP->mimeType == JSONLD) // If Content-Type is application/ld+json, modify slot 2 of ioVec + { + ioVec[2].iov_base = (void*) contentTypeHeaderJsonLd; // REPLACE "application/json" with "application/ld+json" + ioVec[2].iov_len = 35; + addLinkHeader = false; + } + + if ((addLinkHeader == true) && (subP->ngsiv2 == false)) // Add Link header - but not if NGSIv2 Cross Notification + { + char linkHeader[512]; + const char* link = (subP->context == NULL)? ORIONLD_CORE_CONTEXT_URL_V1_0 : subP->context; + + snprintf(linkHeader, sizeof(linkHeader), "Link: <%s>; rel=\"http://www.w3.org/ns/json-ld#context\"; type=\"application/ld+json\"\r\n", link); + + ioVec[headerIx].iov_base = linkHeader; + ioVec[headerIx].iov_len = strlen(linkHeader); + ++headerIx; + } + + // FIXME: Loop over subP->headers.array, and add those to ioVec + for (int ix = 0; ix < subP->headers.items; ix++) + { + ioVec[headerIx].iov_base = subP->headers.array[ix]; + ioVec[headerIx].iov_len = strlen(subP->headers.array[ix]); + ++headerIx; + } + + // + // Ngsild-Tenant + // + if ((subP->tenantP != NULL) && (subP->tenantP != &tenant0) && (subP->ngsiv2 == false)) + { + int len = strlen(subP->tenantP->tenant) + 20; // Ngsild-Tenant: xxx\r\n0 - '\r' seems to not count for strlen ... + char* buf = kaAlloc(&orionldState.kalloc, len); + + ioVec[headerIx].iov_len = snprintf(buf, len, "Ngsild-Tenant: %s\r\n", subP->tenantP->tenant); + ioVec[headerIx].iov_base = buf; + + ++headerIx; + } + + // Empty line delimiting HTTP Headers and Payload Body + ioVec[headerIx].iov_base = (void*) "\r\n"; + ioVec[headerIx].iov_len = 2; + ++headerIx; + + + // Payload Body + ioVec[headerIx].iov_base = payloadBody; + ioVec[headerIx].iov_len = contentLen; + + ioVecLen = headerIx + 1; + + // + // The message is ready - just need to be sent + // + if (subP->protocol == HTTP) + { + LM_T(LmtPernot, ("Sending a Periodic Notification to %s:%d%s", subP->ip, subP->port, subP->rest)); + return httpNotify(NULL, subP, subP->subscriptionId, subP->ip, subP->port, subP->rest, ioVec, ioVecLen, subP->lastNotificationTime); + } +#if 0 + else if (subP->protocol == HTTPS) return httpsNotify(subP, ioVec, ioVecLen, now, curlHandlePP); + else if (subP->protocol == MQTT) return mqttNotify(subP, ioVec, ioVecLen, now); +#endif + + LM_W(("%s: Unsupported protocol for notifications: '%s'", subP->subscriptionId, subP->protocol)); + + return false; +} diff --git a/src/lib/orionld/pernot/pernotSend.h b/src/lib/orionld/pernot/pernotSend.h new file mode 100644 index 0000000000..ecf8922cbb --- /dev/null +++ b/src/lib/orionld/pernot/pernotSend.h @@ -0,0 +1,43 @@ +#ifndef SRC_LIB_ORIONLD_PERNOT_PERNOTSEND_H_ +#define SRC_LIB_ORIONLD_PERNOT_PERNOTSEND_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +extern "C" +{ +#include "kjson/KjNode.h" // KjNode +} + +#include "orionld/pernot/PernotSubscription.h" // PernotSubscription + + + +// ----------------------------------------------------------------------------- +// +// pernotSend - +// +extern bool pernotSend(PernotSubscription* subP, KjNode* entityArray); + +#endif // SRC_LIB_ORIONLD_PERNOT_PERNOTSEND_H_ diff --git a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp index 1e619902ef..5100340b58 100644 --- a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp +++ b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp @@ -22,6 +22,9 @@ * * Author: Ken Zangelin */ +#include // malloc +#include // sem_post + extern "C" { #include "kjson/KjNode.h" // KjNode @@ -34,14 +37,92 @@ extern "C" #include "common/RenderFormat.h" // RenderFormat -#include "orionld/common/orionldState.h" // pernotSubCache +#include "orionld/common/orionldState.h" // orionldState, pernotSubCache +#include "orionld/common/urlParse.h" // urlParse +#include "orionld/types/Protocol.h" // Protocol, protocolFromString +#include "orionld/types/OrionldTenant.h" // OrionldTenant #include "orionld/q/QNode.h" // QNode +#include "orionld/kjTree/kjChildCount.h" // kjChildCount #include "orionld/context/OrionldContext.h" // OrionldContext #include "orionld/pernot/PernotSubscription.h" // PernotSubscription #include "orionld/pernot/PernotSubCache.h" // PernotSubCache +// ----------------------------------------------------------------------------- +// +// receiverInfo - +// +// As there is no incoming info for the HTTP headers of Pernot subscriptions, we can +// safely create the entire string of 'key: value' for all the headers. +// +static void receiverInfo(PernotSubscription* pSubP, KjNode* endpointP) +{ + KjNode* receiverInfoP = kjLookup(endpointP, "receiverInfo"); + + pSubP->headers.items = 0; + + if (receiverInfoP != NULL) + { + pSubP->headers.items = kjChildCount(receiverInfoP); + pSubP->headers.array = (char**) malloc(pSubP->headers.items * sizeof(char*)); + + int ix = 0; + for (KjNode* kvPairP = receiverInfoP->value.firstChildP; kvPairP != NULL; kvPairP = kvPairP->next) + { + KjNode* keyP = kjLookup(kvPairP, "key"); + KjNode* valueP = kjLookup(kvPairP, "value"); + int sLen = strlen(keyP->value.s) + strlen(valueP->value.s) + 4; + char* s = (char*) malloc(sLen); + + snprintf(s, sLen - 1, "%s:%s\r\n", keyP->value.s, valueP->value.s); + pSubP->headers.array[ix] = s; + ++ix; + } + } +} + + + +// ----------------------------------------------------------------------------- +// +// counterFromDb - +// +static void counterFromDb(KjNode* subP, uint32_t* counterP, const char* fieldName) +{ + LM_T(LmtPernot, ("Getting counter '%s' from db", fieldName)); + KjNode* counterNodeP = kjLookup(subP, fieldName); + + if (counterNodeP != NULL) + { + LM_T(LmtPernot, ("Found counter '%s' in db: %d", fieldName, counterNodeP->value.i)); + *counterP = counterNodeP->value.i; + } + else + *counterP = 0; +} + + + +// ----------------------------------------------------------------------------- +// +// timestampFromDb - +// +static void timestampFromDb(KjNode* apiSubP, double* tsP, const char* fieldName) +{ + KjNode* tsNodeP = kjLookup(apiSubP, fieldName); + if (tsNodeP != NULL) + { + // Need to remove it from apiSubP and save it in cache + kjChildRemove(apiSubP, tsNodeP); + *tsP = tsNodeP->value.f; + } + else + *tsP = 0; +} + + + // ----------------------------------------------------------------------------- // // pernotSubCacheAdd - @@ -54,7 +135,7 @@ PernotSubscription* pernotSubCacheAdd QNode* qTree, KjNode* geoCoordinatesP, OrionldContext* contextP, - const char* tenant, + OrionldTenant* tenantP, KjNode* showChangesP, KjNode* sysAttrsP, RenderFormat renderFormat, @@ -62,6 +143,9 @@ PernotSubscription* pernotSubCacheAdd ) { PernotSubscription* pSubP = (PernotSubscription*) malloc(sizeof(PernotSubscription)); + bzero(pSubP, sizeof(PernotSubscription)); + + kjTreeLog(apiSubP, "API Subscription from DB", LmtPernot); if (subscriptionId == NULL) { @@ -76,26 +160,63 @@ PernotSubscription* pernotSubCacheAdd pSubP->subscriptionId = strdup(subscriptionId); pSubP->timeInterval = timeInterval; pSubP->kjSubP = kjClone(NULL, apiSubP); + pSubP->tenantP = tenantP; + pSubP->renderFormat = renderFormat; + pSubP->sysAttrs = (sysAttrsP == NULL)? false : sysAttrsP->value.b; + pSubP->ngsiv2 = (renderFormat >= RF_CROSS_APIS_NORMALIZED); + pSubP->context = (contextP == NULL)? NULL : contextP->url; + pSubP->isActive = true; // Active by default, then we'll see ... + + // notification + KjNode* notificationP = kjLookup(pSubP->kjSubP, "notification"); + + // notification::endpoint + if (endpointP == NULL) + endpointP = kjLookup(notificationP, "endpoint"); + + // URL + KjNode* uriP = kjLookup(endpointP, "uri"); + strncpy(pSubP->url, uriP->value.s, sizeof(pSubP->url) - 1); + urlParse(pSubP->url, &pSubP->protocolString, &pSubP->ip, &pSubP->port, &pSubP->rest); + pSubP->protocol = protocolFromString(pSubP->protocolString); kjTreeLog(pSubP->kjSubP, "Initial Pernot Subscription", LmtPernot); - if (tenant != NULL) - strncpy(pSubP->tenant, tenant, sizeof(pSubP->tenant) - 1); + // Mime Type for notifications + KjNode* acceptP = kjLookup(endpointP, "accept"); + pSubP->mimeType = JSON; // Default setting + if (acceptP != NULL) + { + if (strcmp(acceptP->value.s, "application/json") == 0) pSubP->mimeType = JSON; + else if (strcmp(acceptP->value.s, "application/ld+json") == 0) pSubP->mimeType = JSONLD; + else if (strcmp(acceptP->value.s, "application/geo+json") == 0) pSubP->mimeType = GEOJSON; + } + + // HTTP headers from receiverInfo + receiverInfo(pSubP, endpointP); // State - check also expiresAt+status KjNode* isActiveNodeP = kjLookup(apiSubP, "isActive"); if (isActiveNodeP != NULL) pSubP->state = (isActiveNodeP->value.b == true)? SubActive : SubPaused; + else + pSubP->state = SubActive; + + // Counters + counterFromDb(apiSubP, &pSubP->notificationAttemptsDb, "count"); + counterFromDb(apiSubP, &pSubP->notificationErrorsDb, "timesFailed"); + counterFromDb(apiSubP, &pSubP->noMatchDb, "noMatch"); + + pSubP->notificationAttempts = 0; // cached - added to notificationAttemptsDb + pSubP->notificationErrors = 0; // cached - added to notificationErrorsDb + pSubP->noMatch = 0; // cached - added to noMatchDb + pSubP->consecutiveErrors = 0; // only cached, not saved in DB + + // Timestamps + timestampFromDb(apiSubP, &pSubP->lastNotificationTime, "lastNotification"); + timestampFromDb(apiSubP, &pSubP->lastSuccessTime, "lastSuccess"); + timestampFromDb(apiSubP, &pSubP->lastFailureTime, "lastFailure"); - pSubP->lastNotificationAttempt = 0; // FIXME: get info from apiSubP - pSubP->lastSuccessTime = 0; // FIXME: get info from apiSubP - pSubP->lastFailureTime = 0; // FIXME: get info from apiSubP - pSubP->expiresAt = 0; // FIXME: get info from apiSubP - pSubP->dbNotificationAttempts = 0; // FIXME: get info from apiSubP - pSubP->notificationAttempts = 0; - pSubP->dbNotificationErrors = 0; // FIXME: get info from apiSubP - pSubP->notificationErrors = 0; - pSubP->consecutiveErrors = 0; // FIXME: get info from apiSubP pSubP->cooldown = 0; // FIXME: get info from apiSubP pSubP->curlHandle = NULL; @@ -104,6 +225,7 @@ PernotSubscription* pernotSubCacheAdd // pSubP->next = NULL; + // Take semaphore ... if (pernotSubCache.head == NULL) { pernotSubCache.head = pSubP; @@ -154,9 +276,6 @@ PernotSubscription* pernotSubCacheAdd KjNode* originP = kjString(NULL, "origin", "cache"); kjChildAdd(pSubP->kjSubP, originP); - // notification - KjNode* notificationP = kjLookup(pSubP->kjSubP, "notification"); - // notification::format KjNode* formatP = kjLookup(notificationP, "format"); if (formatP == NULL) @@ -165,12 +284,7 @@ PernotSubscription* pernotSubCacheAdd kjChildAdd(notificationP, formatP); } - // notification::endpoint - if (endpointP == NULL) - endpointP = kjLookup(notificationP, "endpoint"); - // notification::endpoint::accept - KjNode* acceptP = kjLookup(endpointP, "accept"); if (acceptP == NULL) { acceptP = kjString(NULL, "accept", "application/json"); @@ -179,5 +293,11 @@ PernotSubscription* pernotSubCacheAdd kjTreeLog(pSubP->kjSubP, "Pernot Subscription in Cache", LmtPernot); + // + // Caching stuff from the KjNode tree + // + KjNode* langP = kjLookup(pSubP->kjSubP, "lang"); + pSubP->lang = (langP != NULL)? langP->value.s : NULL; + return NULL; } diff --git a/src/lib/orionld/pernot/pernotSubCacheAdd.h b/src/lib/orionld/pernot/pernotSubCacheAdd.h index 525ed75056..f073ef7567 100644 --- a/src/lib/orionld/pernot/pernotSubCacheAdd.h +++ b/src/lib/orionld/pernot/pernotSubCacheAdd.h @@ -32,9 +32,9 @@ extern "C" #include "common/RenderFormat.h" // RenderFormat -#include "orionld/common/orionldState.h" // pernotSubCache -#include "orionld/q/QNode.h" // QNode +#include "orionld/types/OrionldTenant.h" // OrionldTenant #include "orionld/context/OrionldContext.h" // OrionldContext +#include "orionld/q/QNode.h" // QNode #include "orionld/pernot/PernotSubscription.h" // PernotSubscription #include "orionld/pernot/PernotSubCache.h" // PernotSubCache #include "orionld/pernot/PernotSubscription.h" // PernotSubscription @@ -53,7 +53,7 @@ extern PernotSubscription* pernotSubCacheAdd QNode* qTree, KjNode* geoCoordinatesP, OrionldContext* contextP, - const char* tenant, + OrionldTenant* tenantP, KjNode* showChangesP, KjNode* sysAttrsP, RenderFormat renderFormat, diff --git a/src/lib/orionld/pernot/pernotSubCacheLookup.cpp b/src/lib/orionld/pernot/pernotSubCacheLookup.cpp index 07b3566872..abe0f67ca8 100644 --- a/src/lib/orionld/pernot/pernotSubCacheLookup.cpp +++ b/src/lib/orionld/pernot/pernotSubCacheLookup.cpp @@ -41,7 +41,7 @@ PernotSubscription* pernotSubCacheLookup { for (PernotSubscription* pSubP = pernotSubCache.head; pSubP != NULL; pSubP = pSubP->next) { - if ((strcmp(pSubP->subscriptionId, subscriptionId) == 0) && (strcmp(pSubP->tenant, tenant) == 0)) + if ((strcmp(pSubP->subscriptionId, subscriptionId) == 0) && (strcmp(pSubP->tenantP->tenant, tenant) == 0)) return pSubP; } diff --git a/src/lib/orionld/pernot/pernotTreat.cpp b/src/lib/orionld/pernot/pernotTreat.cpp index f584e57591..ed26756333 100644 --- a/src/lib/orionld/pernot/pernotTreat.cpp +++ b/src/lib/orionld/pernot/pernotTreat.cpp @@ -26,41 +26,162 @@ extern "C" { +#include "kalloc/kaBufferInit.h" // kaBufferInit +#include "kjson/kjBufferCreate.h" // kjBufferCreate #include "kjson/KjNode.h" // KjNode +#include "kjson/kjBuilder.h" // kjArray, ... } #include "logMsg/logMsg.h" // LM_x +#include "common/RenderFormat.h" // RenderFormat #include "orionld/common/orionldState.h" // orionldState, pernotSubCache +#include "orionld/types/OrionldGeoInfo.h" // OrionldGeoInfo +#include "orionld/mongoc/mongocEntitiesQuery2.h" // mongocEntitiesQuery2 +#include "orionld/dbModel/dbModelToApiEntity.h" // dbModelToApiEntity2 #include "orionld/pernot/PernotSubscription.h" // PernotSubscription +#include "orionld/pernot/pernotSend.h" // pernotSend #include "orionld/pernot/pernotTreat.h" // Own interface -extern double currentTime(void); // FIXME: Move to orionld/common + +// ----------------------------------------------------------------------------- +// +// dbModelToApiEntities - FIXME: Move to orionld/dbModel +// +KjNode* dbModelToApiEntities(KjNode* dbEntityArray, bool sysAttrs, RenderFormat renderFormat, char* lang) +{ + KjNode* apiEntityArray = kjArray(orionldState.kjsonP, NULL); + + for (KjNode* dbEntityP = dbEntityArray->value.firstChildP; dbEntityP != NULL; dbEntityP = dbEntityP->next) + { + OrionldProblemDetails pd; + KjNode* apiEntityP = dbModelToApiEntity2(dbEntityP, sysAttrs, renderFormat, lang, true, &pd); + + if (apiEntityP == NULL) + LM_E(("dbModelToApiEntity: %s: %s", pd.title, pd.detail)); + else + kjChildAdd(apiEntityArray, apiEntityP); + } + + return apiEntityArray; +} + + + // ----------------------------------------------------------------------------- // // pernotTreat - // +// New Fields for Pernot Subscription: +// - pageSize for pagination - when too many matches +// - maxPages - to cap the total number of paginated requests +// - local=true - only local entities (might be the default setting) +// - totalNotifationMsgsSent - stats: how many notifications (including paginated messages) have been sent +// - timesNoMatch - stats: how many times did the query yield no results? +// static void* pernotTreat(void* vP) { PernotSubscription* subP = (PernotSubscription*) vP; bool ok = true; - double now = currentTime(); + char kallocBuffer[2048]; + + LM_T(LmtPernotFlush, ("In thread for one Periodic Notification Subscription (%s, %f)", subP->subscriptionId, subP->lastNotificationTime)); + + // 01. Initialize kalloc/kjson + // 02. Initialize orionldState + // 03. Prepare orionldState for the call to mongocEntitiesQuery2(): + // - orionldState.uriParams.offset (URL parameter) + // - orionldState.uriParams.limit (URL parameter) + // - orionldState.uriParams.count (URL parameter) + // - orionldState.tenantP (HTTP header) + // 04. KjNode* entitySelectorP (should be created when sub added to cache) + // 05. KjNode* attrsArray (should be created when sub added to cache) + // 06. QNode* qTree (should be created when sub added to cache) + // 07. OrionldGeoInfo* geoInfoP (should be created when sub added to cache) + // 08. char* lang (get it from subP->kjTreeP - cache pointer in PernotSubscription) + // 09. int64_t* countP + // 10. Perform the query/queries + // - First query with count + // - After that, a loop for pagination, if need be + // + + LM_T(LmtPernotLoop, ("%s: lastNotificationTime: %f", subP->subscriptionId, subP->lastNotificationTime)); - // Build the query + orionldStateInit(NULL); + bzero(kallocBuffer, sizeof(kallocBuffer)); + kaBufferInit(&kalloc, kallocBuffer, sizeof(kallocBuffer), 8 * 1024, NULL, "Pernot KAlloc buffer"); + orionldState.kjsonP = kjBufferCreate(&kjson, &kalloc); + LM_T(LmtPernot, ("Creating the query for pernot-subscription %s", subP->subscriptionId)); + KjNode* eSelector = NULL; // should be created when sub added to cache + KjNode* attrsArray = NULL; // should be created when sub added to cache + QNode* qTree = NULL; // should be created when sub added to cache + OrionldGeoInfo* geoInfoP = NULL; // should be created when sub added to cache + int64_t count = 0; + KjNode* dbEntityArray; + KjNode* apiEntityArray; + + orionldState.uriParams.offset = 0; + orionldState.uriParams.limit = 20; // Or: set in subscription + orionldState.uriParams.count = true; + orionldState.tenantP = subP->tenantP; + + LM_T(LmtPernot, ("Calling mongocEntitiesQuery2")); + dbEntityArray = mongocEntitiesQuery2(eSelector, attrsArray, qTree, geoInfoP, subP->lang, &count); + + LM_T(LmtPernot, ("mongocEntitiesQuery2 gave a count of %d", count)); + + if ((dbEntityArray == NULL) || (count == 0)) + { + LM_T(LmtPernotFlush, ("mongocEntitiesQuery2 found no matches (noMatch was %d)", subP->noMatch)); + subP->noMatch += 1; + subP->dirty += 1; + goto done; + } + + apiEntityArray = dbModelToApiEntities(dbEntityArray, subP->sysAttrs, subP->renderFormat, subP->lang); + kjTreeLog(apiEntityArray, "apiEntityArray", LmtPernot); + + if (pernotSend(subP, apiEntityArray) == false) + ok = false; + else + { + int entitiesSent = MIN(count, orionldState.uriParams.limit); + if (entitiesSent < count) + { + orionldState.uriParams.count = false; + while (entitiesSent < count) + { + orionldState.uriParams.offset = entitiesSent; + + dbEntityArray = mongocEntitiesQuery2(eSelector, attrsArray, qTree, geoInfoP, subP->lang, NULL); + apiEntityArray = dbModelToApiEntities(dbEntityArray, subP->sysAttrs, subP->renderFormat, subP->lang); + + if (pernotSend(subP, apiEntityArray) == false) + { + ok = false; + break; + } + + entitiesSent += orionldState.uriParams.limit; + } + } + } // // Timestamps and Status // if (ok == true) { - subP->lastSuccessTime = now; + LM_T(LmtPernot, ("Successful Periodic Notification")); + subP->lastSuccessTime = subP->lastNotificationTime; subP->consecutiveErrors = 0; } else { - subP->lastFailureTime = now; + LM_T(LmtPernot, ("Failed Periodic Notification")); + subP->lastFailureTime = subP->lastNotificationTime; subP->notificationErrors += 1; subP->consecutiveErrors += 1; @@ -71,8 +192,8 @@ static void* pernotTreat(void* vP) } } - subP->notificationAttempts += 1; - +done: + subP->notificationAttempts += 1; // timesSent pthread_exit(0); return NULL; } @@ -85,9 +206,10 @@ static void* pernotTreat(void* vP) // void pernotTreatStart(PernotSubscription* subP) { - pthread_t tid; + pthread_t tid; - LM_T(LmtPernot, ("Starting thread for one Periodic Notification Subscription (%s)", subP->subscriptionId)); + LM_T(LmtPernotLoop, ("Starting thread for one Periodic Notification Subscription (%s, %f)", subP->subscriptionId, subP->lastNotificationTime)); + pthread_create(&tid, NULL, pernotTreat, subP); // It's OK to lose the thread ID ... I think ... diff --git a/src/lib/orionld/q/qAliasCompact.cpp b/src/lib/orionld/q/qAliasCompact.cpp index bd9c84c49d..08478933d4 100644 --- a/src/lib/orionld/q/qAliasCompact.cpp +++ b/src/lib/orionld/q/qAliasCompact.cpp @@ -73,7 +73,7 @@ bool qAliasCompact(KjNode* qP, bool compact) if ((c0 == '(') || (c0 == ')')) { - LM_T(LmtPernot, ("Found a parenthesis - skipping it")); + LM_T(LmtQ, ("Found a parenthesis - skipping it")); ++cP; varStart = cP; out[outIx] = c0; @@ -114,7 +114,7 @@ bool qAliasCompact(KjNode* qP, bool compact) ++eqP; } - LM_T(LmtPernot, ("Compacting '%s'", varStart)); + LM_T(LmtQ, ("Compacting '%s'", varStart)); alias = orionldContextItemAliasLookup(orionldState.contextP, varStart, NULL, NULL); } else diff --git a/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp b/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp index 47e8eae1ee..9f8aeae15e 100644 --- a/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp +++ b/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp @@ -26,6 +26,7 @@ extern "C" { +#include "kalloc/kaStrdup.h" // kaStrdup #include "kjson/KjNode.h" // KjNode #include "kjson/kjBuilder.h" // kjString, kjInteger, kjChildAdd #include "kjson/kjLookup.h" // kjLookup @@ -69,9 +70,8 @@ static void subTimestampSet(KjNode* apiSubP, const char* fieldName, double value } else { - // We have to assume the current timestamp has a char buffer that is of sufficient length. - // We actually know that, as the string was initially created as a timestamp - strncpy(timestampNodeP->value.s, dateTime, 64); + timestampNodeP->type = KjString; + timestampNodeP->value.s = kaStrdup(&orionldState.kalloc, dateTime); } } @@ -111,10 +111,16 @@ void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP, PernotSubscr // 2. pSubP given // 3, None of them (look up apiSubP::subscriptionId in both cashes) // - double lastNotificationTime = -1; - double lastSuccess = -1; - double lastFailure = -1; - int timesSent = -1; + double lastNotificationTime = 0; + double lastSuccess = 0; + double lastFailure = 0; + int timesSent = 0; + int timesFailed = 0; + int noMatch = 0; + KjNode* notificationP = kjLookup(apiSubP, "notification"); + + if (notificationP == NULL) + LM_RVE(("API Subscription without a notification field !!!")); if ((cSubP == NULL) && (pSubP == NULL)) { @@ -129,20 +135,76 @@ void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP, PernotSubscr LM_RVE(("Can't find subscription '%s'", subIdP->value.s)); } + // + // Get timestamps/counters from db + // + KjNode* tsP; + + // lastNotificationTime + tsP = kjLookup(notificationP, "lastNotification"); + if (tsP != NULL) + lastNotificationTime = tsP->value.f; + + // lastSuccess + tsP = kjLookup(notificationP, "lastSuccess"); + if (tsP != NULL) + lastSuccess = tsP->value.f; + + // lastFailure + tsP = kjLookup(notificationP, "lastFailure"); + if (tsP != NULL) + lastFailure = tsP->value.f; + + // timesSent + tsP = kjLookup(notificationP, "count"); + if (tsP != NULL) + timesSent = tsP->value.i; + + // timesFailed + tsP = kjLookup(notificationP, "timesFailed"); + if (tsP != NULL) + timesFailed = tsP->value.i; + + // noMatch + tsP = kjLookup(notificationP, "noMatch"); + if (tsP != NULL) + noMatch = tsP->value.i; + + LM_T(LmtSR, ("DB lastNotificationTime: %f", lastNotificationTime)); + LM_T(LmtSR, ("DB lastSuccess: %f", lastSuccess)); + LM_T(LmtSR, ("DB lastFailure: %f", lastFailure)); + LM_T(LmtSR, ("DB timesSent: %d", timesSent)); + LM_T(LmtSR, ("DB timesFailed: %d", timesFailed)); + LM_T(LmtSR, ("DB noMatch: %d", noMatch)); + + // Overwrite timestamps/counters from cache if (cSubP != NULL) { - lastNotificationTime = cSubP->lastNotificationTime; - lastSuccess = cSubP->lastSuccess; - lastFailure = cSubP->lastFailure; - timesSent = cSubP->dbCount + cSubP->count; + if (cSubP->lastNotificationTime > 0) lastNotificationTime = cSubP->lastNotificationTime; + if (cSubP->lastSuccess > 0) lastSuccess = cSubP->lastSuccess; + if (cSubP->lastFailure > 0) lastFailure = cSubP->lastFailure; + + timesSent += cSubP->count; + timesFailed += cSubP->failures; + noMatch = 0; } else if (pSubP != NULL) { - lastNotificationTime = pSubP->lastNotificationAttempt; - lastSuccess = pSubP->lastSuccessTime; - lastFailure = pSubP->lastFailureTime; - timesSent = pSubP->dbNotificationAttempts + pSubP->notificationAttempts; + if (pSubP->lastNotificationTime > 0) lastNotificationTime = pSubP->lastNotificationTime; + if (pSubP->lastSuccessTime > 0) lastSuccess = pSubP->lastSuccessTime; + if (pSubP->lastFailureTime > 0) lastFailure = pSubP->lastFailureTime; + + timesSent += pSubP->notificationAttempts; + timesFailed += pSubP->notificationErrors; + noMatch += pSubP->noMatch; } + + LM_T(LmtSR, ("lastNotificationTime: %f", lastNotificationTime)); + LM_T(LmtSR, ("lastSuccess: %f", lastSuccess)); + LM_T(LmtSR, ("lastFailure: %f", lastFailure)); + LM_T(LmtSR, ("timesSent: %d", timesSent)); + LM_T(LmtSR, ("timesFailed: %d", timesFailed)); + LM_T(LmtSR, ("noMatch: %d", noMatch)); // // Set the values @@ -154,6 +216,8 @@ void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP, PernotSubscr subTimestampSet(notificationNodeP, "lastSuccess", lastSuccess); subTimestampSet(notificationNodeP, "lastFailure", lastFailure); subCounterSet(notificationNodeP, "timesSent", timesSent); + subCounterSet(notificationNodeP, "timesFailed", timesFailed); + subCounterSet(notificationNodeP, "noMatch", noMatch); } } @@ -248,7 +312,7 @@ bool orionldGetSubscription(void) { orionldState.httpStatusCode = 200; orionldState.responseTree = kjTreeFromCachedSubscription(cSubP, orionldState.uriParamOptions.sysAttrs, orionldState.out.contentType == JSONLD); - + orionldSubCounters(orionldState.responseTree, cSubP, NULL); return true; } @@ -260,6 +324,8 @@ bool orionldGetSubscription(void) orionldState.httpStatusCode = 200; orionldState.responseTree = kjTreeFromPernotSubscription(pSubP, orionldState.uriParamOptions.sysAttrs, orionldState.out.contentType == JSONLD); kjTreeLog(orionldState.responseTree, "pernot sub after kjTreeFromPernotSubscription", LmtSR); + orionldSubCounters(orionldState.responseTree, NULL, pSubP); + kjTreeLog(orionldState.responseTree, "pernot sub after orionldSubCounters", LmtSR); return true; } diff --git a/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp b/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp index faa82cef14..4f2aa383bf 100644 --- a/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp +++ b/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp @@ -113,7 +113,7 @@ bool orionldPostSubscriptions(void) &renderFormat); if (qRenderedForDb != NULL) - LM_T(LmtSR, ("qRenderedForDb: '%s'", qRenderedForDb)); + LM_T(LmtQ, ("qRenderedForDb: '%s'", qRenderedForDb)); if (b == false) { @@ -288,11 +288,15 @@ bool orionldPostSubscriptions(void) qTree, geoCoordinatesP, orionldState.contextP, - orionldState.tenantP->tenant, + orionldState.tenantP, showChangesP, sysAttrsP, renderFormat, timeInterval); + + // Signal that there's a new Pernot subscription in the cache + // ++pernotSubCache.newSubs; + // LM_T(LmtPernotLoop, ("pernotSubCache.newSubs == %d", pernotSubCache.newSubs)); } // dbModel diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test index d6d0489b90..0c6a08eae7 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test @@ -187,7 +187,7 @@ echo echo "14. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==5" echo "============================================================================" -orionCurl --url /ngsi-ld/v1/subscriptions/urn:ngsi-ld:subscriptions:S1 +orionCurl --url /ngsi-ld/v1/subscriptions/urn:ngsi-ld:subscriptions:S1?options=fromDb echo echo @@ -225,7 +225,7 @@ echo echo "18. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==7" echo "============================================================================" -orionCurl --url /ngsi-ld/v1/subscriptions/urn:ngsi-ld:subscriptions:S1 +orionCurl --url /ngsi-ld/v1/subscriptions/urn:ngsi-ld:subscriptions:S1?options=fromDb echo echo diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_timestamps.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_timestamps.test index 1c183035f7..20c2c91f0f 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_timestamps.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_timestamps.test @@ -609,7 +609,7 @@ Date: REGEX(.*) 12. GET the subscription via REST - make sure lastFailure is present and has milliseconds ========================================================================================= HTTP/1.1 200 OK -Content-Length: 820 +Content-Length: 836 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -639,6 +639,7 @@ Link: ; rel="http:/ "lastNotification": "REGEX(.*\.\d\d\dZ)", "lastSuccess": "REGEX(.*\.\d\d\dZ)", "status": "failed", + "timesFailed": 1, "timesSent": 3 }, "origin": "cache", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_notification-stats.test b/test/functionalTest/cases/0000_ngsild/ngsild_notification-stats.test index 30894af311..c94736637b 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_notification-stats.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_notification-stats.test @@ -371,7 +371,7 @@ Link: ; rel="http:/ 10. GET S3, see one failure (lastFailure + timesSent: 1, consecutiveErrors: 1) ============================================================================== HTTP/1.1 200 OK -Content-Length: 625 +Content-Length: 641 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -398,6 +398,7 @@ Link: ; rel="http:/ "lastFailure": "202REGEX(.*)Z", "lastNotification": "202REGEX(.*)Z", "status": "failed", + "timesFailed": 1, "timesSent": 1 }, "origin": "cache", @@ -410,7 +411,7 @@ Link: ; rel="http:/ 11. GET S4, see one failure (lastFailure + timesSent: 1, consecutiveErrors: 1) ============================================================================== HTTP/1.1 200 OK -Content-Length: 631 +Content-Length: 647 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -437,6 +438,7 @@ Link: ; rel="http:/ "lastFailure": "202REGEX(.*)Z", "lastNotification": "202REGEX(.*)Z", "status": "failed", + "timesFailed": 1, "timesSent": 1 }, "origin": "cache", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_notification_issue-1322.test b/test/functionalTest/cases/0000_ngsild/ngsild_notification_issue-1322.test index 4222adefbb..41855a091f 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_notification_issue-1322.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_notification_issue-1322.test @@ -124,7 +124,7 @@ NGSILD-Tenant: captn 03. GET subscription - see failure ================================== HTTP/1.1 200 OK -Content-Length: 826 +Content-Length: 842 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -156,6 +156,7 @@ Link: ; rel="http:/ "lastFailure": "202REGEX(.*)Z", "lastNotification": "202REGEX(.*)Z", "status": "failed", + "timesFailed": 1, "timesSent": 1 }, "origin": "cache", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test index 237818e3a6..15aef5a637 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test @@ -25,7 +25,7 @@ Pernot Subscription - periodic notifications --SHELL-INIT-- dbInit CB -orionldStart CB -pernot -experimental +orionldStart CB -pernot -experimental -subCacheFlushIval 3 --SHELL-- @@ -39,9 +39,11 @@ orionldStart CB -pernot -experimental # 07. See urn:S1 in the database # 08. GET urn:S1 from cache # 09. GET urn:S1 from database -# 10. Restart broker -# 11. GET urn:S1 from cache -# 12. GET urn:S1 from database +# 10. Sleep 3 secs to make sure the broker flushes the subs in cache to DB (counters and timestamps, as nothing else is changed) +# 11. See urn:S1 in database - check "noMatch == 2" +# 12. Restart broker +# 13. GET urn:S1 from cache (noMatch==3 due to initial attempt after restart) +# 14. GET urn:S1 from database # echo "01. Attempt to create a subscription with timeInterval AND watchedAttributes - see 400" @@ -166,7 +168,6 @@ echo echo - echo "06. Attempt to PATCH a pernot subscription" echo "==========================================" payload='{ @@ -197,8 +198,41 @@ orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1?options=fromDb echo echo -# To see traces from the pernot loop in t he logfile -#sleep 4 +echo '10. Sleep 3 secs to make sure the broker flushes the subs in cache to DB (counters and timestamps)' +echo '==================================================================================================' +sleep 3 +echo +echo + + +echo '11. See urn:S1 in database - check "noMatch == 2"' +echo '=================================================' +mongoCmd2 ftest "db.csubs.findOne()" +echo +echo + + +echo "12. Restart broker" +echo "==================" +brokerStop CB +orionldStart CB -pernot -experimental +echo "Broker restarted" +echo +echo + + +echo "13. GET urn:S1 from cache (noMatch==3 due to initial attempt after restart)" +echo "===========================================================================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1 +echo +echo + + +echo "14. GET urn:S1 from database" +echo "============================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1?options=fromDb +echo +echo --REGEXPECT-- @@ -337,7 +371,7 @@ bye 08. GET urn:S1 from cache ========================= HTTP/1.1 200 OK -Content-Length: 598 +Content-Length: 569 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -368,7 +402,7 @@ Link: ; rel="http:/ }, "format": "normalized", "lastNotification": "202REGEX(.*)Z", - "lastSuccess": "202REGEX(.*)Z", + "noMatch": 1, "status": "ok", "timesSent": 1 }, @@ -384,7 +418,7 @@ Link: ; rel="http:/ 09. GET urn:S1 from database ============================ HTTP/1.1 200 OK -Content-Length: 601 +Content-Length: 572 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -415,7 +449,7 @@ Link: ; rel="http:/ }, "format": "normalized", "lastNotification": "202REGEX(.*)Z", - "lastSuccess": "202REGEX(.*)Z", + "noMatch": 1, "status": "ok", "timesSent": 1 }, @@ -428,6 +462,165 @@ Link: ; rel="http:/ } +10. Sleep 3 secs to make sure the broker flushes the subs in cache to DB (counters and timestamps) +================================================================================================== + + +11. See urn:S1 in database - check "noMatch == 2" +================================================= +MongoDB shell version v4.4.23 +connecting to: mongodb://localhost:27017/ftest?compressors=disabled&gssapiServiceName=mongodb +MongoDB server version: 4.4.22 +{ + "_id" : "urn:S1", + "description" : "Test Subscription No 1", + "name" : "S1", + "entities" : [ + { + "id" : "urn:E1", + "type" : "https://uri.etsi.org/ngsi-ld/default-context/T", + "isPattern" : "false", + "isTypePattern" : false + }, + { + "id" : "urn:E.*", + "type" : "https://uri.etsi.org/ngsi-ld/default-context/T", + "isPattern" : "true", + "isTypePattern" : false + } + ], + "ldQ" : "(https://uri=etsi=org/ngsi-ld/default-context/a.value>5;https://uri=etsi=org/ngsi-ld/default-context/b.value<9)|https://uri=etsi=org/ngsi-ld/default-context/a.value<4", + "timeInterval" : 2, + "createdAt" : REGEX(.*), + "modifiedAt" : REGEX(.*), + "throttling" : 0, + "expression" : { + "geometry" : "", + "coords" : "", + "georel" : "", + "geoproperty" : "", + "q" : "P;!P", + "mq" : "P.P;!P.P" + }, + "attrs" : [ + "https://uri.etsi.org/ngsi-ld/default-context/P1", + "https://uri.etsi.org/ngsi-ld/default-context/P2" + ], + "reference" : "http://127.0.0.1:9997/notify", + "mimeType" : "application/json", + "format" : "normalized", + "conditions" : [ ], + "status" : "active", + "custom" : false, + "servicePath" : "/#", + "blacklist" : false, + "ldContext" : "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "lastNotification" : REGEX(.*), + "noMatch" : 2, + "count" : 2 +} +bye + + +12. Restart broker +================== +Broker restarted + + +13. GET urn:S1 from cache (noMatch==3 due to initial attempt after restart) +========================= +HTTP/1.1 200 OK +Content-Length: 705 +Content-Type: application/json +Date: REGEX(.*) +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + +{ + "description": "Test Subscription No 1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + }, + { + "idPattern": "urn:E.*", + "type": "T" + } + ], + "id": "urn:S1", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "lastNotification": "202REGEX(.*)Z", + "noMatch": 3, + "status": "ok", + "timesSent": 3 + }, + "origin": "cache", + "q": "(a>5;b<9)|a<4", + "status": "active", + "subscriptionName": "S1", + "timeInterval": 2, + "type": "Subscription" +} + + +14. GET urn:S1 from database +============================ +HTTP/1.1 200 OK +Content-Length: 572 +Content-Type: application/json +Date: REGEX(.*) +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + +{ + "description": "Test Subscription No 1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + }, + { + "idPattern": "urn:E.*", + "type": "T" + } + ], + "id": "urn:S1", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "lastNotification": "202REGEX(.*)Z", + "noMatch": 3, + "status": "ok", + "timesSent": 3 + }, + "origin": "database", + "q": "(a>5;b<9)|a<4", + "status": "active", + "subscriptionName": "S1", + "timeInterval": 2, + "type": "Subscription" +} + + --TEARDOWN-- brokerStop CB dbDrop CB diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_counters_and_timestamps.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_counters_and_timestamps.test new file mode 100644 index 0000000000..cc6f7d2a66 --- /dev/null +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_counters_and_timestamps.test @@ -0,0 +1,392 @@ +# Copyright 2023 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Pernot Subscription - periodic notifications and counters/timestamps + +--SHELL-INIT-- +dbInit CB +orionldStart CB -pernot -experimental -subCacheFlushIval 2 +accumulatorStart --pretty-print 127.0.0.1 ${LISTENER_PORT} + +--SHELL-- + +# +# Need to test: +# - noMatch +# - lastSuccess +# - lastFailure +# - lastNotification +# - timesSent +# - timesFailed +# - consecutiveErrors +# - lastErrorReason +# * subCacheRefresh (wait a few secs and re-query the sub) +# * restart of broker +# * dirty +# +# noMatch: +# 01. Create a pernot subscription urn:S1 with a timeInterval of 2 seconds +# 02. Sleep 100 ms to await first attempt (that resulted in no match as there are no entities) +# 03. GET urn:S1 and see noMatch == 1, and the other relevant counters/timestamps +# +# lastSuccess+lastNotification+timesSent +# 11. Create a matching entity urn:E4 +# 12. Sleep 2.1 secs to give the broker time to notify the accumulator +# 13. Dump the accumulator to see the notification +# 14. GET urn:S1 and see lastSuccess, lastNotification, no lastFailure, noMatch=1, timesSent=2, timesFailed=0 +# 15. See the subscription in the database +# +# lastFailure+timesFailed +# 21. Kill accumulator +# 22. sleep 2.1 seconds to get another pernot attempt +# 23. GET urn:S1 and see lastFailure, and timesFailed=1, consecutiveErrors=1, lastErrorReason +# +# subCacheRefresh +# 31. Sleep 2 secs to give the broker time to refresh its subCache +# 32. GET urn:S1 and see the exact same results at step 10 +# +# restart of broker +# 41. Restart the broker +# 42. GET urn:S1 and see the exact same results at step 10 (perhaps one more notification attempt) +# +# flush the cache to DB +# 51. Update urn:E4 twice and sleep a second to give the broker time to flush the sub-cache contents to DB +# 52. GET urn:S1 and see the exact same results at step 10 +# 53. See urn:S1 in DB and have the exact same numbers as step 16 +# + +echo "01. Create a pernot subscription urn:S1 with timeInterval" +echo "=========================================================" +payload='{ + "id": "urn:S1", + "type": "Subscription", + "description": "Test Subscription No 1", + "subscriptionName": "S1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + }, + { + "idPattern": "urn:E.*", + "type": "T" + } + ], + "q": "(a>5&b<9)|a<4", + "timeInterval": 2, + "notification": { + "attributes": [ "P1", "P2" ], + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "02. Sleep 100 ms to await first attempt (that resulted in no match as there are no entities)" +echo "============================================================================================" +sleep 0.1 +echo slept 100 ms +echo +echo + + +echo "03. GET urn:S1 and see noMatch == 1, and the other relevant counters/timestamps" +echo "===============================================================================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1 +echo +echo + + +echo "11. Create a matching entity urn:E4" +echo "===================================" +payload='{ + "id": "urn:E1", + "type": "T", + "a": 1, + "P1": 1 +}' +orionCurl --url /ngsi-ld/v1/entities --payload "$payload" +echo +echo + + +echo "12. Sleep 2.1 secs to give the broker time to notify the accumulator" +echo "====================================================================" +sleep 2.1 +echo Slept 2100ms +echo +echo + + +echo "13. Dump the accumulator to see the notification" +echo "================================================" +accumulatorDump +accumulatorReset +echo +echo + + +echo "14. GET urn:S1 and see lastSuccess, lastNotification, no lastFailure, noMatch=1, timesSent=2, timesFailed=0" +echo "===========================================================================================================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1 +echo +echo + + +echo "15. See the subscription in the database" +echo "========================================" +mongoCmd2 ftest "db.csubs.findOne()" +echo +echo + + +# 13. Restart the broker +# orionldStop +# orionldStart CB -pernot -experimental -subCacheIval 4 -cSubCounters 2 +# 14. GET urn:S1 and see the exact same results at step 10 (perhaps one more notification attempt) + +# flush the cache to DB +# 15. Update urn:E4 twice and sleep a second to give the broker time to flush the sub-cache contents to DB +# 16. GET urn:S1 and see the exact same results at step 10 +# 17. See urn:S1 in DB and have the exact same numbers as step 16 + + +--REGEXPECT-- +01. Create a pernot subscription urn:S1 with timeInterval +========================================================= +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:S1 + + + +02. Sleep 100 ms to await first attempt (that resulted in no match as there are no entities) +============================================================================================ +slept 100 ms + + +03. GET urn:S1 and see noMatch == 1, and the other relevant counters/timestamps +=============================================================================== +HTTP/1.1 200 OK +Content-Length: 569 +Content-Type: application/json +Date: REGEX(.*) +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + +{ + "description": "Test Subscription No 1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + }, + { + "idPattern": "urn:E.*", + "type": "T" + } + ], + "id": "urn:S1", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "lastNotification": "202REGEX(.*)Z", + "noMatch": 1, + "status": "ok", + "timesSent": 1 + }, + "origin": "cache", + "q": "(a>5;b<9)|a<4", + "status": "active", + "subscriptionName": "S1", + "timeInterval": 2, + "type": "Subscription" +} + + +11. Create a matching entity urn:E4 +=================================== +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/entities/urn:E1 + + + +12. Sleep 2.1 secs to give the broker time to notify the accumulator +==================================================================== +Slept 2100ms + + +13. Dump the accumulator to see the notification +================================================ +POST http://REGEX(.*)/notify?subscriptionId=urn:S1 +Content-Length: 263 +User-Agent: orionld/REGEX(.*) +Host: REGEX(.*) +Accept: application/json +Content-Type: application/json +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" +Ngsild-Attribute-Format: Normalized + +{ + "data": [ + { + "P1": { + "type": "Property", + "value": 1 + }, + "a": { + "type": "Property", + "value": 1 + }, + "id": "urn:E1", + "type": "T" + } + ], + "id": "urn:ngsi-ld:Notification:REGEX(.*)", + "notifiedAt": "202REGEX(.*)Z", + "subscriptionId": "urn:S1", + "type": "Notification" +} +======================================= + + +14. GET urn:S1 and see lastSuccess, lastNotification, no lastFailure, noMatch=1, timesSent=2, timesFailed=0 +=========================================================================================================== +HTTP/1.1 200 OK +Content-Length: 610 +Content-Type: application/json +Date: REGEX(.*) +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + +{ + "description": "Test Subscription No 1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + }, + { + "idPattern": "urn:E.*", + "type": "T" + } + ], + "id": "urn:S1", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "lastNotification": "202REGEX(.*)Z", + "lastSuccess": "202REGEX(.*)Z", + "noMatch": 1, + "status": "ok", + "timesSent": 2 + }, + "origin": "cache", + "q": "(a>5;b<9)|a<4", + "status": "active", + "subscriptionName": "S1", + "timeInterval": 2, + "type": "Subscription" +} + + +15. See the subscription in the database +======================================== +MongoDB shell version REGEX(.*) +connecting to: REGEX(.*) +MongoDB server version: REGEX(.*) +{ + "_id" : "urn:S1", + "description" : "Test Subscription No 1", + "name" : "S1", + "entities" : [ + { + "id" : "urn:E1", + "type" : "https://uri.etsi.org/ngsi-ld/default-context/T", + "isPattern" : "false", + "isTypePattern" : false + }, + { + "id" : "urn:E.*", + "type" : "https://uri.etsi.org/ngsi-ld/default-context/T", + "isPattern" : "true", + "isTypePattern" : false + } + ], + "ldQ" : "(https://uri=etsi=org/ngsi-ld/default-context/a.value>5;https://uri=etsi=org/ngsi-ld/default-context/b.value<9)|https://uri=etsi=org/ngsi-ld/default-context/a.value<4", + "timeInterval" : 2, + "createdAt" : REGEX(.*), + "modifiedAt" : REGEX(.*), + "throttling" : 0, + "expression" : { + "geometry" : "", + "coords" : "", + "georel" : "", + "geoproperty" : "", + "q" : "P;!P", + "mq" : "P.P;!P.P" + }, + "attrs" : [ + "https://uri.etsi.org/ngsi-ld/default-context/P1", + "https://uri.etsi.org/ngsi-ld/default-context/P2" + ], + "reference" : "http://127.0.0.1:9997/notify", + "mimeType" : "application/json", + "format" : "normalized", + "conditions" : [ ], + "status" : "active", + "custom" : false, + "servicePath" : "/#", + "blacklist" : false, + "ldContext" : "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" +} +bye + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_noMatch.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_noMatch.test new file mode 100644 index 0000000000..39b95042a9 --- /dev/null +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_noMatch.test @@ -0,0 +1,146 @@ +# Copyright 2023 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +periodic notifications and noMatch + +--SHELL-INIT-- +dbInit CB +orionldStart CB -pernot -experimental -subCacheFlushIval 1 + +--SHELL-- +# +# This test looks at subscription::notification::noMatch +# +# The broker is started without subCacheIval (no cache refresh) and a -subCacheFlushIval of 1 second + +# 01. Create a pernot subscription urn:S1 with a timeInterval of 1.40 seconds +# 02. Sleep 0.1 seconds to give the broker a chance for the initial notification attempt (it's a no-match as no entities match) +# 03. GET urnS1 from cache - see noMatch=1, and no other counters/timestamps +# 04. GET urnS1 from database - see noMatch=1, and no other counters/timestamps +# 05. See urn:S1 in mongo - see noMatch=0, and no other counters/timestamps (as no flush to DB has been done) +# +# 06. Sleep 1.0 seconds (time enough for urn:S1 to be flushed to DB, but not for a new notification attempt) +# 07. See urn:S1 in mongo - see noMatch=1, and no other counters/timestamps (no flush to DB has been done + +# 08. Sleep 2.0 more seconds (time enough for two more attempts) +# 09. GET urnS1 from cache - see noMatch=3, and no other counters/timestamps +# 10. GET urnS1 from database - see noMatch=3, and no other counters/timestamps +# 11. See urn:S1 in mongo - see noMatch=3, and no other counters/timestamps + + +echo "01. Create a pernot subscription urn:S1 with a timeInterval of 1.40 seconds" +echo "===========================================================================" +payload='{ + "id": "urn:S1", + "type": "Subscription", + "description": "Test Subscription No 1", + "subscriptionName": "S1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + } + ], + "timeInterval": 1.4, + "notification": { + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "02. Sleep 0.1 seconds to give the broker a chance for the initial notification attempt (it's a no-match as no entities match)" +echo "=============================================================================================================================" +sleep .1 +echo Slept 0.1 seconds +echo +echo + + +echo "03. GET urnS1 from cache - see noMatch=1, and no other counters/timestamps" +echo "==========================================================================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:S1 +echo +echo + + +--REGEXPECT-- +01. Create a pernot subscription urn:S1 with a timeInterval of 1.40 seconds +=========================================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:S1 + + + +02. Sleep 0.1 seconds to give the broker a chance for the initial notification attempt (it's a no-match as no entities match) +============================================================================================================================= +Slept 0.1 seconds + + +03. GET urnS1 from cache - see noMatch=1, and no other counters/timestamps +========================================================================== +HTTP/1.1 200 OK +Content-Length: 491 +Content-Type: application/json +Date: REGEX(.*) +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + +{ + "description": "Test Subscription No 1", + "entities": [ + { + "id": "urn:E1", + "type": "T" + } + ], + "id": "urn:S1", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "notification": { + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "lastNotification": "202REGEX(.*)Z", + "noMatch": 1, + "status": "ok", + "timesSent": 1 + }, + "origin": "cache", + "status": "active", + "subscriptionName": "S1", + "timeInterval": 1.4, + "type": "Subscription" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subCache-counters.test b/test/functionalTest/cases/0000_ngsild/ngsild_subCache-counters.test index 0b795e2830..4c27101ad3 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subCache-counters.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subCache-counters.test @@ -1,4 +1,4 @@ -z# Copyright 2022 FIWARE Foundation e.V. +# Copyright 2022 FIWARE Foundation e.V. # # This file is part of Orion-LD Context Broker. # diff --git a/test/functionalTest/cases/1952_sub_notification_wo_attrs/1952_sub_notification_wo_attrs.test b/test/functionalTest/cases/1952_sub_notification_wo_attrs/1952_sub_notification_wo_attrs.test index 6a80b4e305..e49805604d 100644 --- a/test/functionalTest/cases/1952_sub_notification_wo_attrs/1952_sub_notification_wo_attrs.test +++ b/test/functionalTest/cases/1952_sub_notification_wo_attrs/1952_sub_notification_wo_attrs.test @@ -132,4 +132,3 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) --TEARDOWN-- brokerStop CB dbDrop CB - From 5af7ea3ae9484e8f10116a627feaf88d9e8e645c Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sun, 27 Aug 2023 12:50:25 +0200 Subject: [PATCH 07/29] Merge with develop --- src/app/orionld/orionld.cpp | 17 +++--- .../0000_cli/bool_option_with_value.test | 1 + .../cases/0000_cli/command_line_options.test | 1 + ...ngsild_langprop-and-notifications-new.test | 4 +- ...angprop-and-notifications-with-arrays.test | 4 +- .../ngsild_langprop-and-notifications.test | 10 ++-- ...-error-for-optional-field-not-present.test | 8 +-- ...sild_new_issue_1248_q_in_subscription.test | 8 +-- ...ification_with-q-and-subcache-refresh.test | 8 +-- ...ew_notification_with-q-commalist-attr.test | 4 +- .../0000_ngsild/ngsild_new_subCache.test | 4 +- ...ld_new_subscription-cache-propagation.test | 20 +++---- .../ngsild_new_subscription-get.test | 8 +-- ...ild_new_subscription_and_q_issue_1180.test | 8 +-- ...ption_counters-for-http-notifications.test | 30 +++++----- ...tion_counters-for-https-notifications.test | 24 ++++---- ...ption_counters-for-mqtt-notifications.test | 20 +++---- .../ngsild_new_subscription_create.test | 4 +- .../ngsild_new_subscription_issue_1401.test | 6 +- ...ew_subscription_patch_and_propagation.test | 12 ++-- .../ngsild_new_subscription_patch_geoq.test | 20 +++---- ...ild_new_subscription_patch_issue_1374.test | 28 +++++----- .../ngsild_new_subscription_patch_q.test | 20 +++---- ...ubscription_patch_two_subs-issue_1368.test | 6 +- ...on_patch_update_all_individual_fields.test | 56 +++++++++---------- .../ngsild_new_subscription_timestamps.test | 24 ++++---- .../ngsild_notification-stats.test | 16 +++--- .../ngsild_notification_issue-1322.test | 4 +- ...d_patch_entity2-notification-counters.test | 8 +-- .../cases/0000_ngsild/ngsild_pernot.test | 34 +++++------ ...ngsild_pernot_counters_and_timestamps.test | 19 ++++--- .../0000_ngsild/ngsild_pernot_noMatch.test | 6 +- .../cases/0000_ngsild/ngsild_showChanges.test | 32 +++++------ .../ngsild_subCache-counters-II.test | 12 ++-- ...he-counters-with-ngsiv2-subscriptions.test | 4 +- .../0000_ngsild/ngsild_subCache-counters.test | 4 +- ...ubscription-and-patches-makes-a-crash.test | 3 +- .../ngsild_subscription_cache_issue-1316.test | 8 +-- .../ngsild_subscription_with_sysAttrs.test | 8 +-- 39 files changed, 265 insertions(+), 248 deletions(-) diff --git a/src/app/orionld/orionld.cpp b/src/app/orionld/orionld.cpp index 04ccc57c45..5e5928b3e9 100644 --- a/src/app/orionld/orionld.cpp +++ b/src/app/orionld/orionld.cpp @@ -371,7 +371,6 @@ PaArgument paArgs[] = { { "-coreContext", coreContextVersion, "CORE_CONTEXT", PaString, PaOpt, _i ORIONLD_CORE_CONTEXT_URL_DEFAULT, PaNL, PaNL, CORE_CONTEXT_DESC }, - { "-noswap", &noswap, "NOSWAP", PaBool, PaHid, false, false, true, NOSWAP_DESC }, { "-fg", &fg, "FOREGROUND", PaBool, PaOpt, false, false, true, FG_DESC }, { "-localIp", bindAddress, "LOCALIP", PaString, PaOpt, IP_ALL, PaNL, PaNL, LOCALIP_DESC }, { "-port", &port, "PORT", PaInt, PaOpt, 1026, 1024, 65535, PORT_DESC }, @@ -389,10 +388,8 @@ PaArgument paArgs[] = { "-dbTimeout", &dbTimeout, "MONGO_TIMEOUT", PaDouble, PaOpt, 10000, PaNL, PaNL, DB_TMO_DESC }, { "-dbPoolSize", &dbPoolSize, "MONGO_POOL_SIZE", PaInt, PaOpt, 10, 1, 10000, DBPS_DESC }, { "-writeConcern", &writeConcern, "MONGO_WRITE_CONCERN", PaInt, PaOpt, 1, 0, 1, WRITE_CONCERN_DESC }, - { "-idIndex", &idIndex, "MONGO_ID_INDEX", PaBool, PaHid, false, false, true, ID_INDEX_DESC }, { "-ipv4", &useOnlyIPv4, "USEIPV4", PaBool, PaOpt, false, false, true, USEIPV4_DESC }, { "-ipv6", &useOnlyIPv6, "USEIPV6", PaBool, PaOpt, false, false, true, USEIPV6_DESC }, - { "-harakiri", &harakiri, "HARAKIRI", PaBool, PaHid, false, false, true, HARAKIRI_DESC }, { "-https", &https, "HTTPS", PaBool, PaOpt, false, false, true, HTTPS_DESC }, { "-key", httpsKeyFile, "HTTPS_KEYFILE", PaString, PaOpt, _i "", PaNL, PaNL, HTTPSKEYFILE_DESC }, { "-cert", httpsCertFile, "HTTPS_CERTFILE", PaString, PaOpt, _i "", PaNL, PaNL, HTTPSCERTFILE_DESC }, @@ -429,23 +426,27 @@ PaArgument paArgs[] = { "-ngsiv1Autocast", &ngsiv1Autocast, "NGSIV1_AUTOCAST", PaBool, PaOpt, false, false, true, NGSIV1_AUTOCAST }, { "-ctxTimeout", &contextDownloadTimeout, "CONTEXT_DOWNLOAD_TIMEOUT", PaInt, PaOpt, 5000, 0, 20000, CTX_TMO_DESC }, { "-ctxAttempts", &contextDownloadAttempts, "CONTEXT_DOWNLOAD_ATTEMPTS", PaInt, PaOpt, 3, 0, 100, CTX_ATT_DESC }, - { "-troe", &troe, "TROE", PaBool, PaOpt, false, false, true, TROE_DESC }, { "-pernot", &pernot, "PERNOT", PaBool, PaOpt, false, false, true, PERNOT_DESC }, - { "-lmtmp", &lmtmp, "TMP_TRACES", PaBool, PaHid, true, false, true, TMPTRACES_DESC }, - { "-socketService", &socketService, "SOCKET_SERVICE", PaBool, PaHid, false, false, true, SOCKET_SERVICE_DESC }, + { "-troe", &troe, "TROE", PaBool, PaOpt, false, false, true, TROE_DESC }, { "-troeHost", troeHost, "TROE_HOST", PaString, PaOpt, _i "localhost", PaNL, PaNL, TROE_HOST_DESC }, { "-troePort", &troePort, "TROE_PORT", PaInt, PaOpt, 5432, PaNL, PaNL, TROE_PORT_DESC }, { "-troeUser", troeUser, "TROE_USER", PaString, PaOpt, _i "postgres", PaNL, PaNL, TROE_HOST_USER }, { "-troePwd", troePwd, "TROE_PWD", PaString, PaOpt, _i "password", PaNL, PaNL, TROE_HOST_PWD }, { "-troePoolSize", &troePoolSize, "TROE_POOL_SIZE", PaInt, PaOpt, 10, 0, 1000, TROE_POOL_DESC }, - { "-ssPort", &socketServicePort, "SOCKET_SERVICE_PORT", PaUShort, PaHid, 1027, PaNL, PaNL, SOCKET_SERVICE_PORT_DESC }, - { "-forwarding", &distributed, "FORWARDING", PaBool, PaHid, false, false, true, FORWARDING_DESC }, { "-distributed", &distributed, "DISTRIBUTED", PaBool, PaOpt, false, false, true, DISTRIBUTED_DESC }, { "-noNotifyFalseUpdate", &noNotifyFalseUpdate, "NO_NOTIFY_FALSE_UPDATE", PaBool, PaOpt, false, false, true, NO_NOTIFY_FALSE_UPDATE_DESC }, { "-experimental", &experimental, "EXPERIMENTAL", PaBool, PaOpt, false, false, true, EXPERIMENTAL_DESC }, { "-mongocOnly", &mongocOnly, "MONGOCONLY", PaBool, PaOpt, false, false, true, MONGOCONLY_DESC }, { "-cSubCounters", &cSubCounters, "CSUB_COUNTERS", PaInt, PaOpt, 20, 0, PaNL, CSUBCOUNTERS_DESC }, + { "-forwarding", &distributed, "FORWARDING", PaBool, PaHid, false, false, true, FORWARDING_DESC }, + { "-socketService", &socketService, "SOCKET_SERVICE", PaBool, PaHid, false, false, true, SOCKET_SERVICE_DESC }, + { "-ssPort", &socketServicePort, "SOCKET_SERVICE_PORT", PaUShort, PaHid, 1027, PaNL, PaNL, SOCKET_SERVICE_PORT_DESC }, + { "-harakiri", &harakiri, "HARAKIRI", PaBool, PaHid, false, false, true, HARAKIRI_DESC }, + { "-idIndex", &idIndex, "MONGO_ID_INDEX", PaBool, PaHid, false, false, true, ID_INDEX_DESC }, + { "-noswap", &noswap, "NOSWAP", PaBool, PaHid, false, false, true, NOSWAP_DESC }, + { "-lmtmp", &lmtmp, "TMP_TRACES", PaBool, PaHid, true, false, true, TMPTRACES_DESC }, { "-debugCurl", &debugCurl, "DEBUG_CURL", PaBool, PaHid, false, false, true, DEBUG_CURL_DESC }, + { "-lmtmp", &lmtmp, "TMP_TRACES", PaBool, PaHid, true, false, true, TMPTRACES_DESC }, PA_END_OF_ARGS }; diff --git a/test/functionalTest/cases/0000_cli/bool_option_with_value.test b/test/functionalTest/cases/0000_cli/bool_option_with_value.test index cf7f7f3454..a7e6d4e6cd 100644 --- a/test/functionalTest/cases/0000_cli/bool_option_with_value.test +++ b/test/functionalTest/cases/0000_cli/bool_option_with_value.test @@ -101,6 +101,7 @@ Usage: orionld [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations)] [option '-ctxTimeout' ] [option '-ctxAttempts' ] + [option '-pernot' (enable Pernot - Periodic Notifications)] [option '-troe' (enable TRoE - temporal representation of entities)] [option '-troeHost' ] [option '-troePort' ] diff --git a/test/functionalTest/cases/0000_cli/command_line_options.test b/test/functionalTest/cases/0000_cli/command_line_options.test index 51105dac1e..99bbd2d25f 100644 --- a/test/functionalTest/cases/0000_cli/command_line_options.test +++ b/test/functionalTest/cases/0000_cli/command_line_options.test @@ -90,6 +90,7 @@ Usage: orionld [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations)] [option '-ctxTimeout' ] [option '-ctxAttempts' ] + [option '-pernot' (enable Pernot - Periodic Notifications)] [option '-troe' (enable TRoE - temporal representation of entities)] [option '-troeHost' ] [option '-troePort' ] diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-new.test b/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-new.test index 9c108d3974..029be27d0f 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-new.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_langprop-and-notifications-new.test @@ -226,7 +226,7 @@ bye 04. GET subscription S2 - see lang=en ===================================== HTTP/1.1 200 OK -Content-Length: 418 +Content-Length: 423 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -679,7 +679,7 @@ Link: ; ], "id": "http://a.b.c/subs/sub09", "isActive": true, - "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "modifiedAt": REGEX(.*), "notification": { "attributes": [ diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_issue_1401.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_issue_1401.test index 2b6f61d4ec..903a895267 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_issue_1401.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_issue_1401.test @@ -129,7 +129,7 @@ Location: /ngsi-ld/v1/subscriptions/urn:ngsi-ld:Subscription:S1 02. GET S1 from CB1 =================== HTTP/1.1 200 OK -Content-Length: 423 +Content-Length: 506 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -369,6 +371,7 @@ Link: ; rel="h }, "id": "urn:ngsi-ld:Subscription:S1", "isActive": false, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "lang": "es", "modifiedAt": "202REGEX(.*)Z", "notification": { @@ -448,7 +451,7 @@ OK, slept 2.1 seconds 08. GET Sub 1 from CB2 - make sure it's identical in CB2 ======================================================== HTTP/1.1 200 OK -Content-Length: 598 +Content-Length: 681 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -472,6 +475,7 @@ Link: ; rel="h }, "id": "urn:ngsi-ld:Subscription:S1", "isActive": false, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "lang": "es", "modifiedAt": "202REGEX(.*)Z", "notification": { diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_geoq.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_geoq.test index 255b65b605..efe6031e75 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_geoq.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_patch_geoq.test @@ -173,7 +173,7 @@ Location: /ngsi-ld/v1/subscriptions/urn:S1 02. GET S1 from cache, see the "geoq" fields ============================================ HTTP/1.1 200 OK -Content-Length: 436 +Content-Length: 441 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" +Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, - "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "notification": { "attributes": [ "P1", @@ -418,10 +418,10 @@ Link: ; rel="http:/ 09. GET urn:S1 from database ============================ HTTP/1.1 200 OK -Content-Length: 572 +Content-Length: 577 Content-Type: application/json Date: REGEX(.*) -Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" +Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, - "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "notification": { "attributes": [ "P1", @@ -514,10 +514,10 @@ MongoDB server version: 4.4.22 "custom" : false, "servicePath" : "/#", "blacklist" : false, - "ldContext" : "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "ldContext" : "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", + "count" : 2, "lastNotification" : REGEX(.*), - "noMatch" : 2, - "count" : 2 + "noMatch" : 2 } bye @@ -528,12 +528,12 @@ Broker restarted 13. GET urn:S1 from cache (noMatch==3 due to initial attempt after restart) -========================= +=========================================================================== HTTP/1.1 200 OK -Content-Length: 705 +Content-Length: 724 Content-Type: application/json Date: REGEX(.*) -Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" +Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, - "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "notification": { "attributes": [ "P1", @@ -577,10 +577,10 @@ Link: ; rel="http:/ 14. GET urn:S1 from database ============================ HTTP/1.1 200 OK -Content-Length: 572 +Content-Length: 577 Content-Type: application/json Date: REGEX(.*) -Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" +Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, - "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "notification": { "attributes": [ "P1", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_counters_and_timestamps.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_counters_and_timestamps.test index cc6f7d2a66..1f5b6258bb 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_counters_and_timestamps.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_counters_and_timestamps.test @@ -193,10 +193,10 @@ slept 100 ms 03. GET urn:S1 and see noMatch == 1, and the other relevant counters/timestamps =============================================================================== HTTP/1.1 200 OK -Content-Length: 569 +Content-Length: 574 Content-Type: application/json Date: REGEX(.*) -Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" +Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, - "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "notification": { "attributes": [ "P1", @@ -259,7 +259,7 @@ User-Agent: orionld/REGEX(.*) Host: REGEX(.*) Accept: application/json Content-Type: application/json -Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" +Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, - "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "notification": { "attributes": [ "P1", @@ -381,7 +381,10 @@ MongoDB server version: REGEX(.*) "custom" : false, "servicePath" : "/#", "blacklist" : false, - "ldContext" : "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" + "ldContext" : "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", + "count" : 2, + "lastNotification" : REGEX(.*), + "noMatch" : 1 } bye diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_noMatch.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_noMatch.test index 39b95042a9..05ad871742 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_noMatch.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_noMatch.test @@ -106,10 +106,10 @@ Slept 0.1 seconds 03. GET urnS1 from cache - see noMatch=1, and no other counters/timestamps ========================================================================== HTTP/1.1 200 OK -Content-Length: 491 +Content-Length: 496 Content-Type: application/json Date: REGEX(.*) -Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" +Link: ; rel="http:/ ], "id": "urn:S1", "isActive": true, - "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld", + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test b/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test index 130678b8b9..8d262b4d2d 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test @@ -334,7 +334,7 @@ Ngsild-Attribute-Format: Normalized 06. GET the subscription, from cache, to see showChanges=true ============================================================= HTTP/1.1 200 OK -Content-Length: 475 +Content-Length: 480 Content-Type: application/json Date: REGEX(.*) Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" @@ -202,6 +202,7 @@ Link: ; rel="h ], "id": "urn:ngsi-ld:Subscription:SubscriptionPrivateSub", "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "notification": { "endpoint": { "accept": "application/json", diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue-1316.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue-1316.test index e71b7383e1..415db6800f 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue-1316.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue-1316.test @@ -103,7 +103,7 @@ NGSILD-Tenant: t_02 02. Retrieve S1 =============== HTTP/1.1 200 OK -Content-Length: 473 +Content-Length: 478 Content-Type: application/json Date: REGEX(.*) Link: Date: Mon, 28 Aug 2023 18:17:14 +0200 Subject: [PATCH 08/29] Finally fixed all the problems with timestamps/counters --- CHANGES_NEXT_RELEASE | 1 + src/lib/cache/subCache.cpp | 18 ++- src/lib/logMsg/traceLevels.h | 2 + src/lib/mongoBackend/dbConstants.h | 1 + src/lib/mongoBackend/mongoSubCache.cpp | 31 ++++ src/lib/mongoBackend/mongoSubCache.h | 1 + src/lib/ngsiNotify/QueueWorkers.cpp | 10 -- .../common/subCacheApiSubscriptionInsert.cpp | 57 ++++--- .../dbModel/dbModelToApiSubscription.cpp | 16 +- .../kjTree/kjTreeFromCachedSubscription.cpp | 9 +- .../notifications/notificationFailure.cpp | 19 +-- .../payloadCheck/pCheckNotification.cpp | 23 ++- src/lib/orionld/pernot/pernotSubCacheAdd.cpp | 17 +-- .../orionldGetSubscription.cpp | 142 ++++++------------ ...ption_counters-for-http-notifications.test | 8 +- ...tion_counters-for-https-notifications.test | 8 +- ...ption_counters-for-mqtt-notifications.test | 2 +- ...d_patch_entity2-notification-counters.test | 3 +- ...ngsild_pernot_counters_and_timestamps.test | 3 +- .../0000_ngsild/ngsild_subCache-counters.test | 3 +- 20 files changed, 197 insertions(+), 177 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 2a71a1dae4..abcaf899ae 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ Fixed issues: + * #280 Implemented Periodic Notifications (subscriptions with a 'timeInterval') diff --git a/src/lib/cache/subCache.cpp b/src/lib/cache/subCache.cpp index 8c0f01ae08..f9450455f5 100644 --- a/src/lib/cache/subCache.cpp +++ b/src/lib/cache/subCache.cpp @@ -891,7 +891,10 @@ bool subCacheItemInsert cSubP->lastSuccess = lastNotificationSuccessTime; cSubP->renderFormat = renderFormat; cSubP->next = NULL; + cSubP->dbCount = 0; cSubP->count = 0; + cSubP->failures = 0; + cSubP->dbFailures = 0; cSubP->status = status; cSubP->url = NULL; cSubP->ip = NULL; @@ -1277,6 +1280,7 @@ typedef struct CachedSubSaved { double lastNotificationTime; int64_t count; + int64_t failures; double lastFailure; double lastSuccess; bool ngsild; @@ -1340,6 +1344,7 @@ void subCacheSync(void) cssP->lastNotificationTime = cSubP->lastNotificationTime; cssP->count = cSubP->count; // This count is later pushed ($inc) to DB - needs to go to cache as well + cssP->failures = cSubP->failures; cssP->lastFailure = cSubP->lastFailure; cssP->lastSuccess = cSubP->lastSuccess; cssP->ngsild = (cSubP->ldContext != "")? true : false; @@ -1407,7 +1412,7 @@ void subCacheSync(void) cSubP->subscriptionId, cssP->ngsild, cssP->count, - 0, // FIXME: failures - fix! + cssP->failures, 0, // noMatch - not here - only for PerNot cssP->lastNotificationTime, cssP->lastSuccess, @@ -1418,6 +1423,7 @@ void subCacheSync(void) mongoSubCountersUpdate(tenant, cSubP->subscriptionId, cssP->count, + cssP->failures, cssP->lastNotificationTime, cssP->lastFailure, cssP->lastSuccess, @@ -1430,8 +1436,10 @@ void subCacheSync(void) if (cssP->lastNotificationTime != 0) cSubP->lastNotificationTime = cssP->lastNotificationTime; // Here the delta (just $inc'ed to DB) is also inc'ed to subCache - cSubP->dbCount += cssP->count; - cSubP->count = 0; + cSubP->dbCount += cssP->count; + cSubP->count = 0; + cSubP->dbFailures += cssP->failures; + cSubP->failures = 0; } cSubP = cSubP->next; @@ -1528,9 +1536,9 @@ void subCacheItemNotificationErrorStatus(const std::string& tenant, const std::s // The field 'count' has already been taken care of. Set to 0 in the calls to mongoSubCountersUpdate() if (errors == 0) - mongoSubCountersUpdate(tenant, subscriptionId, 0, kNow, -1, kNow, ngsild); // lastFailure == -1 + mongoSubCountersUpdate(tenant, subscriptionId, 0, 0, kNow, -1, kNow, ngsild); // lastFailure == -1 else - mongoSubCountersUpdate(tenant, subscriptionId, 0, kNow, kNow, -1, ngsild); // lastSuccess == -1, count == 0 + mongoSubCountersUpdate(tenant, subscriptionId, 0, 1, kNow, kNow, -1, ngsild); // lastSuccess == -1, count == 0 return; } diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index 1cdd469b9e..a745087a65 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -64,6 +64,7 @@ typedef enum TraceLevels LmtSubCacheDebug, // Subscription Cache Debug LmtSubCacheStats, // Subscription Cache Counters and Timestamps LmtSubCacheSync, // Subscription Cache Refresh + LmtSubCacheFlush, // Subscription Cache Flush // // Registration Cache @@ -100,6 +101,7 @@ typedef enum TraceLevels LmtPernot = 100, // Periodic Notification Subscription cache LmtPernotLoop, // Pernot loop, when each sub is triggered in time LmtPernotFlush = 102, // Pernot flush to DB + // // Misc // diff --git a/src/lib/mongoBackend/dbConstants.h b/src/lib/mongoBackend/dbConstants.h index 1ad107bd7a..fe1ebf6f06 100644 --- a/src/lib/mongoBackend/dbConstants.h +++ b/src/lib/mongoBackend/dbConstants.h @@ -114,6 +114,7 @@ #define CSUB_ENTITY_ISPATTERN "isPattern" #define CSUB_ENTITY_ISTYPEPATTERN "isTypePattern" #define CSUB_COUNT "count" +#define CSUB_FAILURES "failures" #define CSUB_FORMAT "format" #define CSUB_STATUS "status" #define CSUB_SERVICE_PATH "servicePath" diff --git a/src/lib/mongoBackend/mongoSubCache.cpp b/src/lib/mongoBackend/mongoSubCache.cpp index df7d0fd045..182c385c17 100644 --- a/src/lib/mongoBackend/mongoSubCache.cpp +++ b/src/lib/mongoBackend/mongoSubCache.cpp @@ -637,6 +637,35 @@ static void mongoSubCountersUpdateCount +/* **************************************************************************** +* +* mongoSubCountersUpdateFailures - +*/ +static void mongoSubCountersUpdateFailures +( + const std::string& collection, + const std::string& subId, + long long failures, + bool ngsild +) +{ + BSONObj condition; + BSONObj update; + std::string err; + + if (ngsild == true) + condition = BSON("_id" << subId); + else + condition = BSON("_id" << OID(subId)); + + update = BSON("$inc" << BSON(CSUB_FAILURES << failures)); + + if (collectionUpdate(collection.c_str(), condition, update, false, &err) != true) + LM_E(("Internal Error (error updating 'failures' for a subscription)")); +} + + + /* **************************************************************************** * * mongoSubCountersUpdateLastNotificationTime - @@ -764,6 +793,7 @@ void mongoSubCountersUpdate const std::string& tenant, const std::string& subId, long long count, + long long failures, double lastNotificationTime, double lastFailure, double lastSuccess, @@ -789,6 +819,7 @@ void mongoSubCountersUpdate LM_T(LmtSubCacheStats, ("lastSuccess: %f", lastSuccess)); if (count > 0) mongoSubCountersUpdateCount(collectionPath, subId, count, ngsild); + if (failures > 0) mongoSubCountersUpdateFailures(collectionPath, subId, failures, ngsild); if (lastNotificationTime > 0) mongoSubCountersUpdateLastNotificationTime(collectionPath, subId, lastNotificationTime, ngsild); if (lastFailure > 0) mongoSubCountersUpdateLastFailure(collectionPath, subId, lastFailure, ngsild); if (lastSuccess > 0) mongoSubCountersUpdateLastSuccess(collectionPath, subId, lastSuccess, ngsild); diff --git a/src/lib/mongoBackend/mongoSubCache.h b/src/lib/mongoBackend/mongoSubCache.h index 936d62b51e..77db960081 100644 --- a/src/lib/mongoBackend/mongoSubCache.h +++ b/src/lib/mongoBackend/mongoSubCache.h @@ -88,6 +88,7 @@ extern void mongoSubCountersUpdate const std::string& tenant, const std::string& subId, long long count, + long long failures, double lastNotificationTime, double lastFailure, double lastSuccess, diff --git a/src/lib/ngsiNotify/QueueWorkers.cpp b/src/lib/ngsiNotify/QueueWorkers.cpp index 1a76c0a363..66463e0421 100644 --- a/src/lib/ngsiNotify/QueueWorkers.cpp +++ b/src/lib/ngsiNotify/QueueWorkers.cpp @@ -203,12 +203,7 @@ static void* workerFunc(void* pSyncQ) alarmMgr.notificationErrorReset(url); if (params->registration == false) - { - LM_T(LmtNotificationStats, ("Calling subCacheItemNotificationErrorStatus with OK")); subCacheItemNotificationErrorStatus(params->tenant, params->subscriptionId, 0, ngsildSubscription); - } - else - LM_T(LmtNotificationStats, ("NOT calling subCacheItemNotificationErrorStatus with ERROR")); } else { @@ -216,12 +211,7 @@ static void* workerFunc(void* pSyncQ) alarmMgr.notificationError(url, "notification failure for queue worker"); if (params->registration == false) - { - LM_T(LmtNotificationStats, ("Calling subCacheItemNotificationErrorStatus with ERROR")); subCacheItemNotificationErrorStatus(params->tenant, params->subscriptionId, 1, ngsildSubscription); - } - else - LM_T(LmtNotificationStats, ("NOT calling subCacheItemNotificationErrorStatus with ERROR")); } } diff --git a/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp b/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp index 31d823fc63..250484691b 100644 --- a/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp +++ b/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp @@ -70,7 +70,7 @@ static void subCacheItemFill RenderFormat renderFormat ) { - kjTreeLog(apiSubscriptionP, "apiSubscriptionP", LmtSR); + kjTreeLog(apiSubscriptionP, "apiSubscriptionP", LmtSubCacheSync); cSubP->tenant = (tenant == NULL || *tenant == 0)? NULL : strdup(tenant); cSubP->servicePath = strdup("/#"); @@ -100,16 +100,11 @@ static void subCacheItemFill KjNode* langP = kjLookup(apiSubscriptionP, "lang"); KjNode* createdAtP = kjLookup(apiSubscriptionP, "createdAt"); KjNode* modifiedAtP = kjLookup(apiSubscriptionP, "modifiedAt"); - KjNode* dbCountP = kjLookup(apiSubscriptionP, "count"); - KjNode* lastNotificationP = kjLookup(apiSubscriptionP, "lastNotification"); - KjNode* lastSuccessP = kjLookup(apiSubscriptionP, "lastSuccess"); - KjNode* lastFailureP = kjLookup(apiSubscriptionP, "lastFailure"); - - // Fixing false alarms for q and mq - if ((qP != NULL) && (qP->value.s != NULL) && (qP->value.s[0] == 0)) - qP = NULL; - if ((mqP != NULL) && (mqP->value.s != NULL) && (mqP->value.s[0] == 0)) - mqP = NULL; + KjNode* dbCountP = kjLookup(notificationP, "timesSent"); + KjNode* dbFailuresP = kjLookup(notificationP, "timesFailed"); + KjNode* lastNotificationP = kjLookup(notificationP, "lastNotification"); + KjNode* lastSuccessP = kjLookup(notificationP, "lastSuccess"); + KjNode* lastFailureP = kjLookup(notificationP, "lastFailure"); if (subscriptionIdP == NULL) subscriptionIdP = kjLookup(apiSubscriptionP, "id"); @@ -123,22 +118,34 @@ static void subCacheItemFill if (subscriptionNameP != NULL) cSubP->name = subscriptionNameP->value.s; - if (dbCountP != NULL) - cSubP->dbCount = dbCountP->value.i; - else - cSubP->dbCount = 0; + if (descriptionP != NULL) + cSubP->description = strdup(descriptionP->value.s); - if (lastNotificationP != NULL) - cSubP->lastNotificationTime = lastNotificationP->value.f; - if (lastSuccessP != NULL) - cSubP->lastSuccess = lastSuccessP->value.f; + // + // DB Counters + // + cSubP->dbCount = (dbCountP != NULL)? dbCountP->value.i : 0; + cSubP->dbFailures = (dbFailuresP != NULL)? dbFailuresP->value.i : 0; - if (lastFailureP != NULL) - cSubP->lastFailure = lastFailureP->value.f; + LM_T(LmtSubCacheSync, ("%s: dbFailures == %d", subscriptionIdP->value.s, cSubP->dbFailures)); + LM_T(LmtSubCacheSync, ("%s: count == %d", subscriptionIdP->value.s, cSubP->count)); + LM_T(LmtSubCacheSync, ("%s: dbCount == %d", subscriptionIdP->value.s, cSubP->dbCount)); - if (descriptionP != NULL) - cSubP->description = strdup(descriptionP->value.s); + + // + // timestamps + // + if (lastNotificationP != NULL) cSubP->lastNotificationTime = lastNotificationP->value.f; + if (lastSuccessP != NULL) cSubP->lastSuccess = lastSuccessP->value.f; + if (lastFailureP != NULL) cSubP->lastFailure = lastFailureP->value.f; + + + // Fixing false alarms for q and mq + if ((qP != NULL) && (qP->value.s != NULL) && (qP->value.s[0] == 0)) + qP = NULL; + if ((mqP != NULL) && (mqP->value.s != NULL) && (mqP->value.s[0] == 0)) + mqP = NULL; if (qP != NULL) cSubP->expression.q = qP->value.s; @@ -149,6 +156,7 @@ static void subCacheItemFill if (ldqP != NULL) cSubP->qText = strdup(ldqP->value.s); + if (isActiveP != NULL) { cSubP->isActive = isActiveP->value.b; @@ -474,7 +482,8 @@ static CachedSubscription* subCacheApiSubscriptionUpdate cSubP->modifiedAt = modifiedAtNode->value.f; // The new value is in the DB } - cSubP->count = 0; + cSubP->count = 0; + cSubP->failures = 0; subCacheItemStrip(cSubP); subCacheItemFill(cSubP, apiSubscriptionP, qTree, geoCoordinatesP, contextP, tenant, showChangesP, sysAttrsP, renderFormat); diff --git a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp index 9d79c0b211..50557e73d2 100644 --- a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp +++ b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp @@ -510,8 +510,22 @@ KjNode* dbModelToApiSubscription // timesSent if (dbCountP != NULL) { - dbCountP->name = (char*) "timesSent"; + int timesSent = 0; + + if (dbCountP->type == KjInt) + timesSent = dbCountP->value.i; + else if (dbCountP->type == KjObject) + LM_W(("Subscription::count: find the integer inside the object!")); + + // + // Transform to KjInt named "timesSent" + // + dbCountP->name = (char*) "timesSent"; + dbCountP->type = KjInt; + dbCountP->value.i = timesSent; + kjChildAdd(notificationP, dbCountP); + LM_T(LmtSubCacheStats, ("count/timesSent: %d", timesSent)); } // timesFailed diff --git a/src/lib/orionld/kjTree/kjTreeFromCachedSubscription.cpp b/src/lib/orionld/kjTree/kjTreeFromCachedSubscription.cpp index 4aa74482bb..0b343e52c5 100644 --- a/src/lib/orionld/kjTree/kjTreeFromCachedSubscription.cpp +++ b/src/lib/orionld/kjTree/kjTreeFromCachedSubscription.cpp @@ -384,6 +384,9 @@ KjNode* kjTreeFromCachedSubscription(CachedSubscription* cSubP, bool sysAttrs, b // if (cSubP->count + cSubP->dbCount > 0) { + LM_T(LmtSubCacheStats, ("count: %d", cSubP->count)); + LM_T(LmtSubCacheStats, ("dbCount: %d", cSubP->dbCount)); + nodeP = kjInteger(orionldState.kjsonP, "timesSent", cSubP->count + cSubP->dbCount); NULL_CHECK(nodeP); kjChildAdd(notificationNodeP, nodeP); @@ -392,7 +395,7 @@ KjNode* kjTreeFromCachedSubscription(CachedSubscription* cSubP, bool sysAttrs, b // // notification::lastNotification // - if (cSubP->lastNotificationTime > 0) + if (cSubP->lastNotificationTime > 0.1) { numberToDate(cSubP->lastNotificationTime, dateTime, sizeof(dateTime)); nodeP = kjString(orionldState.kjsonP, "lastNotification", dateTime); @@ -403,7 +406,7 @@ KjNode* kjTreeFromCachedSubscription(CachedSubscription* cSubP, bool sysAttrs, b // // notification::lastFailure // - if (cSubP->lastFailure > 0) + if (cSubP->lastFailure > 0.1) { numberToDate(cSubP->lastFailure, dateTime, sizeof(dateTime)); nodeP = kjString(orionldState.kjsonP, "lastFailure", dateTime); @@ -414,7 +417,7 @@ KjNode* kjTreeFromCachedSubscription(CachedSubscription* cSubP, bool sysAttrs, b // // notification::lastSuccess // - if (cSubP->lastSuccess > 0) + if (cSubP->lastSuccess > 0.1) { numberToDate(cSubP->lastSuccess, dateTime, sizeof(dateTime)); nodeP = kjString(orionldState.kjsonP, "lastSuccess", dateTime); diff --git a/src/lib/orionld/notifications/notificationFailure.cpp b/src/lib/orionld/notifications/notificationFailure.cpp index be93c6d38d..ab350d6e7b 100644 --- a/src/lib/orionld/notifications/notificationFailure.cpp +++ b/src/lib/orionld/notifications/notificationFailure.cpp @@ -75,21 +75,22 @@ void notificationFailure(CachedSubscription* subP, const char* errorReason, doub // if (((cSubCounters != 0) && (subP->dirty >= cSubCounters)) || (forcedToPause == true)) { - LM_T(LmtNotificationStats, ("%s: Calling mongocSubCountersUpdate", subP->subscriptionId)); - - mongocSubCountersUpdate(subP->tenantP, subP->subscriptionId, (subP->ldContext != ""), subP->count, subP->failures, 0, subP->lastNotificationTime, subP->lastSuccess, subP->lastFailure, forcedToPause); + mongocSubCountersUpdate(subP->tenantP, + subP->subscriptionId, + (subP->ldContext != ""), + subP->count, + subP->failures, + 0, + subP->lastNotificationTime, + subP->lastSuccess, + subP->lastFailure, + forcedToPause); subP->dirty = 0; subP->dbCount += subP->count; subP->count = 0; subP->dbFailures += subP->failures; subP->failures = 0; } - else - LM_T(LmtNotificationStats, ("%s: Not calling mongocSubCountersUpdate (cSubCounters: %d, dirty: %d, forcedToPause: %s)", - subP->subscriptionId, - cSubCounters, - subP->dirty, - (forcedToPause == true)? "true" : "false")); } diff --git a/src/lib/orionld/payloadCheck/pCheckNotification.cpp b/src/lib/orionld/payloadCheck/pCheckNotification.cpp index f5bf56aa52..a258fdcdd8 100644 --- a/src/lib/orionld/payloadCheck/pCheckNotification.cpp +++ b/src/lib/orionld/payloadCheck/pCheckNotification.cpp @@ -25,6 +25,7 @@ extern "C" { #include "kjson/KjNode.h" // KjNode +#include "kjson/kjBuilder.h" // kjChildRemove } #include "logMsg/logMsg.h" // LM_* @@ -65,8 +66,12 @@ bool pCheckNotification PCHECK_OBJECT(notificationP, 0, NULL, SubscriptionNotificationPath, 400); PCHECK_OBJECT_EMPTY(notificationP, 0, NULL, SubscriptionNotificationPath, 400); - for (KjNode* nItemP = notificationP->value.firstChildP; nItemP != NULL; nItemP = nItemP->next) + KjNode* nItemP = notificationP->value.firstChildP; + KjNode* next; + while (nItemP != NULL) { + next = nItemP->next; + if (strcmp(nItemP->name, "attributes") == 0) { PCHECK_DUPLICATE(attributesP, nItemP, 0, NULL, SubscriptionNotificationAttributesPath, 400); @@ -115,16 +120,22 @@ bool pCheckNotification *sysAttrsOutP = sysAttrsP; LM_T(LmtSysAttrs, ("Found a 'sysAttrs' in Subscription::notification (%s)", (sysAttrsP->value.b == true)? "true" : "false")); } - else if (strcmp(nItemP->name, "status") == 0) {} // Ignored - else if (strcmp(nItemP->name, "timesSent") == 0) {} // Ignored - else if (strcmp(nItemP->name, "lastNotification") == 0) {} // Ignored - else if (strcmp(nItemP->name, "lastSuccess") == 0) {} // Ignored - else if (strcmp(nItemP->name, "lastFailure") == 0) {} // Ignored + else if ((strcmp(nItemP->name, "status") == 0) || + (strcmp(nItemP->name, "timesSent") == 0) || + (strcmp(nItemP->name, "timesFailed") == 0) || + (strcmp(nItemP->name, "lastNotification") == 0) || + (strcmp(nItemP->name, "lastSuccess") == 0) || + (strcmp(nItemP->name, "lastFailure") == 0)) + { + kjChildRemove(notificationP, nItemP); + } else { orionldError(OrionldBadRequestData, "Invalid field for Subscription::notification", nItemP->name, 400); return false; } + + nItemP = next; } if ((endpointP == NULL) && (patch == false)) diff --git a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp index 5100340b58..440a8d20a5 100644 --- a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp +++ b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp @@ -99,7 +99,10 @@ static void counterFromDb(KjNode* subP, uint32_t* counterP, const char* fieldNam *counterP = counterNodeP->value.i; } else - *counterP = 0; + { + *counterP = 0; + LM_T(LmtPernot, ("Counter '%s' NOT found in db", fieldName)); + } } @@ -145,8 +148,6 @@ PernotSubscription* pernotSubCacheAdd PernotSubscription* pSubP = (PernotSubscription*) malloc(sizeof(PernotSubscription)); bzero(pSubP, sizeof(PernotSubscription)); - kjTreeLog(apiSubP, "API Subscription from DB", LmtPernot); - if (subscriptionId == NULL) { KjNode* idP = kjLookup(apiSubP, "id"); @@ -180,8 +181,6 @@ PernotSubscription* pernotSubCacheAdd urlParse(pSubP->url, &pSubP->protocolString, &pSubP->ip, &pSubP->port, &pSubP->rest); pSubP->protocol = protocolFromString(pSubP->protocolString); - kjTreeLog(pSubP->kjSubP, "Initial Pernot Subscription", LmtPernot); - // Mime Type for notifications KjNode* acceptP = kjLookup(endpointP, "accept"); pSubP->mimeType = JSON; // Default setting @@ -203,9 +202,9 @@ PernotSubscription* pernotSubCacheAdd pSubP->state = SubActive; // Counters - counterFromDb(apiSubP, &pSubP->notificationAttemptsDb, "count"); - counterFromDb(apiSubP, &pSubP->notificationErrorsDb, "timesFailed"); - counterFromDb(apiSubP, &pSubP->noMatchDb, "noMatch"); + counterFromDb(notificationP, &pSubP->notificationAttemptsDb, "timesSent"); + counterFromDb(notificationP, &pSubP->notificationErrorsDb, "timesFailed"); + counterFromDb(notificationP, &pSubP->noMatchDb, "noMatch"); pSubP->notificationAttempts = 0; // cached - added to notificationAttemptsDb pSubP->notificationErrors = 0; // cached - added to notificationErrorsDb @@ -291,8 +290,6 @@ PernotSubscription* pernotSubCacheAdd kjChildAdd(endpointP, acceptP); } - kjTreeLog(pSubP->kjSubP, "Pernot Subscription in Cache", LmtPernot); - // // Caching stuff from the KjNode tree // diff --git a/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp b/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp index 9f8aeae15e..04de817494 100644 --- a/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp +++ b/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp @@ -55,8 +55,13 @@ extern "C" // static void subTimestampSet(KjNode* apiSubP, const char* fieldName, double valueInSubCache) { - if (valueInSubCache <= 0) + if (valueInSubCache <= 0.1) + { + KjNode* timestampNodeP = kjLookup(apiSubP, fieldName); + if (timestampNodeP != NULL) + kjChildRemove(apiSubP, timestampNodeP); return; + } KjNode* timestampNodeP = kjLookup(apiSubP, fieldName); @@ -103,22 +108,15 @@ static void subCounterSet(KjNode* apiSubP, const char* fieldName, int64_t valueI // // orionldSubCounters - FIXME: Own Module // +// Three options: +// 1. cSubP given +// 2. pSubP given +// 3, None of them (look up apiSubP::subscriptionId in both cashes) +// void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP, PernotSubscription* pSubP) { - // - // Three options: - // 1. cSubP given - // 2. pSubP given - // 3, None of them (look up apiSubP::subscriptionId in both cashes) - // - double lastNotificationTime = 0; - double lastSuccess = 0; - double lastFailure = 0; - int timesSent = 0; - int timesFailed = 0; - int noMatch = 0; - KjNode* notificationP = kjLookup(apiSubP, "notification"); - + KjNode* notificationP = kjLookup(apiSubP, "notification"); + if (notificationP == NULL) LM_RVE(("API Subscription without a notification field !!!")); @@ -130,95 +128,42 @@ void orionldSubCounters(KjNode* apiSubP, CachedSubscription* cSubP, PernotSubscr return; cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subIdP->value.s); - pSubP = pernotSubCacheLookup(orionldState.tenantP->tenant, subIdP->value.s); - if ((cSubP == NULL) && (pSubP == NULL)) - LM_RVE(("Can't find subscription '%s'", subIdP->value.s)); - } - // - // Get timestamps/counters from db - // - KjNode* tsP; + if (cSubP == NULL) + { + pSubP = pernotSubCacheLookup(orionldState.tenantP->tenant, subIdP->value.s); + if (pSubP == NULL) + LM_RVE(("Can't find subscription '%s' in any subscription cache", subIdP->value.s)); + } + } - // lastNotificationTime - tsP = kjLookup(notificationP, "lastNotification"); - if (tsP != NULL) - lastNotificationTime = tsP->value.f; + double lastNotificationTime = (cSubP != NULL)? cSubP->lastNotificationTime : pSubP->lastNotificationTime; + double lastSuccess = (cSubP != NULL)? cSubP->lastSuccess : pSubP->lastSuccessTime; + double lastFailure = (cSubP != NULL)? cSubP->lastFailure : pSubP->lastFailureTime; + int timesSent = 0; + int timesFailed = 0; + int noMatch = 0; - // lastSuccess - tsP = kjLookup(notificationP, "lastSuccess"); - if (tsP != NULL) - lastSuccess = tsP->value.f; - - // lastFailure - tsP = kjLookup(notificationP, "lastFailure"); - if (tsP != NULL) - lastFailure = tsP->value.f; - - // timesSent - tsP = kjLookup(notificationP, "count"); - if (tsP != NULL) - timesSent = tsP->value.i; - - // timesFailed - tsP = kjLookup(notificationP, "timesFailed"); - if (tsP != NULL) - timesFailed = tsP->value.i; - - // noMatch - tsP = kjLookup(notificationP, "noMatch"); - if (tsP != NULL) - noMatch = tsP->value.i; - - LM_T(LmtSR, ("DB lastNotificationTime: %f", lastNotificationTime)); - LM_T(LmtSR, ("DB lastSuccess: %f", lastSuccess)); - LM_T(LmtSR, ("DB lastFailure: %f", lastFailure)); - LM_T(LmtSR, ("DB timesSent: %d", timesSent)); - LM_T(LmtSR, ("DB timesFailed: %d", timesFailed)); - LM_T(LmtSR, ("DB noMatch: %d", noMatch)); - - // Overwrite timestamps/counters from cache if (cSubP != NULL) { - if (cSubP->lastNotificationTime > 0) lastNotificationTime = cSubP->lastNotificationTime; - if (cSubP->lastSuccess > 0) lastSuccess = cSubP->lastSuccess; - if (cSubP->lastFailure > 0) lastFailure = cSubP->lastFailure; - - timesSent += cSubP->count; - timesFailed += cSubP->failures; - noMatch = 0; + timesSent = cSubP->dbCount + cSubP->count; + timesFailed = cSubP->dbFailures + cSubP->failures; } - else if (pSubP != NULL) + else { - if (pSubP->lastNotificationTime > 0) lastNotificationTime = pSubP->lastNotificationTime; - if (pSubP->lastSuccessTime > 0) lastSuccess = pSubP->lastSuccessTime; - if (pSubP->lastFailureTime > 0) lastFailure = pSubP->lastFailureTime; - - timesSent += pSubP->notificationAttempts; - timesFailed += pSubP->notificationErrors; - noMatch += pSubP->noMatch; + timesSent = pSubP->notificationAttempts + pSubP->notificationAttemptsDb; + timesFailed = pSubP->notificationErrors + pSubP->notificationErrorsDb; + noMatch = pSubP->noMatch + pSubP->noMatchDb; } - - LM_T(LmtSR, ("lastNotificationTime: %f", lastNotificationTime)); - LM_T(LmtSR, ("lastSuccess: %f", lastSuccess)); - LM_T(LmtSR, ("lastFailure: %f", lastFailure)); - LM_T(LmtSR, ("timesSent: %d", timesSent)); - LM_T(LmtSR, ("timesFailed: %d", timesFailed)); - LM_T(LmtSR, ("noMatch: %d", noMatch)); - // // Set the values // - KjNode* notificationNodeP = kjLookup(apiSubP, "notification"); - if (notificationNodeP != NULL) - { - subTimestampSet(notificationNodeP, "lastNotification", lastNotificationTime); - subTimestampSet(notificationNodeP, "lastSuccess", lastSuccess); - subTimestampSet(notificationNodeP, "lastFailure", lastFailure); - subCounterSet(notificationNodeP, "timesSent", timesSent); - subCounterSet(notificationNodeP, "timesFailed", timesFailed); - subCounterSet(notificationNodeP, "noMatch", noMatch); - } + subTimestampSet(notificationP, "lastNotification", lastNotificationTime); + subTimestampSet(notificationP, "lastSuccess", lastSuccess); + subTimestampSet(notificationP, "lastFailure", lastFailure); + subCounterSet(notificationP, "timesSent", timesSent); + subCounterSet(notificationP, "timesFailed", timesFailed); + subCounterSet(notificationP, "noMatch", noMatch); } @@ -238,7 +183,8 @@ static bool orionldGetSubscriptionFromDb(void) orionldError(OrionldResourceNotFound, "Subscription Not Found", orionldState.wildcard[0], 404); return false; } - + kjTreeLog(dbSubP, "DB Sub", LmtSubCacheStats); + KjNode* coordinatesNodeP = NULL; // Not needed here, but dbModelToApiSubscription requires it KjNode* contextNodeP = NULL; // Not needed here, but dbModelToApiSubscription requires it KjNode* showChangesP = NULL; // Not needed here, but dbModelToApiSubscription requires it @@ -304,15 +250,17 @@ bool orionldGetSubscription(void) return orionldGetSubscriptionFromDb(); } - char* subscriptionId = orionldState.wildcard[0]; + char* subscriptionId = orionldState.wildcard[0]; // "Normal" (onchange) subscription? CachedSubscription* cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subscriptionId); if (cSubP != NULL) { orionldState.httpStatusCode = 200; + orionldState.responseTree = kjTreeFromCachedSubscription(cSubP, orionldState.uriParamOptions.sysAttrs, orionldState.out.contentType == JSONLD); orionldSubCounters(orionldState.responseTree, cSubP, NULL); + return true; } @@ -320,12 +268,10 @@ bool orionldGetSubscription(void) PernotSubscription* pSubP = pernotSubCacheLookup(subscriptionId, orionldState.tenantP->tenant); if (pSubP != NULL) { - kjTreeLog(pSubP->kjSubP, "pernot sub in cache", LmtSR); orionldState.httpStatusCode = 200; + orionldState.responseTree = kjTreeFromPernotSubscription(pSubP, orionldState.uriParamOptions.sysAttrs, orionldState.out.contentType == JSONLD); - kjTreeLog(orionldState.responseTree, "pernot sub after kjTreeFromPernotSubscription", LmtSR); orionldSubCounters(orionldState.responseTree, NULL, pSubP); - kjTreeLog(orionldState.responseTree, "pernot sub after orionldSubCounters", LmtSR); return true; } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test index c4025c5d88..49b4d19c0f 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription_counters-for-http-notifications.test @@ -400,7 +400,7 @@ Location: /ngsi-ld/v1/entities/urn:ngsi-ld:T:E5 11. GET S1, see lastSuccess, lastFailure, lastNotification, and timesSent==5 ============================================================================ HTTP/1.1 200 OK -Content-Length: 581 +Content-Length: 597 Content-Type: application/json Date: REGEX(.*) Link: Date: Tue, 29 Aug 2023 16:49:03 +0200 Subject: [PATCH 09/29] Support for entities and notification::attributes for pernot subs --- src/lib/logMsg/traceLevels.h | 3 +- src/lib/orionld/kjTree/kjTreeLog.cpp | 2 +- src/lib/orionld/pernot/PernotSubscription.h | 6 + src/lib/orionld/pernot/pernotSubCacheAdd.cpp | 7 +- src/lib/orionld/pernot/pernotTreat.cpp | 12 +- .../cases/0000_ngsild/ngsild_pernot.test | 6 +- ...ngsild_pernot_counters_and_timestamps.test | 6 +- .../0000_ngsild/ngsild_pernot_queries.test | 218 ++++++++++++++++++ 8 files changed, 242 insertions(+), 18 deletions(-) create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index a745087a65..54a527a960 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -100,7 +100,8 @@ typedef enum TraceLevels // LmtPernot = 100, // Periodic Notification Subscription cache LmtPernotLoop, // Pernot loop, when each sub is triggered in time - LmtPernotFlush = 102, // Pernot flush to DB + LmtPernotFlush, // Pernot flush to DB + LmtPernotQuery, // Pernot query // // Misc diff --git a/src/lib/orionld/kjTree/kjTreeLog.cpp b/src/lib/orionld/kjTree/kjTreeLog.cpp index 5ea655c755..d99209833e 100644 --- a/src/lib/orionld/kjTree/kjTreeLog.cpp +++ b/src/lib/orionld/kjTree/kjTreeLog.cpp @@ -56,7 +56,7 @@ void kjTreeLogFunction(KjNode* tree, const char* msg, const char* path, int line int bufSize = kjFastRenderSize(tree); - // Too big trees will not be rendered - this is just tracing + // Too big trees will not be rendered - this is just logging if (bufSize < 10 * 1024) { char* treeBuf = kaAlloc(&orionldState.kalloc, bufSize + 512); diff --git a/src/lib/orionld/pernot/PernotSubscription.h b/src/lib/orionld/pernot/PernotSubscription.h index c36ac5d5ef..d3490c4e75 100644 --- a/src/lib/orionld/pernot/PernotSubscription.h +++ b/src/lib/orionld/pernot/PernotSubscription.h @@ -39,6 +39,7 @@ extern "C" #include "orionld/types/Protocol.h" // Protocol #include "orionld/types/StringArray.h" // StringArray #include "orionld/types/OrionldTenant.h" // OrionldTenant +#include "orionld/q/QNode.h" // QNode @@ -79,6 +80,11 @@ typedef struct PernotSubscription double lastFailureTime; double expiresAt; + // For the Query + KjNode* eSelector; + KjNode* attrsSelector; + QNode* qTreeSelector; + // Counters uint32_t notificationAttemptsDb; // Total number of notification attempts, in DB uint32_t notificationAttempts; // Total number of notification attempts, in cache (to be added to dbCount) diff --git a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp index 440a8d20a5..3fd426c10a 100644 --- a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp +++ b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp @@ -168,9 +168,14 @@ PernotSubscription* pernotSubCacheAdd pSubP->context = (contextP == NULL)? NULL : contextP->url; pSubP->isActive = true; // Active by default, then we'll see ... - // notification KjNode* notificationP = kjLookup(pSubP->kjSubP, "notification"); + // Query parameters + pSubP->eSelector = kjLookup(pSubP->kjSubP, "entities"); + pSubP->attrsSelector = kjLookup(notificationP, "attributes"); + pSubP->qTreeSelector = qTree; // I probably need to clone this ... + + // notification::endpoint if (endpointP == NULL) endpointP = kjLookup(notificationP, "endpoint"); diff --git a/src/lib/orionld/pernot/pernotTreat.cpp b/src/lib/orionld/pernot/pernotTreat.cpp index ed26756333..ec5a8d9278 100644 --- a/src/lib/orionld/pernot/pernotTreat.cpp +++ b/src/lib/orionld/pernot/pernotTreat.cpp @@ -114,10 +114,6 @@ static void* pernotTreat(void* vP) orionldState.kjsonP = kjBufferCreate(&kjson, &kalloc); LM_T(LmtPernot, ("Creating the query for pernot-subscription %s", subP->subscriptionId)); - KjNode* eSelector = NULL; // should be created when sub added to cache - KjNode* attrsArray = NULL; // should be created when sub added to cache - QNode* qTree = NULL; // should be created when sub added to cache - OrionldGeoInfo* geoInfoP = NULL; // should be created when sub added to cache int64_t count = 0; KjNode* dbEntityArray; KjNode* apiEntityArray; @@ -127,8 +123,10 @@ static void* pernotTreat(void* vP) orionldState.uriParams.count = true; orionldState.tenantP = subP->tenantP; - LM_T(LmtPernot, ("Calling mongocEntitiesQuery2")); - dbEntityArray = mongocEntitiesQuery2(eSelector, attrsArray, qTree, geoInfoP, subP->lang, &count); + kjTreeLog(subP->eSelector, "eSelector", LmtPernotQuery); + kjTreeLog(subP->attrsSelector, "attrsSelector", LmtPernotQuery); + + dbEntityArray = mongocEntitiesQuery2(subP->eSelector, subP->attrsSelector, NULL, NULL, subP->lang, &count); LM_T(LmtPernot, ("mongocEntitiesQuery2 gave a count of %d", count)); @@ -155,7 +153,7 @@ static void* pernotTreat(void* vP) { orionldState.uriParams.offset = entitiesSent; - dbEntityArray = mongocEntitiesQuery2(eSelector, attrsArray, qTree, geoInfoP, subP->lang, NULL); + dbEntityArray = mongocEntitiesQuery2(subP->eSelector, subP->attrsSelector, NULL, NULL, subP->lang, NULL); apiEntityArray = dbModelToApiEntities(dbEntityArray, subP->sysAttrs, subP->renderFormat, subP->lang); if (pernotSend(subP, apiEntityArray) == false) diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test index 94ce01c0dc..c3fe1c1d52 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test @@ -468,9 +468,9 @@ Link: Date: Thu, 31 Aug 2023 18:29:25 +0200 Subject: [PATCH 10/29] Started with q for pernot - committing just to refresh with develop --- src/lib/orionld/pernot/PernotSubscription.h | 2 +- src/lib/orionld/pernot/pernotSubCacheAdd.cpp | 2 +- src/lib/orionld/pernot/pernotTreat.cpp | 5 +++-- .../cases/0000_ngsild/ngsild_pernot_queries.test | 7 +++++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lib/orionld/pernot/PernotSubscription.h b/src/lib/orionld/pernot/PernotSubscription.h index d3490c4e75..f862521f94 100644 --- a/src/lib/orionld/pernot/PernotSubscription.h +++ b/src/lib/orionld/pernot/PernotSubscription.h @@ -83,7 +83,7 @@ typedef struct PernotSubscription // For the Query KjNode* eSelector; KjNode* attrsSelector; - QNode* qTreeSelector; + QNode* qSelector; // Counters uint32_t notificationAttemptsDb; // Total number of notification attempts, in DB diff --git a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp index 3fd426c10a..c8f4077185 100644 --- a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp +++ b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp @@ -173,7 +173,7 @@ PernotSubscription* pernotSubCacheAdd // Query parameters pSubP->eSelector = kjLookup(pSubP->kjSubP, "entities"); pSubP->attrsSelector = kjLookup(notificationP, "attributes"); - pSubP->qTreeSelector = qTree; // I probably need to clone this ... + pSubP->qSelector = qTree; // I probably need to clone this ... // notification::endpoint diff --git a/src/lib/orionld/pernot/pernotTreat.cpp b/src/lib/orionld/pernot/pernotTreat.cpp index ec5a8d9278..648b3936d8 100644 --- a/src/lib/orionld/pernot/pernotTreat.cpp +++ b/src/lib/orionld/pernot/pernotTreat.cpp @@ -125,8 +125,9 @@ static void* pernotTreat(void* vP) kjTreeLog(subP->eSelector, "eSelector", LmtPernotQuery); kjTreeLog(subP->attrsSelector, "attrsSelector", LmtPernotQuery); + LM_T(LmtPernotQuery, ("qSelector at %p", subP->qSelector)); - dbEntityArray = mongocEntitiesQuery2(subP->eSelector, subP->attrsSelector, NULL, NULL, subP->lang, &count); + dbEntityArray = mongocEntitiesQuery2(subP->eSelector, subP->attrsSelector, subP->qSelector, NULL, subP->lang, &count); LM_T(LmtPernot, ("mongocEntitiesQuery2 gave a count of %d", count)); @@ -153,7 +154,7 @@ static void* pernotTreat(void* vP) { orionldState.uriParams.offset = entitiesSent; - dbEntityArray = mongocEntitiesQuery2(subP->eSelector, subP->attrsSelector, NULL, NULL, subP->lang, NULL); + dbEntityArray = mongocEntitiesQuery2(subP->eSelector, subP->attrsSelector, subP->qSelector, NULL, subP->lang, NULL); apiEntityArray = dbModelToApiEntities(dbEntityArray, subP->sysAttrs, subP->renderFormat, subP->lang); if (pernotSend(subP, apiEntityArray) == false) diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test index 3fdacbf8a4..2856e52d8d 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test @@ -44,11 +44,12 @@ echo "========================================================================== # # Test also: # q +# geoQ # expiresAt # lang # jsonldContext # notification::format == keyValues/concise -# sysAttrs? +# notification::sysAttrs # payload='{ "id": "urn:S1", @@ -58,6 +59,7 @@ payload='{ "type": "T" } ], + "q": "P3==1", "timeInterval": 2, "notification": { "endpoint": { @@ -91,7 +93,8 @@ payload='{ "id": "urn:E2", "type": "T", "P1": 2, - "P2": 2 + "P2": 2, + "P3": 2 }' orionCurl --url /ngsi-ld/v1/entities --payload "$payload" echo From 454aa741bb5a1b0226b12d66433ad5db2fc1983e Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sun, 3 Sep 2023 16:54:20 +0200 Subject: [PATCH 11/29] Q Filter now also working for pernot subscriptions --- src/lib/cache/subCache.cpp | 2 +- src/lib/logMsg/traceLevels.h | 1 + .../notifications/subCacheAlterationMatch.cpp | 2 +- .../payloadCheck/pCheckSubscription.cpp | 30 +++++-- src/lib/orionld/payloadCheck/pcheckQ.cpp | 2 +- src/lib/orionld/pernot/pernotLoop.cpp | 14 ++-- src/lib/orionld/q/qBuild.cpp | 4 +- src/lib/orionld/q/qBuild.h | 2 +- src/lib/orionld/q/qPresent.cpp | 78 +++++++++---------- src/lib/orionld/q/qPresent.h | 4 +- .../orionldPostSubscriptions.cpp | 3 + .../0000_ngsild/ngsild_pernot_queries.test | 2 +- 12 files changed, 83 insertions(+), 61 deletions(-) diff --git a/src/lib/cache/subCache.cpp b/src/lib/cache/subCache.cpp index f9450455f5..06364f11df 100644 --- a/src/lib/cache/subCache.cpp +++ b/src/lib/cache/subCache.cpp @@ -958,7 +958,7 @@ bool subCacheItemInsert // FIXME: Instead of calling qBuild here, I should pass the pointer from pCheckSubscription if (cSubP->qP != NULL) qRelease(cSubP->qP); - cSubP->qP = qBuild(q.c_str(), &cSubP->qText, &validForV2, &isMq, true); // cSubP->qText needs real allocation + cSubP->qP = qBuild(q.c_str(), &cSubP->qText, &validForV2, &isMq, true, false); // cSubP->qText needs real allocation if (cSubP->qText != NULL) cSubP->qText = strdup(cSubP->qText); diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index 58dd6aee21..f8abd8ce76 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -101,6 +101,7 @@ typedef enum TraceLevels // LmtPernot = 100, // Periodic Notification Subscription cache LmtPernotLoop, // Pernot loop, when each sub is triggered in time + LmtPernotLoopTimes, // Pernot loop, details on timestamps LmtPernotFlush, // Pernot flush to DB LmtPernotQuery, // Pernot query diff --git a/src/lib/orionld/notifications/subCacheAlterationMatch.cpp b/src/lib/orionld/notifications/subCacheAlterationMatch.cpp index 2677ccab6f..2bf7bd81b0 100644 --- a/src/lib/orionld/notifications/subCacheAlterationMatch.cpp +++ b/src/lib/orionld/notifications/subCacheAlterationMatch.cpp @@ -1157,7 +1157,7 @@ OrionldAlterationMatch* subCacheAlterationMatch(OrionldAlteration* alterationLis // Only done if its an NGSI-LD operation AND if it's an NGSI-LD Subscription (ldContext has a value != "") // if ((subP->qP == NULL) && (subP->ldContext != "") && (subP->qText != NULL)) - subP->qP = qBuild(subP->qText, NULL, NULL, NULL, false); + subP->qP = qBuild(subP->qText, NULL, NULL, NULL, false, false); if (subP->qP != NULL) { diff --git a/src/lib/orionld/payloadCheck/pCheckSubscription.cpp b/src/lib/orionld/payloadCheck/pCheckSubscription.cpp index 466d0cb581..8ea76cf32f 100644 --- a/src/lib/orionld/payloadCheck/pCheckSubscription.cpp +++ b/src/lib/orionld/payloadCheck/pCheckSubscription.cpp @@ -27,6 +27,7 @@ extern "C" #include "kbase/kMacros.h" // K_FT #include "kjson/KjNode.h" // KjNode #include "kjson/kjBuilder.h" // kjChildRemove +#include "kjson/kjLookup.h" // kjLookup } #include "logMsg/logMsg.h" // LM_* @@ -164,6 +165,27 @@ bool pCheckSubscription } } + // + // First we need to know whether it is a n ormal subscription or a periodic notification subscription (timeInterval) + // + bool pernot = false; + + KjNode* timeIntervalNodeP = kjLookup(subP, "timeInterval"); + if (timeIntervalNodeP != NULL) + { + PCHECK_NUMBER(timeIntervalNodeP, 0, NULL, SubscriptionTimeIntervalPath, 400); + PCHECK_NUMBER_GT(timeIntervalNodeP, 0, "Non-supported timeInterval (must be greater than zero)", SubscriptionTimeIntervalPath, 400, 0); + + pernot = true; + *timeInterval = (timeIntervalNodeP->type == KjInt)? timeIntervalNodeP->value.i : timeIntervalNodeP->value.f; + } + else + *timeInterval = 0; + + + // + // Now loop over the entiry payload + // KjNode* subItemP = subP->value.firstChildP; KjNode* next; while (subItemP != NULL) @@ -201,19 +223,13 @@ bool pCheckSubscription return false; } else if (strcmp(subItemP->name, "timeInterval") == 0) - { PCHECK_DUPLICATE(timeIntervalP, subItemP, 0, NULL, SubscriptionTimeIntervalPath, 400); - PCHECK_NUMBER(timeIntervalP, 0, NULL, SubscriptionTimeIntervalPath, 400); - PCHECK_NUMBER_GT(timeIntervalP, 0, "Non-supported timeInterval (must be greater than zero)", SubscriptionTimeIntervalPath, 400, 0); - - *timeInterval = (timeIntervalP->type == KjInt)? timeIntervalP->value.i : timeIntervalP->value.f; - } else if (strcmp(subItemP->name, "q") == 0) { PCHECK_DUPLICATE(qP, subItemP, 0, NULL, SubscriptionQPath, 400); PCHECK_STRING(qP, 0, NULL, SubscriptionQPath, 400); - *qTreeP = qBuild(qP->value.s, qRenderedForDbP, qValidForV2P, qIsMqP, true); // 5th parameter: qToDbModel == true + *qTreeP = qBuild(qP->value.s, qRenderedForDbP, qValidForV2P, qIsMqP, true, pernot); // 5th parameter: qToDbModel == true *qNodeP = qP; if (*qTreeP == NULL) diff --git a/src/lib/orionld/payloadCheck/pcheckQ.cpp b/src/lib/orionld/payloadCheck/pcheckQ.cpp index 82678f5412..488bc71414 100644 --- a/src/lib/orionld/payloadCheck/pcheckQ.cpp +++ b/src/lib/orionld/payloadCheck/pcheckQ.cpp @@ -65,6 +65,6 @@ QNode* pcheckQ(char* qString) return NULL; } - qPresent(qTree, "QP", "pcheckQ"); + qPresent(qTree, "QP", "pcheckQ", LmtQ); return qTree; } diff --git a/src/lib/orionld/pernot/pernotLoop.cpp b/src/lib/orionld/pernot/pernotLoop.cpp index 7f9c903faf..4d00a26bbb 100644 --- a/src/lib/orionld/pernot/pernotLoop.cpp +++ b/src/lib/orionld/pernot/pernotLoop.cpp @@ -156,13 +156,13 @@ static void* pernotLoop(void* vP) } double diffTime = subP->lastNotificationTime + subP->timeInterval - now; - LM_T(LmtPernotLoop, ("%s: lastNotificationTime: %f", subP->subscriptionId, subP->lastNotificationTime)); - LM_T(LmtPernotLoop, ("%s: now: %f", subP->subscriptionId, now)); - LM_T(LmtPernotLoop, ("%s: diffTime: %f", subP->subscriptionId, diffTime)); - LM_T(LmtPernotLoop, ("%s: noMatch: %d", subP->subscriptionId, subP->noMatch)); - LM_T(LmtPernotLoop, ("%s: noMatchDb: %d", subP->subscriptionId, subP->noMatchDb)); - LM_T(LmtPernotLoop, ("%s: notificationAttempts: %d", subP->subscriptionId, subP->notificationAttempts)); - LM_T(LmtPernotLoop, ("%s: notificationAttemptsDb: %d", subP->subscriptionId, subP->notificationAttemptsDb)); + LM_T(LmtPernotLoopTimes, ("%s: lastNotificationTime: %f", subP->subscriptionId, subP->lastNotificationTime)); + LM_T(LmtPernotLoopTimes, ("%s: now: %f", subP->subscriptionId, now)); + LM_T(LmtPernotLoopTimes, ("%s: diffTime: %f", subP->subscriptionId, diffTime)); + LM_T(LmtPernotLoopTimes, ("%s: noMatch: %d", subP->subscriptionId, subP->noMatch)); + LM_T(LmtPernotLoopTimes, ("%s: noMatchDb: %d", subP->subscriptionId, subP->noMatchDb)); + LM_T(LmtPernotLoopTimes, ("%s: notificationAttempts: %d", subP->subscriptionId, subP->notificationAttempts)); + LM_T(LmtPernotLoopTimes, ("%s: notificationAttemptsDb: %d", subP->subscriptionId, subP->notificationAttemptsDb)); if (diffTime <= 0) { diff --git a/src/lib/orionld/q/qBuild.cpp b/src/lib/orionld/q/qBuild.cpp index ba45e0235f..0c3b9f7bed 100644 --- a/src/lib/orionld/q/qBuild.cpp +++ b/src/lib/orionld/q/qBuild.cpp @@ -67,7 +67,7 @@ // isMqP output parameter indicating whether the q para meter corresponds to 'mq' in NGSIv2 // qToDbModel transform the variables in 'q' to look like they look in the q in the database // -QNode* qBuild(const char* q, char** qRenderP, bool* v2ValidP, bool* isMqP, bool qToDbModel) +QNode* qBuild(const char* q, char** qRenderP, bool* v2ValidP, bool* isMqP, bool qToDbModel, bool pernot) { QNode* qP = NULL; char* title; @@ -104,7 +104,7 @@ QNode* qBuild(const char* q, char** qRenderP, bool* v2ValidP, bool* isMqP, bool } } - qP = qParse(qList, NULL, false, qToDbModel, &title, &detail); // 3rd parameter: forDb=false + qP = qParse(qList, NULL, pernot, qToDbModel, &title, &detail); // 3rd parameter: 'forDb' false unless pernot subscription if (qP == NULL) { orionldError(OrionldBadRequestData, "Invalid Q-Filter", detail, 400); diff --git a/src/lib/orionld/q/qBuild.h b/src/lib/orionld/q/qBuild.h index effd155bea..42eb2f607a 100644 --- a/src/lib/orionld/q/qBuild.h +++ b/src/lib/orionld/q/qBuild.h @@ -55,6 +55,6 @@ // v2ValidP output parameter indicating whether the q string id valid for NGSIv2 // isMdP output parameter indicating whether the q para meter corresponds to 'mq' in NGSIv2 // -extern QNode* qBuild(const char* q, char** qRenderP, bool* v2ValidP, bool* isMdP, bool qToDbModel); +extern QNode* qBuild(const char* q, char** qRenderP, bool* v2ValidP, bool* isMdP, bool qToDbModel, bool pernot); #endif // SRC_LIB_ORIONLD_Q_QBUILD_H_ diff --git a/src/lib/orionld/q/qPresent.cpp b/src/lib/orionld/q/qPresent.cpp index 7c18e27e4c..a91e2c9f50 100644 --- a/src/lib/orionld/q/qPresent.cpp +++ b/src/lib/orionld/q/qPresent.cpp @@ -24,7 +24,7 @@ */ #include // memset -#include "logMsg/logMsg.h" // LM_* +#include "logMsg/logMsg.h" // TraceLevels, LM_* #include "orionld/q/QNode.h" // QNode #include "orionld/q/qPresent.h" // Own interface @@ -35,7 +35,7 @@ // // qTreePresent - // -static void qTreePresent(QNode* qP, int indent, const char* prefix) +static void qTreePresent(QNode* qP, int indent, const char* prefix, TraceLevels tLevel) { char indentV[100]; @@ -44,78 +44,78 @@ static void qTreePresent(QNode* qP, int indent, const char* prefix) if (qP->type == QNodeEQ) { - LM_T(LmtQ, ("%s:%sEQ:", prefix, indentV)); - qTreePresent(qP->value.children, indent+2, prefix); - qTreePresent(qP->value.children->next, indent+2, prefix); + LM_T(tLevel, ("%s:%sEQ:", prefix, indentV)); + qTreePresent(qP->value.children, indent+2, prefix, tLevel); + qTreePresent(qP->value.children->next, indent+2, prefix, tLevel); } else if (qP->type == QNodeNE) { - LM_T(LmtQ, ("%s:%sNE:", prefix, indentV)); - qTreePresent(qP->value.children, indent+2, prefix); - qTreePresent(qP->value.children->next, indent+2, prefix); + LM_T(tLevel, ("%s:%sNE:", prefix, indentV)); + qTreePresent(qP->value.children, indent+2, prefix, tLevel); + qTreePresent(qP->value.children->next, indent+2, prefix, tLevel); } else if (qP->type == QNodeLT) { - LM_T(LmtQ, ("%s:%sLT:", prefix, indentV)); - qTreePresent(qP->value.children, indent+2, prefix); - qTreePresent(qP->value.children->next, indent+2, prefix); + LM_T(tLevel, ("%s:%sLT:", prefix, indentV)); + qTreePresent(qP->value.children, indent+2, prefix, tLevel); + qTreePresent(qP->value.children->next, indent+2, prefix, tLevel); } else if (qP->type == QNodeLE) { - LM_T(LmtQ, ("%s:%sLE:", prefix, indentV)); - qTreePresent(qP->value.children, indent+2, prefix); - qTreePresent(qP->value.children->next, indent+2, prefix); + LM_T(tLevel, ("%s:%sLE:", prefix, indentV)); + qTreePresent(qP->value.children, indent+2, prefix, tLevel); + qTreePresent(qP->value.children->next, indent+2, prefix, tLevel); } else if (qP->type == QNodeGT) { - LM_T(LmtQ, ("%s:%sGT:", prefix, indentV)); - qTreePresent(qP->value.children, indent+2, prefix); - qTreePresent(qP->value.children->next, indent+2, prefix); + LM_T(tLevel, ("%s:%sGT:", prefix, indentV)); + qTreePresent(qP->value.children, indent+2, prefix, tLevel); + qTreePresent(qP->value.children->next, indent+2, prefix, tLevel); } else if (qP->type == QNodeGE) { - LM_T(LmtQ, ("%s:%sGE:", prefix, indentV)); - qTreePresent(qP->value.children, indent+2, prefix); - qTreePresent(qP->value.children->next, indent+2, prefix); + LM_T(tLevel, ("%s:%sGE:", prefix, indentV)); + qTreePresent(qP->value.children, indent+2, prefix, tLevel); + qTreePresent(qP->value.children->next, indent+2, prefix, tLevel); } else if (qP->type == QNodeVariable) - LM_T(LmtQ, ("%s:%s%s (Variable) (v at %p, qP at %p)", prefix, indentV, qP->value.v, qP->value.v, qP)); + LM_T(tLevel, ("%s:%s%s (Variable) (v at %p, qP at %p)", prefix, indentV, qP->value.v, qP->value.v, qP)); else if (qP->type == QNodeIntegerValue) - LM_T(LmtQ, ("%s:%s%d (Int)", prefix, indentV, qP->value.i)); + LM_T(tLevel, ("%s:%s%d (Int)", prefix, indentV, qP->value.i)); else if (qP->type == QNodeFloatValue) - LM_T(LmtQ, ("%s:%s%f (Float)", prefix, indentV, qP->value.f)); + LM_T(tLevel, ("%s:%s%f (Float)", prefix, indentV, qP->value.f)); else if (qP->type == QNodeStringValue) - LM_T(LmtQ, ("%s:%s%s (String) at %p (String at %p)", prefix, indentV, qP->value.s, qP, qP->value.s)); + LM_T(tLevel, ("%s:%s%s (String) at %p (String at %p)", prefix, indentV, qP->value.s, qP, qP->value.s)); else if (qP->type == QNodeTrueValue) - LM_T(LmtQ, ("%s:%sTRUE (Bool)", prefix, indentV)); + LM_T(tLevel, ("%s:%sTRUE (Bool)", prefix, indentV)); else if (qP->type == QNodeFalseValue) - LM_T(LmtQ, ("%s:%sFALSE (Bool)", prefix, indentV)); + LM_T(tLevel, ("%s:%sFALSE (Bool)", prefix, indentV)); else if (qP->type == QNodeExists) { - LM_T(LmtQ, ("%s:%s Exists (at %p):", prefix, indentV, qP)); - qTreePresent(qP->value.children, indent+2, prefix); + LM_T(tLevel, ("%s:%s Exists (at %p):", prefix, indentV, qP)); + qTreePresent(qP->value.children, indent+2, prefix, tLevel); } else if (qP->type == QNodeNotExists) { - LM_T(LmtQ, ("%s:%s Not Exists:", prefix, indentV)); - qTreePresent(qP->value.children, indent+2, prefix); + LM_T(tLevel, ("%s:%s Not Exists:", prefix, indentV)); + qTreePresent(qP->value.children, indent+2, prefix, tLevel); } else if (qP->type == QNodeOr) { - LM_T(LmtQ, ("%s:%sOR:", prefix, indentV)); + LM_T(tLevel, ("%s:%sOR:", prefix, indentV)); indent+=2; for (QNode* childP = qP->value.children; childP != NULL; childP = childP->next) - qTreePresent(childP, indent, prefix); + qTreePresent(childP, indent, prefix, tLevel); } else if (qP->type == QNodeAnd) { - LM_T(LmtQ, ("%s:%sAND:", prefix, indentV)); + LM_T(tLevel, ("%s:%sAND:", prefix, indentV)); indent+=2; for (QNode* childP = qP->value.children; childP != NULL; childP = childP->next) - qTreePresent(childP, indent, prefix); + qTreePresent(childP, indent, prefix, tLevel); } else - LM_T(LmtQ, ("%s:%s%s (presentation TBI)", prefix, indentV, qNodeType(qP->type))); + LM_T(tLevel, ("%s:%s%s (presentation TBI)", prefix, indentV, qNodeType(qP->type))); } @@ -124,11 +124,11 @@ static void qTreePresent(QNode* qP, int indent, const char* prefix) // // qPresent - // -void qPresent(QNode* qP, const char* prefix, const char* what) +void qPresent(QNode* qP, const char* prefix, const char* what, TraceLevels tLevel) { - LM_T(LmtQ, ("%s: --------------------- %s -----------------------------------", prefix, what)); - qTreePresent(qP, 0, prefix); - LM_T(LmtQ, ("%s: --------------------------------------------------------", prefix)); + LM_T(tLevel, ("%s: --------------------- %s -----------------------------------", prefix, what)); + qTreePresent(qP, 0, prefix, tLevel); + LM_T(tLevel, ("%s: --------------------------------------------------------", prefix)); } diff --git a/src/lib/orionld/q/qPresent.h b/src/lib/orionld/q/qPresent.h index 53f7d82081..1a2f847bf2 100644 --- a/src/lib/orionld/q/qPresent.h +++ b/src/lib/orionld/q/qPresent.h @@ -25,6 +25,8 @@ * * Author: Ken Zangelin */ +#include "logMsg/logMsg.h" // TraceLevels + #include "orionld/q/QNode.h" // QNode @@ -33,7 +35,7 @@ // // qPresent - // -extern void qPresent(QNode* qP, const char* prefix, const char* what); +extern void qPresent(QNode* qP, const char* prefix, const char* what, TraceLevels tLevel); diff --git a/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp b/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp index 4f2aa383bf..3d532f1c26 100644 --- a/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp +++ b/src/lib/orionld/serviceRoutines/orionldPostSubscriptions.cpp @@ -282,6 +282,9 @@ bool orionldPostSubscriptions(void) else { // Add subscription to the pernot-cache + LM_T(LmtPernot, ("qRenderedForDb: '%s'", qRenderedForDb)); + if (qTree != NULL) + qPresent(qTree, "Pernot", "Q for pernot subscription", LmtPernot); pSubP = pernotSubCacheAdd(subscriptionId, subP, endpointP, diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test index 2856e52d8d..e91d482d7a 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test @@ -59,7 +59,7 @@ payload='{ "type": "T" } ], - "q": "P3==1", + "q": "P3==1|P3==2", "timeInterval": 2, "notification": { "endpoint": { From 70d1b1f8ac5c1823b27d30162349c48de05c4c37 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sun, 3 Sep 2023 18:29:14 +0200 Subject: [PATCH 12/29] tested also sysAttrs, format for pernot subscriptions - all OK --- .../cases/0000_ngsild/ngsild_pernot_queries.test | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test index e91d482d7a..a068d77996 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot_queries.test @@ -43,13 +43,10 @@ echo "01. Create a pernot subscription on entity type T only and with a time int echo "============================================================================================" # # Test also: -# q # geoQ # expiresAt # lang # jsonldContext -# notification::format == keyValues/concise -# notification::sysAttrs # payload='{ "id": "urn:S1", @@ -65,7 +62,9 @@ payload='{ "endpoint": { "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" }, - "attributes": [ "P1", "P2" ] + "attributes": [ "P1", "P2" ], + "sysAttrs": false, + "format": "normalized" } }' orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" From 78e9718ddc66ddd98100ae46b7d148221d56967c Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Mon, 4 Sep 2023 11:52:49 +0200 Subject: [PATCH 13/29] Better 501 handling of temporal operations --- CHANGES_NEXT_RELEASE | 1 + src/app/orionld/orionldRestServices.cpp | 69 ++++++----- src/lib/orionld/rest/OrionLdRestService.h | 2 + .../orionld/rest/orionldMhdConnectionInit.cpp | 7 ++ .../rest/orionldMhdConnectionTreat.cpp | 15 +++ src/lib/orionld/rest/orionldServiceInit.cpp | 30 ++++- .../orionld/serviceRoutines/CMakeLists.txt | 6 +- .../orionldDeleteTemporalAttribute.cpp | 43 +++++++ .../orionldDeleteTemporalAttribute.h | 37 ++++++ ...orionldDeleteTemporalAttributeInstance.cpp | 43 +++++++ .../orionldDeleteTemporalAttributeInstance.h | 37 ++++++ .../orionldDeleteTemporalEntity.cpp | 43 +++++++ .../orionldDeleteTemporalEntity.h | 37 ++++++ .../orionldPatchTemporalAttributeInstance.cpp | 43 +++++++ .../orionldPatchTemporalAttributeInstance.h | 37 ++++++ .../orionldPostTemporalAttributes.cpp | 43 +++++++ .../orionldPostTemporalAttributes.h | 37 ++++++ .../ngsild_temporal_representation_501.test | 109 ++++++++++++++---- 18 files changed, 584 insertions(+), 55 deletions(-) create mode 100644 src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttribute.cpp create mode 100644 src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttribute.h create mode 100644 src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.cpp create mode 100644 src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.h create mode 100644 src/lib/orionld/serviceRoutines/orionldDeleteTemporalEntity.cpp create mode 100644 src/lib/orionld/serviceRoutines/orionldDeleteTemporalEntity.h create mode 100644 src/lib/orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.cpp create mode 100644 src/lib/orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.h create mode 100644 src/lib/orionld/serviceRoutines/orionldPostTemporalAttributes.cpp create mode 100644 src/lib/orionld/serviceRoutines/orionldPostTemporalAttributes.h diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 6cbb84ae51..6fb469aee3 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -5,3 +5,4 @@ Fixed issues: * #280 - Changed all error codes from InvalidRequest to BadRequestData, except for JSON Parse Error that is still an InvalidRequest * #280 - Giving errors for expiresAt in the past (for registrations and subscriptions) * #280 - Fixed a bug in error detection of downloading errors in arrays of contexts, at lower nesting levels + * #280 - Better 501 handling of temporal operations diff --git a/src/app/orionld/orionldRestServices.cpp b/src/app/orionld/orionldRestServices.cpp index 4c4bdb3ccc..2c518b0e41 100644 --- a/src/app/orionld/orionldRestServices.cpp +++ b/src/app/orionld/orionldRestServices.cpp @@ -50,7 +50,6 @@ #include "orionld/serviceRoutines/orionldGetContexts.h" #include "orionld/serviceRoutines/orionldGetVersion.h" #include "orionld/serviceRoutines/orionldGetPing.h" -#include "orionld/serviceRoutines/orionldNotImplemented.h" #include "orionld/serviceRoutines/orionldPostBatchUpsert.h" #include "orionld/serviceRoutines/orionldPostBatchCreate.h" #include "orionld/serviceRoutines/orionldPostBatchUpdate.h" @@ -61,16 +60,22 @@ #include "orionld/serviceRoutines/orionldGetTenants.h" #include "orionld/serviceRoutines/orionldGetDbIndexes.h" #include "orionld/serviceRoutines/orionldPostQuery.h" -#include "orionld/serviceRoutines/orionldGetTemporalEntities.h" -#include "orionld/serviceRoutines/orionldGetTemporalEntity.h" -#include "orionld/serviceRoutines/orionldPostTemporalQuery.h" -#include "orionld/serviceRoutines/orionldPostTemporalEntities.h" #include "orionld/serviceRoutines/orionldPostContexts.h" #include "orionld/serviceRoutines/orionldDeleteContext.h" #include "orionld/serviceRoutines/orionldOptions.h" #include "orionld/serviceRoutines/orionldPutEntity.h" #include "orionld/serviceRoutines/orionldPutAttribute.h" +#include "orionld/serviceRoutines/orionldGetTemporalEntities.h" +#include "orionld/serviceRoutines/orionldGetTemporalEntity.h" +#include "orionld/serviceRoutines/orionldPostTemporalQuery.h" +#include "orionld/serviceRoutines/orionldPostTemporalEntities.h" +#include "orionld/serviceRoutines/orionldDeleteTemporalAttribute.h" // orionldDeleteTemporalAttribute +#include "orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.h" // orionldDeleteTemporalAttributeInstance +#include "orionld/serviceRoutines/orionldDeleteTemporalEntity.h" // orionldDeleteTemporalEntity +#include "orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.h" // orionldPatchTemporalAttributeInstance +#include "orionld/serviceRoutines/orionldPostTemporalAttributes.h" // orionldPostTemporalAttributes + #include "orionld/rest/OrionLdRestService.h" // OrionLdRestServiceSimplified #include "orionld/orionldRestServices.h" // Own Interface @@ -111,20 +116,20 @@ static const int getServices = (sizeof(getServiceV) / sizeof(getServiceV[0])); // static OrionLdRestServiceSimplified postServiceV[] = { - { "/ngsi-ld/v1/entities/*/attrs", orionldPostEntity }, - { "/ngsi-ld/v1/entities", orionldPostEntities }, - { "/ngsi-ld/ex/v1/notify", orionldPostNotify }, - { "/ngsi-ld/v1/entityOperations/create", orionldPostBatchCreate }, - { "/ngsi-ld/v1/entityOperations/upsert", orionldPostBatchUpsert }, - { "/ngsi-ld/v1/entityOperations/update", orionldPostBatchUpdate }, - { "/ngsi-ld/v1/entityOperations/delete", orionldPostBatchDelete }, - { "/ngsi-ld/v1/entityOperations/query", orionldPostQuery }, - { "/ngsi-ld/v1/subscriptions", orionldPostSubscriptions }, - { "/ngsi-ld/v1/csourceRegistrations", orionldPostRegistrations }, - { "/ngsi-ld/v1/temporal/entities/*/attrs", orionldNotImplemented }, - { "/ngsi-ld/v1/temporal/entities", orionldPostTemporalEntities }, - { "/ngsi-ld/v1/temporal/entityOperations/query", orionldPostTemporalQuery }, - { "/ngsi-ld/v1/jsonldContexts", orionldPostContexts } + { "/ngsi-ld/v1/entities/*/attrs", orionldPostEntity }, + { "/ngsi-ld/v1/entities", orionldPostEntities }, + { "/ngsi-ld/ex/v1/notify", orionldPostNotify }, + { "/ngsi-ld/v1/entityOperations/create", orionldPostBatchCreate }, + { "/ngsi-ld/v1/entityOperations/upsert", orionldPostBatchUpsert }, + { "/ngsi-ld/v1/entityOperations/update", orionldPostBatchUpdate }, + { "/ngsi-ld/v1/entityOperations/delete", orionldPostBatchDelete }, + { "/ngsi-ld/v1/entityOperations/query", orionldPostQuery }, + { "/ngsi-ld/v1/subscriptions", orionldPostSubscriptions }, + { "/ngsi-ld/v1/csourceRegistrations", orionldPostRegistrations }, + { "/ngsi-ld/v1/temporal/entities/*/attrs", orionldPostTemporalAttributes }, + { "/ngsi-ld/v1/temporal/entities", orionldPostTemporalEntities }, + { "/ngsi-ld/v1/temporal/entityOperations/query", orionldPostTemporalQuery }, + { "/ngsi-ld/v1/jsonldContexts", orionldPostContexts } }; static const int postServices = (sizeof(postServiceV) / sizeof(postServiceV[0])); @@ -136,12 +141,12 @@ static const int postServices = (sizeof(postServiceV) / sizeof(postServiceV[0])) // static OrionLdRestServiceSimplified patchServiceV[] = { - { "/ngsi-ld/v1/entities/*/attrs/*", orionldPatchAttribute }, - { "/ngsi-ld/v1/entities/*/attrs", orionldPatchEntity }, - { "/ngsi-ld/v1/entities/*", orionldPatchEntity2 }, - { "/ngsi-ld/v1/subscriptions/*", orionldPatchSubscription }, - { "/ngsi-ld/v1/csourceRegistrations/*", orionldPatchRegistration }, - { "/ngsi-ld/v1/temporal/entities/*/attrs/*/*", orionldNotImplemented } + { "/ngsi-ld/v1/entities/*/attrs/*", orionldPatchAttribute }, + { "/ngsi-ld/v1/entities/*/attrs", orionldPatchEntity }, + { "/ngsi-ld/v1/entities/*", orionldPatchEntity2 }, + { "/ngsi-ld/v1/subscriptions/*", orionldPatchSubscription }, + { "/ngsi-ld/v1/csourceRegistrations/*", orionldPatchRegistration }, + { "/ngsi-ld/v1/temporal/entities/*/attrs/*/*", orionldPatchTemporalAttributeInstance } }; static const int patchServices = (sizeof(patchServiceV) / sizeof(patchServiceV[0])); @@ -166,13 +171,13 @@ static const int putServices = (sizeof(putServiceV) / sizeof(putServiceV[0])); // static OrionLdRestServiceSimplified deleteServiceV[] = { - { "/ngsi-ld/v1/entities/*/attrs/*", orionldDeleteAttribute }, - { "/ngsi-ld/v1/entities/*", orionldDeleteEntity }, - { "/ngsi-ld/v1/subscriptions/*", orionldDeleteSubscription }, - { "/ngsi-ld/v1/csourceRegistrations/*", orionldDeleteRegistration }, - { "/ngsi-ld/v1/jsonldContexts/*", orionldDeleteContext }, - { "/ngsi-ld/v1/temporal/entities/*/attrs/*", orionldNotImplemented }, - { "/ngsi-ld/v1/temporal/entities/*", orionldNotImplemented } + { "/ngsi-ld/v1/entities/*/attrs/*", orionldDeleteAttribute }, + { "/ngsi-ld/v1/entities/*", orionldDeleteEntity }, + { "/ngsi-ld/v1/subscriptions/*", orionldDeleteSubscription }, + { "/ngsi-ld/v1/csourceRegistrations/*", orionldDeleteRegistration }, + { "/ngsi-ld/v1/jsonldContexts/*", orionldDeleteContext }, + { "/ngsi-ld/v1/temporal/entities/*/attrs/*", orionldDeleteTemporalAttribute }, + { "/ngsi-ld/v1/temporal/entities/*", orionldDeleteTemporalEntity }, }; static const int deleteServices = (sizeof(deleteServiceV) / sizeof(deleteServiceV[0])); diff --git a/src/lib/orionld/rest/OrionLdRestService.h b/src/lib/orionld/rest/OrionLdRestService.h index c8b762d0c0..358ec62459 100644 --- a/src/lib/orionld/rest/OrionLdRestService.h +++ b/src/lib/orionld/rest/OrionLdRestService.h @@ -179,6 +179,8 @@ typedef struct OrionLdRestService uint32_t options; // Peculiarities of this type of requests (bitmask) uint64_t uriParams; // Supported URI parameters (bitmask) bool isBatchOp; // true for BATCH operations + bool notImplemented; // Flags that the service hasn't been implemented + bool mintaka; // Flags that the service is not for Orion-LD but for Mintaka } OrionLdRestService; diff --git a/src/lib/orionld/rest/orionldMhdConnectionInit.cpp b/src/lib/orionld/rest/orionldMhdConnectionInit.cpp index a76d361295..737b1b663d 100644 --- a/src/lib/orionld/rest/orionldMhdConnectionInit.cpp +++ b/src/lib/orionld/rest/orionldMhdConnectionInit.cpp @@ -1049,6 +1049,13 @@ MHD_Result orionldMhdConnectionInit if (orionldState.serviceP == NULL) // 405 or 404 - no need to continue - prettyPrint not possible here return MHD_YES; + if (orionldState.serviceP->mintaka == true) + return MHD_YES; + + if (orionldState.serviceP->notImplemented == true) + return MHD_YES; + + // // 5. GET URI params // diff --git a/src/lib/orionld/rest/orionldMhdConnectionTreat.cpp b/src/lib/orionld/rest/orionldMhdConnectionTreat.cpp index b898eded56..135efc628e 100644 --- a/src/lib/orionld/rest/orionldMhdConnectionTreat.cpp +++ b/src/lib/orionld/rest/orionldMhdConnectionTreat.cpp @@ -940,6 +940,21 @@ MHD_Result orionldMhdConnectionTreat(void) promCounterIncrease(promNgsildRequests); + if (orionldState.serviceP == NULL) + goto respond; + + if (orionldState.serviceP->mintaka == true) + { + serviceRoutineResult = orionldState.serviceP->serviceRoutine(); + goto respond; + } + + if (orionldState.serviceP->notImplemented == true) + { + serviceRoutineResult = orionldState.serviceP->serviceRoutine(); + goto respond; + } + // If OPTIONS verb, we skip all checks, go straight to the service routine if (orionldState.verb == OPTIONS) goto serviceRoutine; diff --git a/src/lib/orionld/rest/orionldServiceInit.cpp b/src/lib/orionld/rest/orionldServiceInit.cpp index 3b03e0654e..002329331f 100644 --- a/src/lib/orionld/rest/orionldServiceInit.cpp +++ b/src/lib/orionld/rest/orionldServiceInit.cpp @@ -73,13 +73,23 @@ #include "orionld/serviceRoutines/orionldGetEntityType.h" // orionldGetEntityType #include "orionld/serviceRoutines/orionldGetEntityAttributes.h" // orionldGetEntityAttributes #include "orionld/serviceRoutines/orionldGetEntityAttribute.h" // orionldGetEntityAttribute -#include "orionld/serviceRoutines/orionldPostTemporalEntities.h" // orionldPostTemporalEntities + #include "orionld/serviceRoutines/orionldGetContexts.h" // orionldGetContexts #include "orionld/serviceRoutines/orionldGetContext.h" // orionldGetContext #include "orionld/serviceRoutines/orionldPostContexts.h" // orionldPostContexts #include "orionld/serviceRoutines/orionldDeleteContext.h" // orionldDeleteContext #include "orionld/serviceRoutines/orionldPostNotify.h" // orionldPostNotify +#include "orionld/serviceRoutines/orionldGetTemporalEntities.h" // orionldGetTemporalEntities +#include "orionld/serviceRoutines/orionldGetTemporalEntity.h" // orionldGetTemporalEntity +#include "orionld/serviceRoutines/orionldPostTemporalQuery.h" // orionldPostTemporalQuery +#include "orionld/serviceRoutines/orionldPostTemporalEntities.h" // orionldPostTemporalEntities +#include "orionld/serviceRoutines/orionldDeleteTemporalAttribute.h" // orionldDeleteTemporalAttribute +#include "orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.h" // orionldDeleteTemporalAttributeInstance +#include "orionld/serviceRoutines/orionldDeleteTemporalEntity.h" // orionldDeleteTemporalEntity +#include "orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.h" // orionldPatchTemporalAttributeInstance +#include "orionld/serviceRoutines/orionldPostTemporalAttributes.h" // orionldPostTemporalAttributes + #include "orionld/troe/troePostEntities.h" // troePostEntities #include "orionld/troe/troePostBatchDelete.h" // troePostBatchDelete #include "orionld/troe/troeDeleteAttribute.h" // troeDeleteAttribute @@ -403,6 +413,24 @@ static void restServicePrepare(OrionLdRestService* serviceP, OrionLdRestServiceS serviceP->options |= ORIONLD_SERVICE_OPTION_PREFETCH_ID_AND_TYPE; serviceP->options |= ORIONLD_SERVICE_OPTION_EXPAND_TYPE; } + else if (serviceP->serviceRoutine == orionldGetTemporalEntities) + serviceP->mintaka = true; + else if (serviceP->serviceRoutine == orionldGetTemporalEntity) + serviceP->mintaka = true; + else if (serviceP->serviceRoutine == orionldPostTemporalQuery) + serviceP->mintaka = true; + else if (serviceP->serviceRoutine == orionldDeleteTemporalAttribute) + serviceP->notImplemented = true; + else if (serviceP->serviceRoutine == orionldDeleteTemporalAttributeInstance) + serviceP->notImplemented = true; + else if (serviceP->serviceRoutine == orionldDeleteTemporalEntity) + serviceP->notImplemented = true; + else if (serviceP->serviceRoutine == orionldPatchTemporalAttributeInstance) + serviceP->notImplemented = true; + else if (serviceP->serviceRoutine == orionldPostTemporalAttributes) + serviceP->notImplemented = true; + + else if (serviceP->serviceRoutine == orionldGetVersion) { serviceP->options = 0; // Tenant is Ignored diff --git a/src/lib/orionld/serviceRoutines/CMakeLists.txt b/src/lib/orionld/serviceRoutines/CMakeLists.txt index e31ce26b03..2d0b103cba 100644 --- a/src/lib/orionld/serviceRoutines/CMakeLists.txt +++ b/src/lib/orionld/serviceRoutines/CMakeLists.txt @@ -44,7 +44,6 @@ SET (SOURCES orionldGetContext.cpp orionldGetContexts.cpp orionldGetVersion.cpp - orionldNotImplemented.cpp orionldPostBatchCreate.cpp orionldPostBatchUpsert.cpp orionldPostBatchUpdate.cpp @@ -67,6 +66,11 @@ SET (SOURCES orionldOptions.cpp orionldPutEntity.cpp orionldPutAttribute.cpp + orionldDeleteTemporalAttribute.cpp + orionldDeleteTemporalAttributeInstance.cpp + orionldDeleteTemporalEntity.cpp + orionldPatchTemporalAttributeInstance.cpp + orionldPostTemporalAttributes.cpp ) # Include directories diff --git a/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttribute.cpp b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttribute.cpp new file mode 100644 index 0000000000..e0c67c5f25 --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttribute.cpp @@ -0,0 +1,43 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "logMsg/logMsg.h" + +#include "orionld/common/orionldState.h" // orionldState +#include "orionld/common/orionldError.h" // orionldError +#include "orionld/rest/OrionLdRestService.h" // OrionLdRestService +#include "orionld/serviceRoutines/orionldDeleteTemporalAttribute.h" // Own Interface + + + +// ---------------------------------------------------------------------------- +// +// orionldDeleteTemporalAttribute - +// +bool orionldDeleteTemporalAttribute(void) +{ + orionldError(OrionldOperationNotSupported, "Not Implemented", orionldState.serviceP->url, 501); + orionldState.noLinkHeader = true; // We don't want the Link header for non-implemented requests + return false; +} diff --git a/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttribute.h b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttribute.h new file mode 100644 index 0000000000..08e475ee29 --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttribute.h @@ -0,0 +1,37 @@ +#ifndef SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDDELETETEMPORALATTRIBUTE_H_ +#define SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDDELETETEMPORALATTRIBUTE_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ + + + +// ---------------------------------------------------------------------------- +// +// orionldDeleteTemporalAttribute - +// +extern bool orionldDeleteTemporalAttribute(void); + +#endif // SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDDELETETEMPORALATTRIBUTE_H_ diff --git a/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.cpp b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.cpp new file mode 100644 index 0000000000..e586049d2c --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.cpp @@ -0,0 +1,43 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "logMsg/logMsg.h" + +#include "orionld/common/orionldState.h" // orionldState +#include "orionld/common/orionldError.h" // orionldError +#include "orionld/rest/OrionLdRestService.h" // OrionLdRestService +#include "orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.h" // Own Interface + + + +// ---------------------------------------------------------------------------- +// +// orionldDeleteTemporalAttributeInstance - +// +bool orionldDeleteTemporalAttributeInstance(void) +{ + orionldError(OrionldOperationNotSupported, "Not Implemented", orionldState.serviceP->url, 501); + orionldState.noLinkHeader = true; // We don't want the Link header for non-implemented requests + return false; +} diff --git a/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.h b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.h new file mode 100644 index 0000000000..4c81de9e71 --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalAttributeInstance.h @@ -0,0 +1,37 @@ +#ifndef SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDDELETETEMPORALATTRIBUTEINSTANCE_H_ +#define SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDDELETETEMPORALATTRIBUTEINSTANCE_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ + + + +// ---------------------------------------------------------------------------- +// +// orionldDeleteTemporalAttributeInstance - +// +extern bool orionldDeleteTemporalAttributeInstance(void); + +#endif // SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDDELETETEMPORALATTRIBUTEINSTANCE_H_ diff --git a/src/lib/orionld/serviceRoutines/orionldDeleteTemporalEntity.cpp b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalEntity.cpp new file mode 100644 index 0000000000..c76acdeecb --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalEntity.cpp @@ -0,0 +1,43 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "logMsg/logMsg.h" + +#include "orionld/common/orionldState.h" // orionldState +#include "orionld/common/orionldError.h" // orionldError +#include "orionld/rest/OrionLdRestService.h" // OrionLdRestService +#include "orionld/serviceRoutines/orionldDeleteTemporalEntity.h" // Own Interface + + + +// ---------------------------------------------------------------------------- +// +// orionldDeleteTemporalEntity - +// +bool orionldDeleteTemporalEntity(void) +{ + orionldError(OrionldOperationNotSupported, "Not Implemented", orionldState.serviceP->url, 501); + orionldState.noLinkHeader = true; // We don't want the Link header for non-implemented requests + return false; +} diff --git a/src/lib/orionld/serviceRoutines/orionldDeleteTemporalEntity.h b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalEntity.h new file mode 100644 index 0000000000..d56bb7dd45 --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldDeleteTemporalEntity.h @@ -0,0 +1,37 @@ +#ifndef SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDDELETETEMPORALENTITY_H_ +#define SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDDELETETEMPORALENTITY_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ + + + +// ---------------------------------------------------------------------------- +// +// orionldDeleteTemporalEntity - +// +extern bool orionldDeleteTemporalEntity(void); + +#endif // SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDDELETETEMPORALENTITY_H_ diff --git a/src/lib/orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.cpp b/src/lib/orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.cpp new file mode 100644 index 0000000000..66a2b710d7 --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.cpp @@ -0,0 +1,43 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "logMsg/logMsg.h" + +#include "orionld/common/orionldState.h" // orionldState +#include "orionld/common/orionldError.h" // orionldError +#include "orionld/rest/OrionLdRestService.h" // OrionLdRestService +#include "orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.h" // Own Interface + + + +// ---------------------------------------------------------------------------- +// +// orionldPatchTemporalAttributeInstance - +// +bool orionldPatchTemporalAttributeInstance(void) +{ + orionldError(OrionldOperationNotSupported, "Not Implemented", orionldState.serviceP->url, 501); + orionldState.noLinkHeader = true; // We don't want the Link header for non-implemented requests + return false; +} diff --git a/src/lib/orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.h b/src/lib/orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.h new file mode 100644 index 0000000000..3d1301b102 --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldPatchTemporalAttributeInstance.h @@ -0,0 +1,37 @@ +#ifndef SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDPATCHTEMPORALATTRIBUTEINSTANCE_H_ +#define SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDPATCHTEMPORALATTRIBUTEINSTANCE_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ + + + +// ---------------------------------------------------------------------------- +// +// orionldPatchTemporalAttributeInstance - +// +extern bool orionldPatchTemporalAttributeInstance(void); + +#endif // SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDPATCHTEMPORALATTRIBUTEINSTANCE_H_ diff --git a/src/lib/orionld/serviceRoutines/orionldPostTemporalAttributes.cpp b/src/lib/orionld/serviceRoutines/orionldPostTemporalAttributes.cpp new file mode 100644 index 0000000000..91f536112e --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldPostTemporalAttributes.cpp @@ -0,0 +1,43 @@ +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ +#include "logMsg/logMsg.h" + +#include "orionld/common/orionldState.h" // orionldState +#include "orionld/common/orionldError.h" // orionldError +#include "orionld/rest/OrionLdRestService.h" // OrionLdRestService +#include "orionld/serviceRoutines/orionldPostTemporalAttributes.h" // Own Interface + + + +// ---------------------------------------------------------------------------- +// +// orionldPostTemporalAttributes - +// +bool orionldPostTemporalAttributes(void) +{ + orionldError(OrionldOperationNotSupported, "Not Implemented", orionldState.serviceP->url, 501); + orionldState.noLinkHeader = true; // We don't want the Link header for non-implemented requests + return false; +} diff --git a/src/lib/orionld/serviceRoutines/orionldPostTemporalAttributes.h b/src/lib/orionld/serviceRoutines/orionldPostTemporalAttributes.h new file mode 100644 index 0000000000..e2a07441a0 --- /dev/null +++ b/src/lib/orionld/serviceRoutines/orionldPostTemporalAttributes.h @@ -0,0 +1,37 @@ +#ifndef SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDPOSTTEMPORALATTRIBUTES_H_ +#define SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDPOSTTEMPORALATTRIBUTES_H_ + +/* +* +* Copyright 2023 FIWARE Foundation e.V. +* +* This file is part of Orion-LD Context Broker. +* +* Orion-LD Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion-LD Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* orionld at fiware dot org +* +* Author: Ken Zangelin +*/ + + + +// ---------------------------------------------------------------------------- +// +// orionldPostTemporalAttributes - +// +extern bool orionldPostTemporalAttributes(void); + +#endif // SRC_LIB_ORIONLD_SERVICEROUTINES_ORIONLDPOSTTEMPORALATTRIBUTES_H_ diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_temporal_representation_501.test b/test/functionalTest/cases/0000_ngsild/ngsild_temporal_representation_501.test index 0554480e55..f758948b58 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_temporal_representation_501.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_temporal_representation_501.test @@ -35,27 +35,40 @@ orionldStart CB # Requests that still give 501: # 02. Get 501 for GET /ngsi-ld/v1/temporal/entities # 03. Get 501 for GET /ngsi-ld/v1/temporal/entities/ -# 04. Get 501 for DELETE /ngsi-ld/v1/temporal/entities/ -# 05. Get 501 for POST /ngsi-ld/v1/temporal/entities//attrs -# 06. Get 501 for DELETE /ngsi-ld/v1/temporal/entities//attrs/{attrId} -# 07. Get 501 for POST /ngsi-ld/v1/temporal/entityOperations/query +# 04. Get 501 for POST /ngsi-ld/v1/temporal/entityOperations/query +# +# 05. Get 501 for DELETE /ngsi-ld/v1/temporal/entities/ +# 06. Get 501 for POST /ngsi-ld/v1/temporal/entities//attrs +# 07. Get 501 for DELETE /ngsi-ld/v1/temporal/entities//attrs/{attrId} +# 08. Get 501 for DELETE /ngsi-ld/v1/temporal/entities//attrs/{attrId}/{instanceId} +# 09. Get 501 for PATCH /ngsi-ld/v1/temporal/entities//attrs/{attrId}/{instanceId} # echo "02. Get 501 for GET /ngsi-ld/v1/temporal/entities" echo "=================================================" -orionCurl --url /ngsi-ld/v1/temporal/entities +orionCurl --url /ngsi-ld/v1/temporal/entities?badParam=urn:S1 echo echo echo "03. Get 501 for GET /ngsi-ld/v1/temporal/entities/" echo "=======================================================" -orionCurl --url /ngsi-ld/v1/temporal/entities/urn:ngsi-ld:City:Madrid +orionCurl --url /ngsi-ld/v1/temporal/entities/urn:ngsi-ld:City:Madrid?badParam=urn:S1 echo echo -echo "04. Get 501 for DELETE /ngsi-ld/v1/temporal/entities/" +echo "04. Get 501 for POST /ngsi-ld/v1/temporal/entityOperations/query" +echo "================================================================" +payload='{ + "q": "A1>10" +}' +orionCurl --url /ngsi-ld/v1/temporal/entityOperations/query --payload "$payload" +echo +echo + + +echo "05. Get 501 for DELETE /ngsi-ld/v1/temporal/entities/" echo "==========================================================" orionCurl --url /ngsi-ld/v1/temporal/entities/urn:ngsi-ld:City:Madrid -X DELETE echo @@ -63,7 +76,7 @@ echo -echo "05. Get 501 for POST /ngsi-ld/v1/temporal/entities//attrs" +echo "06. Get 501 for POST /ngsi-ld/v1/temporal/entities//attrs" echo "==============================================================" payload='{ "attr1": { @@ -76,19 +89,36 @@ echo echo -echo "06. Get 501 for DELETE /ngsi-ld/v1/temporal/entities//attrs/{attrId}" +echo "07. Get 501 for DELETE /ngsi-ld/v1/temporal/entities//attrs/{attrId}" echo "=========================================================================" orionCurl --url /ngsi-ld/v1/temporal/entities/urn:ngsi-ld:City:Madrid/attrs/attr1 -X DELETE echo echo -echo "07. Get 501 for POST /ngsi-ld/v1/temporal/entityOperations/query" -echo "================================================================" +echo "08. Get 501 for DELETE /ngsi-ld/v1/temporal/entities//attrs/{attrId}/{instanceId}" +echo "======================================================================================" +orionCurl --url /ngsi-ld/v1/temporal/entities/urn:ngsi-ld:City:Madrid/attrs/attr1/instance1 -X DELETE +echo +echo + + +echo "09. Get 501 for PATCH /ngsi-ld/v1/temporal/entities//attrs/{attrId}/{instanceId}" +echo "=====================================================================================" payload='{ - "q": "A1>10" + "value": 1 }' -orionCurl --url /ngsi-ld/v1/temporal/entityOperations/query --payload "$payload" +orionCurl --url /ngsi-ld/v1/temporal/entities/urn:ngsi-ld:City:Madrid/attrs/attr1/instance1 -X PATCH --payload "$payload" +echo +echo + + +echo "10. Get 405 for POST /ngsi-ld/v1/temporal/entities//attrs/{attrId}/{instanceId}" +echo "====================================================================================" +payload='{ + "value": 1 +}' +orionCurl --url /ngsi-ld/v1/temporal/entities/urn:ngsi-ld:City:Madrid/attrs/attr1/instance1 --payload "$payload" echo echo @@ -122,7 +152,21 @@ Date: REGEX(.*) } -04. Get 501 for DELETE /ngsi-ld/v1/temporal/entities/ +04. Get 501 for POST /ngsi-ld/v1/temporal/entityOperations/query +================================================================ +HTTP/1.1 501 Not Implemented +Content-Length: 200 +Content-Type: application/json +Date: REGEX(.*) + +{ + "detail": "/ngsi-ld/v1/temporal/entityOperations/query", + "title": "Not Implemented in Orion-LD, please use Mintaka for this operation", + "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" +} + + +05. Get 501 for DELETE /ngsi-ld/v1/temporal/entities/ ========================================================== HTTP/1.1 501 Not Implemented Content-Length: 137 @@ -136,7 +180,7 @@ Date: REGEX(.*) } -05. Get 501 for POST /ngsi-ld/v1/temporal/entities//attrs +06. Get 501 for POST /ngsi-ld/v1/temporal/entities//attrs ============================================================== HTTP/1.1 501 Not Implemented Content-Length: 143 @@ -150,7 +194,7 @@ Date: REGEX(.*) } -06. Get 501 for DELETE /ngsi-ld/v1/temporal/entities//attrs/{attrId} +07. Get 501 for DELETE /ngsi-ld/v1/temporal/entities//attrs/{attrId} ========================================================================= HTTP/1.1 501 Not Implemented Content-Length: 145 @@ -164,20 +208,43 @@ Date: REGEX(.*) } -07. Get 501 for POST /ngsi-ld/v1/temporal/entityOperations/query -================================================================ +08. Get 501 for DELETE /ngsi-ld/v1/temporal/entities//attrs/{attrId}/{instanceId} +====================================================================================== HTTP/1.1 501 Not Implemented -Content-Length: 200 +Content-Length: 145 Content-Type: application/json Date: REGEX(.*) { - "detail": "/ngsi-ld/v1/temporal/entityOperations/query", - "title": "Not Implemented in Orion-LD, please use Mintaka for this operation", + "detail": "/ngsi-ld/v1/temporal/entities/*/attrs/*", + "title": "Not Implemented", "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" } +09. Get 501 for PATCH /ngsi-ld/v1/temporal/entities//attrs/{attrId}/{instanceId} +===================================================================================== +HTTP/1.1 501 Not Implemented +Content-Length: 147 +Content-Type: application/json +Date: REGEX(.*) + +{ + "detail": "/ngsi-ld/v1/temporal/entities/*/attrs/*/*", + "title": "Not Implemented", + "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" +} + + +10. Get 405 for POST /ngsi-ld/v1/temporal/entities//attrs/{attrId}/{instanceId} +==================================================================================== +HTTP/1.1 405 Method Not Allowed +Allow: GET,PATCH,DELETE +Content-Length: 0 +Date: REGEX(.*) + + + --TEARDOWN-- brokerStop CB dbDrop CB From 6ee94255ed9ee73f1e1a1d2b40195e7bde45bee0 Mon Sep 17 00:00:00 2001 From: Tim Smyth Date: Tue, 5 Sep 2023 08:05:28 +0200 Subject: [PATCH 14/29] Remove outdated and deprecated repository --- docker/Dockerfile-test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile-test b/docker/Dockerfile-test index c0db1ce2bb..695442baf9 100644 --- a/docker/Dockerfile-test +++ b/docker/Dockerfile-test @@ -14,6 +14,8 @@ WORKDIR lcov-1.14 RUN make install WORKDIR ${PATH_TO_SRC} +RUN yum-config-manager --disable pgdg10 + RUN yum install -y gnupg RUN yum install -y https://repo.mongodb.org/yum/redhat/8Server/mongodb-org/4.4/x86_64/RPMS/mongodb-database-tools-100.4.1.x86_64.rpm From 8f084dd3804e02f56d580ee71022c0e9d8158c64 Mon Sep 17 00:00:00 2001 From: Tim Smyth Date: Tue, 5 Sep 2023 10:55:11 +0200 Subject: [PATCH 15/29] lib update --- docker/build-ubi/install-gnutls.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build-ubi/install-gnutls.sh b/docker/build-ubi/install-gnutls.sh index 6023353aa6..9b65d612df 100755 --- a/docker/build-ubi/install-gnutls.sh +++ b/docker/build-ubi/install-gnutls.sh @@ -35,8 +35,8 @@ yum -y install https://rpmfind.net/linux/centos/8-stream/BaseOS/x86_64/os/Packag yum -y install https://rpmfind.net/linux/centos/8-stream/AppStream/x86_64/os/Packages/nettle-devel-3.4.1-7.el8.x86_64.rpm yum -y install https://rpmfind.net/linux/centos/8-stream/AppStream/x86_64/os/Packages/libidn2-devel-2.2.0-1.el8.x86_64.rpm yum -y install https://rpmfind.net/linux/centos/8-stream/AppStream/x86_64/os/Packages/libtasn1-devel-4.13-3.el8.x86_64.rpm -yum -y install https://dl.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/Packages/g/gnutls-c++-3.6.16-5.el8_6.x86_64.rpm -yum -y install https://dl.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/Packages/g/gnutls-devel-3.6.16-5.el8_6.x86_64.rpm +yum -y install https://dl.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/Packages/g/gnutls-c++-3.6.16-6.el8_7.x86_64.rpm +yum -y install https://dl.rockylinux.org/pub/rocky/8/AppStream/x86_64/os/Packages/g/gnutls-devel-3.6.16-6.el8_7.x86_64.rpm #yum -y install http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/gmp-devel-6.1.2-10.el8.x86_64.rpm #yum -y install http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/pkgconf-pkg-config-1.4.2-1.el8.x86_64.rpm From 51d27f32bbdaaa153e46700c6e822543c1905b29 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Tue, 5 Sep 2023 10:59:53 +0200 Subject: [PATCH 16/29] Fixed another bson array index problem --- src/lib/orionld/mongoc/mongocEntityUpdate.cpp | 11 ++ src/lib/orionld/mongoc/mongocKjTreeToBson.cpp | 9 +- ..._PATCH_Entity-with-Relationship-array.test | 145 ++++++++++++++++++ 3 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_PATCH_Entity-with-Relationship-array.test diff --git a/src/lib/orionld/mongoc/mongocEntityUpdate.cpp b/src/lib/orionld/mongoc/mongocEntityUpdate.cpp index 8c2f0f6388..5a5696af53 100644 --- a/src/lib/orionld/mongoc/mongocEntityUpdate.cpp +++ b/src/lib/orionld/mongoc/mongocEntityUpdate.cpp @@ -36,6 +36,7 @@ extern "C" #include "orionld/common/orionldState.h" // orionldState #include "orionld/common/dotForEq.h" // dotForEq +#include "orionld/mongoc/mongocWriteLog.h" // MONGOC_WLOG #include "orionld/mongoc/mongocConnectionGet.h" // mongocConnectionGet #include "orionld/mongoc/mongocKjTreeToBson.h" // mongocKjTreeToBson #include "orionld/mongoc/mongocIndexString.h" // mongocIndexString @@ -100,6 +101,7 @@ static void mongocKjTreeToUpdateBson if (isValue) { bson_t compound; + LM_T(LmtMongoc, ("Member '%s' has a value that is a compound object - calling mongocKjTreeToBson", subAttrP->name)); mongocKjTreeToBson(subAttrP, &compound); bson_append_document(setP, path, pathLen, &compound); } @@ -108,6 +110,7 @@ static void mongocKjTreeToUpdateBson int bsonPathLen = strlen(bsonPath) + 1 + strlen(attrP->name) + 1; char* bsonPath2 = kaAlloc(&orionldState.kalloc, bsonPathLen); + LM_T(LmtMongoc, ("Member '%s' has a value that is a simple value - calling mongocKjTreeToUpdateBson", subAttrP->name)); snprintf(bsonPath2, bsonPathLen, "%s.%s", bsonPath, attrP->name); mongocKjTreeToUpdateBson(subAttrP, setP, unsetP, bsonPath2, NULL, NULL, NULL, level + 1); // Perhaps another function for sub-attributes ...? } @@ -118,9 +121,12 @@ static void mongocKjTreeToUpdateBson { bson_t compound; + LM_T(LmtMongoc, ("Array member '%s' is a compound object - calling mongocKjTreeToBson", subAttrP->name)); mongocKjTreeToBson(subAttrP, &compound); bson_append_array(setP, path, pathLen, &compound); } + else + LM_T(LmtMongoc, ("Wait, what???")); } ++jx; @@ -210,6 +216,7 @@ static bool patchApply char buf[16]; int bufLen = mongocIndexString(ix, buf); + LM_T(LmtMongoc, ("Array Index: '%s'", buf)); bson_append_utf8(&commaArrayBson, buf, bufLen, itemNameP->value.s, -1); ++ix; } @@ -235,12 +242,15 @@ static bool patchApply else if (tree->type == KjBoolean) bson_append_bool(setP, path, -1, tree->value.b); else if (tree->type == KjArray) { + LM_T(LmtMongoc, ("'%s' is an Array - calling mongocKjTreeToBson", tree->name)); + kjTreeLog(tree, "TREE", LmtMongoc); mongocKjTreeToBson(tree, &compound); bson_append_array(setP, path, -1, &compound); bson_destroy(&compound); } else if (tree->type == KjObject) { + LM_T(LmtMongoc, ("'%s' is an Object - calling mongocKjTreeToBson", tree->name)); mongocKjTreeToBson(tree, &compound); bson_append_document(setP, path, -1, &compound); bson_destroy(&compound); @@ -294,6 +304,7 @@ bool mongocEntityUpdate(const char* entityId, KjNode* patchTree) if (pulls > 0) bson_append_document(&request, "$pull", 5, &pull); if (pushes > 0) bson_append_document(&request, "$push", 5, &push); + MONGOC_WLOG("PATCH Entity", orionldState.tenantP->mongoDbName, "entities", &selector, &request, LmtMongoc); bool b = mongoc_collection_update_one(orionldState.mongoc.entitiesP, &selector, &request, NULL, &reply, &orionldState.mongoc.error); if (b == false) { diff --git a/src/lib/orionld/mongoc/mongocKjTreeToBson.cpp b/src/lib/orionld/mongoc/mongocKjTreeToBson.cpp index 361142aea2..d3b2837dca 100644 --- a/src/lib/orionld/mongoc/mongocKjTreeToBson.cpp +++ b/src/lib/orionld/mongoc/mongocKjTreeToBson.cpp @@ -54,11 +54,13 @@ static void kjTreeToBson(KjNode* nodeP, bson_t* parentP, bool inArray, int array { slen = snprintf(nameBuf, sizeof(nameBuf), "%d", arrayIndex); name = nameBuf; + LM_T(LmtMongoc, ("Array Index: '%s'", name)); } else { name = nodeP->name; slen = strlen(name); + LM_T(LmtMongoc, ("Name: '%s'", name)); } if (nodeP->type == KjString) bson_append_utf8(parentP, name, slen, nodeP->value.s, -1); @@ -75,6 +77,7 @@ static void kjTreeToBson(KjNode* nodeP, bson_t* parentP, bool inArray, int array for (KjNode* itemP = nodeP->value.firstChildP; itemP != NULL; itemP = itemP->next) { + LM_T(LmtMongoc, ("Calling kjTreeToBson for item '%s' of container '%s'", itemP->name, nodeP->name)); kjTreeToBson(itemP, &bObject, false); } bson_append_document_end(parentP, &bObject); @@ -88,6 +91,7 @@ static void kjTreeToBson(KjNode* nodeP, bson_t* parentP, bool inArray, int array bson_append_array_begin(parentP, name, slen, &bArray); for (KjNode* itemP = nodeP->value.firstChildP; itemP != NULL; itemP = itemP->next) { + LM_T(LmtMongoc, ("Calling kjTreeToBson for array item %d of '%s'", arrayIndex, nodeP->name)); kjTreeToBson(itemP, &bArray, true, arrayIndex); arrayIndex += 1; } @@ -108,8 +112,11 @@ void mongocKjTreeToBson(KjNode* treeP, bson_t* bsonP) bson_init(bsonP); + int arrayIx = 0; for (KjNode* itemP = treeP->value.firstChildP; itemP != NULL; itemP = itemP->next) { - kjTreeToBson(itemP, bsonP, inArray); + LM_T(LmtMongoc, ("Calling kjTreeToBson for '%s' of '%s'", itemP->name, treeP->name)); + kjTreeToBson(itemP, bsonP, inArray, arrayIx); + ++arrayIx; } } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_PATCH_Entity-with-Relationship-array.test b/test/functionalTest/cases/0000_ngsild/ngsild_PATCH_Entity-with-Relationship-array.test new file mode 100644 index 0000000000..55e6a29ac3 --- /dev/null +++ b/test/functionalTest/cases/0000_ngsild/ngsild_PATCH_Entity-with-Relationship-array.test @@ -0,0 +1,145 @@ +# Copyright 2023 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Entity Patch using Smart Input Format + +--SHELL-INIT-- +dbInit CB +orionldStart CB -experimental + +--SHELL-- + +# +# 01. Create an entity 'urn:E1' with a Relationship 'occupier' whose object is an array of one item +# 02. GET 'urn:E1' +# 03. PATCH entity 'urn:E1', making 'occupier' have an array of two +# 04. GET the entity and see that occupier::object is an array of two items +# + + +echo "01. Create an entity 'urn:E1' with a Relationship 'occupier' whose object is an array of one item" +echo "=================================================================================================" +payload='{ + "id": "urn:E1", + "type": "T", + "occupier": { + "object": [ + "urn:ngsi-ld:Person:You" + ] + } +}' +orionCurl --url /ngsi-ld/v1/entities --payload "$payload" +echo +echo + + +echo "02. GET 'urn:E1'" +echo "================" +orionCurl --url /ngsi-ld/v1/entities/urn:E1 +echo +echo + + +echo "03. PATCH entity 'urn:E1', making 'occupier' have an array of two" +echo "=================================================================" +payload='{ + "occupier": { + "object": [ + "urn:ngsi-ld:Person:He", + "urn:ngsi-ld:Person:She" + ] + } +}' +orionCurl --url /ngsi-ld/v1/entities/urn:E1 -X PATCH --payload "$payload" +echo +echo + + +echo "04. GET the entity and see that occupier::object is an array of two items" +echo "=========================================================================" +orionCurl --url /ngsi-ld/v1/entities/urn:E1 +echo +echo + + +--REGEXPECT-- +01. Create an entity 'urn:E1' with a Relationship 'occupier' whose object is an array of one item +================================================================================================= +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/entities/urn:E1 + + + +02. GET 'urn:E1' +================ +HTTP/1.1 200 OK +Content-Length: 97 +Content-Type: application/json +Date: REGEX(.*) +Link: Date: Tue, 5 Sep 2023 11:35:47 +0200 Subject: [PATCH 17/29] no https for centos mirrors --- docker/build-ubi/install-paho-client.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build-ubi/install-paho-client.sh b/docker/build-ubi/install-paho-client.sh index 8e03ff1314..949b6ca90d 100755 --- a/docker/build-ubi/install-paho-client.sh +++ b/docker/build-ubi/install-paho-client.sh @@ -25,7 +25,7 @@ set -e echo echo -e "\e[1;32m Debian Builder: installing Paho MQTT C library \e[0m" -wget https://mirror.centos.org/centos/7/os/x86_64/Packages/doxygen-1.8.5-4.el7.x86_64.rpm +wget http://mirror.centos.org/centos/7/os/x86_64/Packages/doxygen-1.8.5-4.el7.x86_64.rpm yum install -y doxygen-1.8.5-4.el7.x86_64.rpm From 95284ab70c4bef6478035b50af8c3628a7636486 Mon Sep 17 00:00:00 2001 From: Tim Smyth Date: Tue, 5 Sep 2023 12:58:39 +0200 Subject: [PATCH 18/29] Retrieve old version --- docker/build-ubi/install-postgres-client.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build-ubi/install-postgres-client.sh b/docker/build-ubi/install-postgres-client.sh index 0380cd8243..d98d813eda 100755 --- a/docker/build-ubi/install-postgres-client.sh +++ b/docker/build-ubi/install-postgres-client.sh @@ -34,7 +34,7 @@ yum -y install https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Pa yum -y install https://rpmfind.net/linux/centos/8-stream/AppStream/x86_64/os/Packages/blas-3.8.0-8.el8.x86_64.rpm yum -y install https://rpmfind.net/linux/centos/8-stream/AppStream/x86_64/os/Packages/lapack-3.8.0-8.el8.x86_64.rpm #yum -y install https://rpmfind.net/linux/epel/8/Everything/x86_64/Packages/a/armadillo-10.8.2-1.el8.x86_64.rpm -yum -y install http://yum.stanford.edu/mrepo/epel-EL8-x86_64/RPMS.all/armadillo-9.700.2-1.el8.x86_64.rpm +yum -y install https://archives.fedoraproject.org/pub/archive/epel/8.1/Everything/x86_64/Packages/a/armadillo-9.700.2-1.el8.x86_64.rpm echo "Add repos" yum update -y --nogpgcheck From 80c882576431ef3309d5897423a9d2bca6c5aae9 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Tue, 5 Sep 2023 15:47:17 +0200 Subject: [PATCH 19/29] Array indox only if inside array --- src/lib/orionld/mongoc/mongocKjTreeToBson.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/orionld/mongoc/mongocKjTreeToBson.cpp b/src/lib/orionld/mongoc/mongocKjTreeToBson.cpp index d3b2837dca..bfe1f7f71f 100644 --- a/src/lib/orionld/mongoc/mongocKjTreeToBson.cpp +++ b/src/lib/orionld/mongoc/mongocKjTreeToBson.cpp @@ -117,6 +117,8 @@ void mongocKjTreeToBson(KjNode* treeP, bson_t* bsonP) { LM_T(LmtMongoc, ("Calling kjTreeToBson for '%s' of '%s'", itemP->name, treeP->name)); kjTreeToBson(itemP, bsonP, inArray, arrayIx); - ++arrayIx; + + if (inArray == true) + ++arrayIx; } } From 6b1ec268e068d6552f0ae90409e2cc6c337d85cf Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Tue, 5 Sep 2023 17:49:43 +0200 Subject: [PATCH 20/29] Highly experimental feature for subscriptions: allowing a wildcard as value for entity type, to not filter on entity type --- CHANGES_NEXT_RELEASE | 1 + .../notifications/subCacheAlterationMatch.cpp | 8 +- .../orionld/payloadCheck/pcheckEntityInfo.cpp | 9 +- .../ngsild_subscription-on-type-star.test | 134 ++++++++++++++++++ 4 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_subscription-on-type-star.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 6fb469aee3..9058bc6e9d 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -6,3 +6,4 @@ Fixed issues: * #280 - Giving errors for expiresAt in the past (for registrations and subscriptions) * #280 - Fixed a bug in error detection of downloading errors in arrays of contexts, at lower nesting levels * #280 - Better 501 handling of temporal operations + * #280 - Highly experimental feature for subscriptions: allowing a wildcard as value for entity type, to not filter on entity type diff --git a/src/lib/orionld/notifications/subCacheAlterationMatch.cpp b/src/lib/orionld/notifications/subCacheAlterationMatch.cpp index 2677ccab6f..152dcf1617 100644 --- a/src/lib/orionld/notifications/subCacheAlterationMatch.cpp +++ b/src/lib/orionld/notifications/subCacheAlterationMatch.cpp @@ -91,9 +91,13 @@ static bool entityTypeMatch(CachedSubscription* subP, const char* entityType, in { for (int ix = 0; ix < eItems; ++ix) { - EntityInfo* eiP = subP->entityIdInfos[ix]; + EntityInfo* eiP = subP->entityIdInfos[ix]; + const char* eType = eiP->entityType.c_str(); + + if (strcmp(entityType, eType) == 0) + return true; - if (strcmp(entityType, eiP->entityType.c_str()) == 0) + if ((eType[0] == '*') && (eType[1] == 0)) return true; } diff --git a/src/lib/orionld/payloadCheck/pcheckEntityInfo.cpp b/src/lib/orionld/payloadCheck/pcheckEntityInfo.cpp index f2345651e0..4276b7a372 100644 --- a/src/lib/orionld/payloadCheck/pcheckEntityInfo.cpp +++ b/src/lib/orionld/payloadCheck/pcheckEntityInfo.cpp @@ -115,8 +115,13 @@ bool pcheckEntityInfo(KjNode* entityInfoP, bool typeMandatory, bool idMandatory, // Expand, unless already expanded // If a ':' is found inside the first 10 chars, the value is assumed to be expanded ... // - if (orionldContextItemAlreadyExpanded(entityItemP->value.s) == false) - entityItemP->value.s = orionldContextItemExpand(orionldState.contextP, entityItemP->value.s, true, NULL); // entity type + // No expansion if the type is '*' - meaning "ALL ENTITY TYPES" + // + if ((entityItemP->value.s[0] != '*') || (entityItemP->value.s[1] != 0)) + { + if (orionldContextItemAlreadyExpanded(entityItemP->value.s) == false) + entityItemP->value.s = orionldContextItemExpand(orionldState.contextP, entityItemP->value.s, true, NULL); // entity type + } } else { diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription-on-type-star.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription-on-type-star.test new file mode 100644 index 0000000000..5b33bb7385 --- /dev/null +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription-on-type-star.test @@ -0,0 +1,134 @@ +# Copyright 2023 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Subscription with type: "*", and test of notifications + +--SHELL-INIT-- +dbInit CB +orionldStart CB -experimental +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create a subscription on EVERYTHING +# 02. Create an entity, any +# 03. See the notification in the accumulator +# + + +echo "01. Create a subscription on EVERYTHING" +echo "=======================================" +payload='{ + "id": "urn:S1", + "description": "Notify ALL", + "type": "Subscription", + "entities": [ + { + "type": "*" + } + ], + "notification": { + "endpoint": { + "uri": "http://localhost:'$LISTENER_PORT'/notify" + } + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create an entity, any" +echo "=========================" +payload='{ + "id": "urn:ngsi-ld:Blower:bd01", + "type": "BlowerDevice", + "airflow": 127 +}' +orionCurl --url /ngsi-ld/v1/entities --payload "$payload" +echo +echo + + +echo "03. See the notification in the accumulator" +echo "===========================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create a subscription on EVERYTHING +======================================= +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:S1 + + + +02. Create an entity, any +========================= +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/entities/urn:ngsi-ld:Blower:bd01 + + + +03. See the notification in the accumulator +=========================================== +POST http://REGEX(.*)/notify?subscriptionId=urn:S1 +Content-Length: 264 +User-Agent: REGEX(.*) +Host: REGEX(.*) +Accept: application/json +Content-Type: application/json +Link: Date: Wed, 6 Sep 2023 17:42:55 +0200 Subject: [PATCH 21/29] GET Subscriptions now include pernot subscriptions :) --- .../dbModel/dbModelToApiSubscription.cpp | 2 +- .../orionldGetSubscriptions.cpp | 122 ++++++--- ...d_get_pernot_and_normal_subscriptions.test | 259 ++++++++++++++++++ 3 files changed, 351 insertions(+), 32 deletions(-) create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_get_pernot_and_normal_subscriptions.test diff --git a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp index 50557e73d2..2443f2b0f7 100644 --- a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp +++ b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp @@ -240,7 +240,7 @@ KjNode* dbModelToApiSubscription // sysAttrs if ((dbSysAttrsP != NULL) && (sysAttrsP != NULL)) - *sysAttrsP = dbShowChangesP; + *sysAttrsP = dbSysAttrsP; // // If dbSubIdP is a JSON Object, it's an NGSIv2 subscription and its "id" looks like this: diff --git a/src/lib/orionld/serviceRoutines/orionldGetSubscriptions.cpp b/src/lib/orionld/serviceRoutines/orionldGetSubscriptions.cpp index 8534a5a7d7..4d91d92bd5 100644 --- a/src/lib/orionld/serviceRoutines/orionldGetSubscriptions.cpp +++ b/src/lib/orionld/serviceRoutines/orionldGetSubscriptions.cpp @@ -36,6 +36,7 @@ extern "C" #include "orionld/common/orionldError.h" // orionldError #include "orionld/types/OrionldHeader.h" // orionldHeaderAdd, HttpResultsCount #include "orionld/common/tenantList.h" // tenant0 +#include "orionld/kjTree/kjTreeFromPernotSubscription.h" // kjTreeFromPernotSubscription #include "orionld/legacyDriver/legacyGetSubscriptions.h" // legacyGetSubscriptions #include "orionld/kjTree/kjTreeFromCachedSubscription.h" // kjTreeFromCachedSubscription #include "orionld/mongoc/mongocSubscriptionsGet.h" // mongocSubscriptionsGet @@ -150,6 +151,43 @@ static bool orionldGetSubscriptionsFromDb(void) +// ---------------------------------------------------------------------------- +// +// subCacheCount - move to subCache.cpp +// +int subCacheCount(void) +{ + int count = 0; + for (CachedSubscription* cSubP = subCacheHeadGet(); cSubP != NULL; cSubP = cSubP->next) + { + if (tenantMatch(orionldState.tenantP, cSubP->tenant) == true) + ++count; + } + + return count; +} + + + +// ---------------------------------------------------------------------------- +// +// pernotSubCacheCount - move to pernotSubCacheCount.cpp +// +int pernotSubCacheCount(void) +{ + int count = 0; + + for (PernotSubscription* pSubP = pernotSubCache.head; pSubP != NULL; pSubP = pSubP->next) + { + if (pSubP->tenantP == orionldState.tenantP) + ++count; + } + + return count; +} + + + // ---------------------------------------------------------------------------- // // orionldGetSubscriptions - @@ -168,50 +206,72 @@ bool orionldGetSubscriptions(void) int offset = orionldState.uriParams.offset; int limit = orionldState.uriParams.limit; + int subs = 0; KjNode* subArray = kjArray(orionldState.kjsonP, NULL); int ix = 0; - int subs = 0; + int cSubs = subCacheCount(); + int pSubs = pernotSubCacheCount(); - if (orionldState.uriParams.count == true) // Empty loop over the subscriptions, just to count how many there are - { - int count = 0; - for (CachedSubscription* cSubP = subCacheHeadGet(); cSubP != NULL; cSubP = cSubP->next) - { - if (tenantMatch(orionldState.tenantP, cSubP->tenant) == true) - ++count; - } - - orionldHeaderAdd(&orionldState.out.headers, HttpResultsCount, NULL, count); - } + if (orionldState.uriParams.count == true) + orionldHeaderAdd(&orionldState.out.headers, HttpResultsCount, NULL, cSubs + pSubs); if (limit != 0) { - for (CachedSubscription* cSubP = subCacheHeadGet(); cSubP != NULL; cSubP = cSubP->next) + // Start with Pernot Subscriptions + if (offset < pSubs) { - if (tenantMatch(orionldState.tenantP, cSubP->tenant) == false) - continue; - - if (ix < offset) + for (PernotSubscription* pSubP = pernotSubCache.head; pSubP != NULL; pSubP = pSubP->next) { - ++ix; - continue; + if (pSubP->tenantP == orionldState.tenantP) + { + KjNode* subP = kjTreeFromPernotSubscription(pSubP, orionldState.uriParamOptions.sysAttrs, orionldState.out.contentType == JSONLD); + + if (subP == NULL) + { + LM_E(("Internal Error (kjTreeFromPernotSubscription failed for subscription '%s')", pSubP->subscriptionId)); + ++ix; + continue; + } + + kjChildAdd(subArray, subP); + ++ix; + ++subs; + + if (subs >= limit) + break; + } } + } - KjNode* subP = kjTreeFromCachedSubscription(cSubP, orionldState.uriParamOptions.sysAttrs, orionldState.out.contentType == JSONLD); - - if (subP == NULL) + if (subs < limit) + { + for (CachedSubscription* cSubP = subCacheHeadGet(); cSubP != NULL; cSubP = cSubP->next) { - LM_E(("Internal Error (kjTreeFromCachedSubscription failed for subscription '%s')", cSubP->subscriptionId)); - ++ix; - continue; - } + if (tenantMatch(orionldState.tenantP, cSubP->tenant) == false) + continue; - kjChildAdd(subArray, subP); - ++ix; - ++subs; + if (ix < offset) + { + ++ix; + continue; + } - if (subs >= limit) - break; + KjNode* subP = kjTreeFromCachedSubscription(cSubP, orionldState.uriParamOptions.sysAttrs, orionldState.out.contentType == JSONLD); + + if (subP == NULL) + { + LM_E(("Internal Error (kjTreeFromCachedSubscription failed for subscription '%s')", cSubP->subscriptionId)); + ++ix; + continue; + } + + kjChildAdd(subArray, subP); + ++ix; + ++subs; + + if (subs >= limit) + break; + } } } else diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_get_pernot_and_normal_subscriptions.test b/test/functionalTest/cases/0000_ngsild/ngsild_get_pernot_and_normal_subscriptions.test new file mode 100644 index 0000000000..bba13cf0d0 --- /dev/null +++ b/test/functionalTest/cases/0000_ngsild/ngsild_get_pernot_and_normal_subscriptions.test @@ -0,0 +1,259 @@ +# Copyright 2023 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Pernot Subscription - periodic notifications + +--SHELL-INIT-- +dbInit CB +orionldStart CB -pernot -experimental +accumulatorStart --pretty-print 127.0.0.1 ${LISTENER_PORT} + +--SHELL-- + +# +# 01. Create a pernot subscription urn:S1 +# 02. Create a "normal" subscription urn:S2 +# 03. GET subs (from cache) +# 04. GET subs (from DB) +# + +echo "01. Create a pernot subscription urn:S1" +echo "=======================================" +payload='{ + "id": "urn:S1", + "type": "Subscription", + "entities": [ + { + "type": "T" + } + ], + "q": "P3==1|P3==2", + "timeInterval": 2, + "notification": { + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + }, + "attributes": [ "P1", "P2" ], + "sysAttrs": false, + "format": "normalized" + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo '02. Create a "normal" subscription urn:S2' +echo '=========================================' +payload='{ + "id": "urn:S2", + "type": "Subscription", + "entities": [ + { + "type": "T" + } + ], + "q": "P3==1|P3==2", + "notification": { + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + }, + "attributes": [ "P1", "P2" ], + "sysAttrs": false, + "format": "normalized" + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo '03. GET subs (from cache)' +echo '=========================' +orionCurl --url /ngsi-ld/v1/subscriptions +echo +echo + + +echo '04. GET subs (from DB)' +echo '======================' +orionCurl --url /ngsi-ld/v1/subscriptions?options=fromDb +echo +echo + + +--REGEXPECT-- +01. Create a pernot subscription urn:S1 +======================================= +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:S1 + + + +02. Create a "normal" subscription urn:S2 +========================================= +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:S2 + + + +03. GET subs (from cache) +========================= +HTTP/1.1 200 OK +Content-Length: 839 +Content-Type: application/json +Date: REGEX(.*) +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + +[ + { + "entities": [ + { + "type": "T" + } + ], + "id": "urn:S1", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "lastNotification": "202REGEX(.*)Z", + "status": "ok", + "sysAttrs": false, + "timesSent": 1 + }, + "origin": "cache", + "q": "P3==1|P3==2", + "status": "active", + "timeInterval": 2, + "type": "Subscription" + }, + { + "entities": [ + { + "type": "T" + } + ], + "id": "urn:S2", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "status": "ok" + }, + "origin": "cache", + "q": "P3==1|P3==2", + "status": "active", + "type": "Subscription" + } +] + + +04. GET subs (from DB) +====================== +HTTP/1.1 200 OK +Content-Length: 768 +Content-Type: application/json +Date: REGEX(.*) +Link: ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json" + +[ + { + "entities": [ + { + "type": "T" + } + ], + "id": "urn:S1", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "status": "ok" + }, + "origin": "database", + "q": "P3==1|P3==2", + "status": "active", + "timeInterval": 2, + "type": "Subscription" + }, + { + "entities": [ + { + "type": "T" + } + ], + "id": "urn:S2", + "isActive": true, + "jsonldContext": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", + "notification": { + "attributes": [ + "P1", + "P2" + ], + "endpoint": { + "accept": "application/json", + "uri": "http://127.0.0.1:9997/notify" + }, + "format": "normalized", + "status": "ok" + }, + "origin": "database", + "q": "P3==1|P3==2", + "status": "active", + "type": "Subscription" + } +] + + +--TEARDOWN-- +brokerStop CB +dbDrop CB From 2a681cc10fcbd5f3f10967c40f0f34097e19d791 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Fri, 8 Sep 2023 10:38:20 +0200 Subject: [PATCH 22/29] Subs with timeInterval: Giving 501 for for non-experimental. And for experimental when geoQ is present --- .../dbModel/dbModelToApiSubscription.cpp | 5 -- src/lib/orionld/kjTree/kjTreeLog.cpp | 4 +- .../orionld/kjTree/kjTreeToSubscription.cpp | 11 +-- .../orionld/mongoc/mongocEntitiesQuery.cpp | 5 +- .../orionld/mongoc/mongocEntitiesQuery2.cpp | 2 +- src/lib/orionld/notifications/httpNotify.cpp | 2 +- src/lib/orionld/notifications/httpNotify.h | 2 +- src/lib/orionld/payloadCheck/pCheckQuery.cpp | 2 +- .../payloadCheck/pCheckSubscription.cpp | 8 ++- src/lib/orionld/payloadCheck/pcheckGeoQ.cpp | 5 +- src/lib/orionld/payloadCheck/pcheckGeoQ.h | 3 +- src/lib/orionld/payloadCheck/pcheckQuery.cpp | 2 +- .../payloadCheck/pcheckSubscription.cpp | 2 +- src/lib/orionld/pernot/PernotSubscription.h | 4 +- src/lib/orionld/pernot/pernotSubCacheAdd.cpp | 25 ++++++- src/lib/orionld/pernot/pernotTreat.cpp | 4 +- .../orionldGetSubscription.cpp | 2 +- ...ew_subscription-create-error-handling.test | 6 -- .../ngsild_subscription_create.test | 26 +++---- .../ngsild_subscription_creation_errors.test | 72 ------------------- 20 files changed, 66 insertions(+), 126 deletions(-) diff --git a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp index 2443f2b0f7..4a5b31740b 100644 --- a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp +++ b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp @@ -306,10 +306,6 @@ KjNode* dbModelToApiSubscription // There is no "isPattern" in NGSI-LD kjChildRemove(entityP, isPatternP); - LM_T(LmtSR, ("entityId: %s", idP->value.s)); - LM_T(LmtSR, ("isPattern: %s", isPatternP->value.s)); - LM_T(LmtSR, ("type: %s", typeP->value.s)); - if (strcmp(isPatternP->value.s, "true") == 0) { if (strcmp(idP->value.s, ".*") == 0) @@ -539,7 +535,6 @@ KjNode* dbModelToApiSubscription // Add "notification" to top level kjChildAdd(apiSubP, notificationP); - if (dbReferenceP != NULL) { kjChildAdd(endpointP, dbReferenceP); diff --git a/src/lib/orionld/kjTree/kjTreeLog.cpp b/src/lib/orionld/kjTree/kjTreeLog.cpp index d99209833e..ad696b7e96 100644 --- a/src/lib/orionld/kjTree/kjTreeLog.cpp +++ b/src/lib/orionld/kjTree/kjTreeLog.cpp @@ -50,7 +50,9 @@ void kjTreeLogFunction(KjNode* tree, const char* msg, const char* path, int line if (tree == NULL) { - lmOut((char*) "NULL Tree", 'T', fileNameOnly, lineNo, functionName, traceLevel, NULL); + char error[256]; + snprintf(error, sizeof(error) - 1, "%s: NULL", msg); + lmOut(error, 'T', fileNameOnly, lineNo, functionName, traceLevel, NULL); return; } diff --git a/src/lib/orionld/kjTree/kjTreeToSubscription.cpp b/src/lib/orionld/kjTree/kjTreeToSubscription.cpp index 870b951817..7df8a44135 100644 --- a/src/lib/orionld/kjTree/kjTreeToSubscription.cpp +++ b/src/lib/orionld/kjTree/kjTreeToSubscription.cpp @@ -260,15 +260,8 @@ bool kjTreeToSubscription(ngsiv2::Subscription* subP, char** subIdPP, KjNode** e } else if (strcmp(kNodeP->name, "timeInterval") == 0) { - DUPLICATE_CHECK(timeIntervalP, "Subscription::timeInterval", kNodeP); - NUMBER_CHECK(timeIntervalP, "Subscription::timeInterval"); - subP->timeInterval = (timeIntervalP->type == KjInt)? timeIntervalP->value.i : timeIntervalP->value.f; - - if (subP->timeInterval <= 0) - { - orionldError(OrionldBadRequestData, "Invalid value for Subscription::timeInterval", "must be a Number > 0", 400); - return false; - } + orionldError(OrionldOperationNotSupported, "Not Implemented", "Periodic Notification Subscriptions are not implemented", 501); + return false; } else if ((kNodeP->name[0] == 'q') && (kNodeP->name[1] == 0)) { diff --git a/src/lib/orionld/mongoc/mongocEntitiesQuery.cpp b/src/lib/orionld/mongoc/mongocEntitiesQuery.cpp index e3d405c142..867ec76660 100644 --- a/src/lib/orionld/mongoc/mongocEntitiesQuery.cpp +++ b/src/lib/orionld/mongoc/mongocEntitiesQuery.cpp @@ -255,7 +255,9 @@ static bool geoNearFilter(bson_t* mongoFilterP, OrionldGeoInfo* geoInfoP) char geoPropertyPath[512]; int geoPropertyPathLen; if (geoPropertyDbPath(geoPropertyPath, sizeof(geoPropertyPath), geoInfoP->geoProperty, &geoPropertyPathLen) == false) - return false; + LM_RE(false, ("Failed to aeemble the geoProperty path")); + + LM_T(LmtMongoc, ("geoPropertyPath: '%s'", geoPropertyPath)); bson_append_document(mongoFilterP, geoPropertyPath, geoPropertyPathLen, &location); @@ -310,6 +312,7 @@ static bool geoWithinFilter(bson_t* mongoFilterP, OrionldGeoInfo* geoInfoP) bson_init(&within); bson_init(&coordinates); + kjTreeLog(geoInfoP->coordinates, "coordinates", LmtMongoc); mongocKjTreeToBson(geoInfoP->coordinates, &coordinates); bson_append_array(&geometry, "coordinates", 11, &coordinates); diff --git a/src/lib/orionld/mongoc/mongocEntitiesQuery2.cpp b/src/lib/orionld/mongoc/mongocEntitiesQuery2.cpp index c5b9d072fb..93ef62336c 100644 --- a/src/lib/orionld/mongoc/mongocEntitiesQuery2.cpp +++ b/src/lib/orionld/mongoc/mongocEntitiesQuery2.cpp @@ -320,7 +320,7 @@ KjNode* mongocEntitiesQuery2 if (geoInfoP != NULL) { if (geoFilter(&mongoFilter, geoInfoP) == false) - return NULL; + LM_RE(NULL, ("geoFilter flagged an error")); geojsonGeometry = geoInfoP->geoProperty; } diff --git a/src/lib/orionld/notifications/httpNotify.cpp b/src/lib/orionld/notifications/httpNotify.cpp index 0d0e7f48a7..2cc90da274 100644 --- a/src/lib/orionld/notifications/httpNotify.cpp +++ b/src/lib/orionld/notifications/httpNotify.cpp @@ -44,7 +44,7 @@ int httpNotify const char* subscriptionId, const char* ip, unsigned short port, - const char* path, + const char* path, struct iovec* ioVec, int ioVecLen, double notificationTime diff --git a/src/lib/orionld/notifications/httpNotify.h b/src/lib/orionld/notifications/httpNotify.h index 1c7393d073..f5aa0ba51d 100644 --- a/src/lib/orionld/notifications/httpNotify.h +++ b/src/lib/orionld/notifications/httpNotify.h @@ -43,7 +43,7 @@ extern int httpNotify const char* subscriptionId, const char* ip, unsigned short port, - const char* path, + const char* path, struct iovec* ioVec, int ioVecLen, double notificationTime diff --git a/src/lib/orionld/payloadCheck/pCheckQuery.cpp b/src/lib/orionld/payloadCheck/pCheckQuery.cpp index a3fd3ff411..894601cff7 100644 --- a/src/lib/orionld/payloadCheck/pCheckQuery.cpp +++ b/src/lib/orionld/payloadCheck/pCheckQuery.cpp @@ -269,7 +269,7 @@ TreeNode* pCheckQuery(KjNode* queryP) if ((treeNodeV[1].nodeP != NULL) && (pCheckEntities(treeNodeV[1].nodeP) == false)) return NULL; if ((treeNodeV[2].nodeP != NULL) && (pCheckAttrs(treeNodeV[2].nodeP) == false)) return NULL; if ((treeNodeV[3].nodeP != NULL) && ((treeNodeV[3].output = pcheckQ(treeNodeV[3].nodeP->value.s)) == NULL)) return NULL; - if ((treeNodeV[4].nodeP != NULL) && ((treeNodeV[4].output = pcheckGeoQ(treeNodeV[4].nodeP, false)) == NULL)) return NULL; + if ((treeNodeV[4].nodeP != NULL) && ((treeNodeV[4].output = pcheckGeoQ(&orionldState.kalloc, treeNodeV[4].nodeP, false)) == NULL)) return NULL; // // Now, for the query to not be too wide, we need at least one of: diff --git a/src/lib/orionld/payloadCheck/pCheckSubscription.cpp b/src/lib/orionld/payloadCheck/pCheckSubscription.cpp index 8ea76cf32f..ba10286057 100644 --- a/src/lib/orionld/payloadCheck/pCheckSubscription.cpp +++ b/src/lib/orionld/payloadCheck/pCheckSubscription.cpp @@ -237,11 +237,17 @@ bool pCheckSubscription } else if (strcmp(subItemP->name, "geoQ") == 0) { + if (*timeInterval > 0) + { + orionldError(OrionldOperationNotSupported, "Not Implemented", "Geo-Query is not yet implemented for Periodic Notification Subscriptions", 501); + return false; + } + PCHECK_OBJECT(subItemP, 0, NULL, SubscriptionGeoqPath, 400); PCHECK_DUPLICATE(geoqP, subItemP, 0, NULL, SubscriptionGeoqPath, 400); OrionldGeoInfo* geoInfoP; - if ((geoInfoP = pcheckGeoQ(geoqP, true)) == NULL) + if ((geoInfoP = pcheckGeoQ(&orionldState.kalloc, geoqP, true)) == NULL) return false; *geoCoordinatesPP = geoInfoP->coordinates; diff --git a/src/lib/orionld/payloadCheck/pcheckGeoQ.cpp b/src/lib/orionld/payloadCheck/pcheckGeoQ.cpp index adb1ca6886..1db98f6aeb 100644 --- a/src/lib/orionld/payloadCheck/pcheckGeoQ.cpp +++ b/src/lib/orionld/payloadCheck/pcheckGeoQ.cpp @@ -25,6 +25,7 @@ #include // strcmp extern "C" { +#include "kalloc/KAlloc.h" // KAlloc #include "kjson/KjNode.h" // KjNode #include "kjson/kjParse.h" // kjParse } @@ -52,13 +53,13 @@ extern "C" // // pcheckGeoQ - // -OrionldGeoInfo* pcheckGeoQ(KjNode* geoqNodeP, bool isSubscription) +OrionldGeoInfo* pcheckGeoQ(KAlloc* kallocP, KjNode* geoqNodeP, bool isSubscription) { KjNode* geometryP = NULL; KjNode* coordinatesP = NULL; KjNode* georelP = NULL; KjNode* geopropertyP = NULL; - OrionldGeoInfo* geoInfoP = (OrionldGeoInfo*) kaAlloc(&orionldState.kalloc, sizeof(OrionldGeoInfo)); + OrionldGeoInfo* geoInfoP = (OrionldGeoInfo*) ((kallocP != NULL)? kaAlloc(kallocP, sizeof(OrionldGeoInfo)) : malloc(sizeof(OrionldGeoInfo))); if (geoInfoP == NULL) { diff --git a/src/lib/orionld/payloadCheck/pcheckGeoQ.h b/src/lib/orionld/payloadCheck/pcheckGeoQ.h index df282211ef..7f0b3dc89e 100644 --- a/src/lib/orionld/payloadCheck/pcheckGeoQ.h +++ b/src/lib/orionld/payloadCheck/pcheckGeoQ.h @@ -27,6 +27,7 @@ */ extern "C" { +#include "kalloc/KAlloc.h" // KAlloc #include "kjson/KjNode.h" // KjNode } @@ -38,6 +39,6 @@ extern "C" // // pcheckGeoQ - // -extern OrionldGeoInfo* pcheckGeoQ(KjNode* geoqNodeP, bool pCheckGeoGeometry); +extern OrionldGeoInfo* pcheckGeoQ(KAlloc* kallocP, KjNode* geoqNodeP, bool pCheckGeoGeometry); #endif // SRC_LIB_ORIONLD_PAYLOADCHECK_PCHECKGEOQ_H_ diff --git a/src/lib/orionld/payloadCheck/pcheckQuery.cpp b/src/lib/orionld/payloadCheck/pcheckQuery.cpp index 893b42bff8..28e221c60f 100644 --- a/src/lib/orionld/payloadCheck/pcheckQuery.cpp +++ b/src/lib/orionld/payloadCheck/pcheckQuery.cpp @@ -200,7 +200,7 @@ bool pcheckQuery(KjNode* tree, KjNode** entitiesPP, KjNode** attrsPP, QNode** qT return false; if ((qP != NULL) && ((*qTreePP = pcheckQ(qP->value.s)) == NULL)) return false; - if ((geoqP != NULL) && (pcheckGeoQ(geoqP, false) == NULL)) + if ((geoqP != NULL) && (pcheckGeoQ(&orionldState.kalloc, geoqP, false) == NULL)) return false; return true; diff --git a/src/lib/orionld/payloadCheck/pcheckSubscription.cpp b/src/lib/orionld/payloadCheck/pcheckSubscription.cpp index 1197f01879..d34706d293 100644 --- a/src/lib/orionld/payloadCheck/pcheckSubscription.cpp +++ b/src/lib/orionld/payloadCheck/pcheckSubscription.cpp @@ -161,7 +161,7 @@ bool pcheckSubscription DUPLICATE_CHECK(geoqP, "geoQ", nodeP); OBJECT_CHECK(nodeP, "geoQ"); EMPTY_OBJECT_CHECK(nodeP, "geoQ"); - OrionldGeoInfo* geoInfoP = pcheckGeoQ(nodeP, true); + OrionldGeoInfo* geoInfoP = pcheckGeoQ(&orionldState.kalloc, nodeP, true); if (geoInfoP == NULL) return false; *geoCoordinatesPP = geoInfoP->coordinates; diff --git a/src/lib/orionld/pernot/PernotSubscription.h b/src/lib/orionld/pernot/PernotSubscription.h index f862521f94..ee9410db5b 100644 --- a/src/lib/orionld/pernot/PernotSubscription.h +++ b/src/lib/orionld/pernot/PernotSubscription.h @@ -36,9 +36,10 @@ extern "C" #include "common/RenderFormat.h" // RenderFormat #include "common/MimeType.h" // MimeType +#include "orionld/types/OrionldTenant.h" // OrionldTenant #include "orionld/types/Protocol.h" // Protocol #include "orionld/types/StringArray.h" // StringArray -#include "orionld/types/OrionldTenant.h" // OrionldTenant +#include "orionld/types/OrionldGeoInfo.h" // OrionldGeoInfo #include "orionld/q/QNode.h" // QNode @@ -84,6 +85,7 @@ typedef struct PernotSubscription KjNode* eSelector; KjNode* attrsSelector; QNode* qSelector; + OrionldGeoInfo* geoSelector; // Counters uint32_t notificationAttemptsDb; // Total number of notification attempts, in DB diff --git a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp index c8f4077185..f4d799396a 100644 --- a/src/lib/orionld/pernot/pernotSubCacheAdd.cpp +++ b/src/lib/orionld/pernot/pernotSubCacheAdd.cpp @@ -42,6 +42,7 @@ extern "C" #include "orionld/types/Protocol.h" // Protocol, protocolFromString #include "orionld/types/OrionldTenant.h" // OrionldTenant #include "orionld/q/QNode.h" // QNode +#include "orionld/payloadCheck/pcheckGeoQ.h" // pcheckGeoQ #include "orionld/kjTree/kjChildCount.h" // kjChildCount #include "orionld/context/OrionldContext.h" // OrionldContext #include "orionld/pernot/PernotSubscription.h" // PernotSubscription @@ -74,7 +75,7 @@ static void receiverInfo(PernotSubscription* pSubP, KjNode* endpointP) KjNode* valueP = kjLookup(kvPairP, "value"); int sLen = strlen(keyP->value.s) + strlen(valueP->value.s) + 4; char* s = (char*) malloc(sLen); - + snprintf(s, sLen - 1, "%s:%s\r\n", keyP->value.s, valueP->value.s); pSubP->headers.array[ix] = s; ++ix; @@ -173,8 +174,28 @@ PernotSubscription* pernotSubCacheAdd // Query parameters pSubP->eSelector = kjLookup(pSubP->kjSubP, "entities"); pSubP->attrsSelector = kjLookup(notificationP, "attributes"); - pSubP->qSelector = qTree; // I probably need to clone this ... + pSubP->qSelector = qTree; + pSubP->geoSelector = NULL; + KjNode* geoqP = kjLookup(apiSubP, "geoQ"); + if (geoqP != NULL) + { + pSubP->geoSelector = pcheckGeoQ(NULL, geoqP, true); // FIXME: Already done in pcheckSubscription() + if (pSubP->geoSelector != NULL) + { + if (pSubP->geoSelector->geoProperty == NULL) + pSubP->geoSelector->geoProperty = (char*) "location"; + + LM_T(LmtPernotQuery, ("geometry: %d", pSubP->geoSelector->geometry)); + LM_T(LmtPernotQuery, ("georel: %d", pSubP->geoSelector->georel)); + LM_T(LmtPernotQuery, ("minDistance: %d", pSubP->geoSelector->minDistance)); + LM_T(LmtPernotQuery, ("maxDistance: %d", pSubP->geoSelector->maxDistance)); + LM_T(LmtPernotQuery, ("geoProperty: '%s'", pSubP->geoSelector->geoProperty)); + pSubP->geoSelector->coordinates = kjClone(NULL, pSubP->geoSelector->coordinates); + kjTreeLog(pSubP->geoSelector->coordinates, "coordinates", LmtPernotQuery); + + } + } // notification::endpoint if (endpointP == NULL) diff --git a/src/lib/orionld/pernot/pernotTreat.cpp b/src/lib/orionld/pernot/pernotTreat.cpp index 648b3936d8..8fa6f1f962 100644 --- a/src/lib/orionld/pernot/pernotTreat.cpp +++ b/src/lib/orionld/pernot/pernotTreat.cpp @@ -119,8 +119,8 @@ static void* pernotTreat(void* vP) KjNode* apiEntityArray; orionldState.uriParams.offset = 0; - orionldState.uriParams.limit = 20; // Or: set in subscription - orionldState.uriParams.count = true; + orionldState.uriParams.limit = 20; // Or: set in subscription + orionldState.uriParams.count = true; // Need the count to be able to paginate orionldState.tenantP = subP->tenantP; kjTreeLog(subP->eSelector, "eSelector", LmtPernotQuery); diff --git a/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp b/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp index 04de817494..73e7448bf5 100644 --- a/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp +++ b/src/lib/orionld/serviceRoutines/orionldGetSubscription.cpp @@ -184,7 +184,7 @@ static bool orionldGetSubscriptionFromDb(void) return false; } kjTreeLog(dbSubP, "DB Sub", LmtSubCacheStats); - + KjNode* coordinatesNodeP = NULL; // Not needed here, but dbModelToApiSubscription requires it KjNode* contextNodeP = NULL; // Not needed here, but dbModelToApiSubscription requires it KjNode* showChangesP = NULL; // Not needed here, but dbModelToApiSubscription requires it diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-create-error-handling.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-create-error-handling.test index 098cbe1304..14eca64283 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-create-error-handling.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-create-error-handling.test @@ -312,12 +312,6 @@ payload='{ "watchedAttributes": [ "P2" ], "timeInterval": 12, "q": "P2>10", - "geoQ": { - "geometry": "Point", - "coordinates": "[1,2]", - "georel": "near;maxDistance==1000", - "geoproperty": "not supported" - }, "isActive": false, "notification": { "attributes": [ "P1", "P2", "A3" ], diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_create.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_create.test index c03754d36a..7dc4d02615 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_create.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_create.test @@ -47,7 +47,7 @@ orionldStart CB # 04. GET /ngsi-ld/v1/subscription/http://a.b.c/subs/sub01, accepting jsonld - see @context in payload # 05. Attempt to create a subscription with an ID that is not a URI - see failure # 06. Attempt to create a subscription without 'entities' nor 'watchedAttributes' - see failure -# 07. Attempt to create a subscription with both 'watchedAttributes' and 'timeInterval' - see failure +# 07. Attempt to create a subscription with 'timeInterval' - see 501 # 08. Create a subscription without 'timeInterval' nor 'watchedAttributes' - OK # 09. Create a subscription 'sub09' without 'expires' and 'throttling' - OK # 10. GET sub09 with sysAttrs and make sure no 'expires' nor 'throttling' is returned @@ -234,8 +234,8 @@ echo echo -echo "07. Attempt to create a subscription with both 'watchedAttributes' and 'timeInterval' - see failure" -echo "===================================================================================================" +echo "07. Attempt to create a subscription with 'timeInterval' - see 501" +echo "==================================================================" payload='{ "id": "http://a.b.c/subs/sub03", "type": "Subscription", @@ -258,12 +258,6 @@ payload='{ "watchedAttributes": [ "P2" ], "timeInterval": 12, "q": "P2>10", - "geoQ": { - "geometry": "Point", - "coordinates": [1,2], - "georel": "near", - "geoproperty": "location" - }, "csf": "not implemented", "isActive": false, "notification": { @@ -653,17 +647,17 @@ Date: REGEX(.*) } -07. Attempt to create a subscription with both 'watchedAttributes' and 'timeInterval' - see failure -=================================================================================================== -HTTP/1.1 400 Bad Request -Content-Length: 160 +07. Attempt to create a subscription with 'timeInterval' - see 501 +================================================================== +HTTP/1.1 501 Not Implemented +Content-Length: 161 Content-Type: application/json Date: REGEX(.*) { - "detail": "Both 'timeInterval' and 'watchedAttributes' present", - "title": "Inconsistent subscription", - "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" + "detail": "Periodic Notification Subscriptions are not implemented", + "title": "Not Implemented", + "type": "https://uri.etsi.org/ngsi-ld/errors/OperationNotSupported" } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_creation_errors.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_creation_errors.test index 797a4e4378..77daf2af26 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_creation_errors.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_creation_errors.test @@ -96,7 +96,6 @@ orionldStart CB # 18.1. Attempt to create a subscription with an invalid field called 'INVALID' # # *19.1. Attempt to create a subscription without ENTITIES nor WATCHEDATTRIBUTES (duplicate of 06.1) -# *19.2. Attempt to create a subscription with both TIMEINTERVAL and WATCHEDATTRIBUTES present # 19.3. Attempt to create a subscription without NOTIFICATION # *19.4. Creation of a subscription without TIMEINTERVAL nor WATCHEDATTRIBUTES present (entities is present) # @@ -292,63 +291,6 @@ echo echo -echo "19.2. Attempt to create a subscription with both TIMEINTERVAL and WATCHEDATTRIBUTES present" -echo "===========================================================================================" -payload='{ - "id": "http://a.b.c/subs/sub03", - "type": "Subscription", - "name": "Test subscription 01", - "description": "Description of Test subscription 01", - "entities": [ - { - "id": "urn:ngsi-ld:E01", - "type": "T1" - }, - { - "id": "http://a.b.c/E02", - "type": "T2" - }, - { - "idPattern": ".*E03.*", - "type": "T3" - } - ], - "watchedAttributes": [ "P2" ], - "timeInterval": 12, - "q": "P2>10", - "geoQ": { - "geometry": "Point", - "coordinates": [1,2], - "georel": "near;maxDistance==5", - "geoproperty": "location" - }, - "csf": "not implemented", - "isActive": false, - "notification": { - "attributes": [ "P1", "P2", "A3" ], - "format": "keyValues", - "endpoint": { - "uri": "http://valid.url/url", - "accept": "application/ld+json" - }, - "status": "ignored", - "timesSent": "ignored", - "lastNotification": "ignored", - "lastFailure": "ignored", - "lastSuccess": "ignored" - }, - "expires": "2028-12-31T10:00:00", - "throttling": 5, - "status": "to be ignored - read-only", - "@context": "https://fiware.github.io/NGSI-LD_TestSuite/ldContext/testContext.jsonld", - "createdAt": "ignored", - "modifiedAt": "ignored" -}' -orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" -H "Content-Type: application/ld+json" -echo -echo - - echo "19.4. Creation of a subscription without TIMEINTERVAL nor WATCHEDATTRIBUTES present (entities is present)" echo "=========================================================================================================" payload='{ @@ -551,20 +493,6 @@ Date: REGEX(.*) } -19.2. Attempt to create a subscription with both TIMEINTERVAL and WATCHEDATTRIBUTES present -=========================================================================================== -HTTP/1.1 400 Bad Request -Content-Length: 160 -Content-Type: application/json -Date: REGEX(.*) - -{ - "detail": "Both 'timeInterval' and 'watchedAttributes' present", - "title": "Inconsistent subscription", - "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData" -} - - 19.4. Creation of a subscription without TIMEINTERVAL nor WATCHEDATTRIBUTES present (entities is present) ========================================================================================================= HTTP/1.1 201 Created From a49ae22054870e98ccd2e78573a7359d09bc0956 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Fri, 8 Sep 2023 11:51:58 +0200 Subject: [PATCH 23/29] release compilation warning ... --- .../serviceRoutines/orionldPatchSubscription.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp b/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp index d247c753f1..2f2a695330 100644 --- a/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp +++ b/src/lib/orionld/serviceRoutines/orionldPatchSubscription.cpp @@ -959,14 +959,19 @@ bool orionldPatchSubscription(void) // modified. // ngsildSubscriptionPatch() performs that modification. // - CachedSubscription* cSubP; + CachedSubscription* cSubP = NULL; if (timeInterval == 0) - cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subscriptionId); - else { - // Can't get here right now + cSubP = subCacheItemLookup(orionldState.tenantP->tenant, subscriptionId); + if (cSubP == NULL) + { + orionldError(OrionldResourceNotFound, "Subscription not found", subscriptionId, 404); + return false; + } } + else + LM_X(131, ("Can't reach this point, right? ;-)")); if (ngsildSubscriptionPatch(dbSubscriptionP, cSubP, orionldState.requestTree, qP, geoqP, qRenderedForDb) == false) { From 65b5f4812e8d7bc7ead217416d87924374992790 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Fri, 8 Sep 2023 13:45:38 +0200 Subject: [PATCH 24/29] cSubCounters an uint32_t in unit test --- test/unittests/main_UnitTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unittests/main_UnitTest.cpp b/test/unittests/main_UnitTest.cpp index 078754b848..fd909ec06f 100644 --- a/test/unittests/main_UnitTest.cpp +++ b/test/unittests/main_UnitTest.cpp @@ -105,7 +105,7 @@ char dbPwd[64] = { 0 }; bool experimental = false; bool mongocOnly = false; bool debugCurl = false; -int cSubCounters = 0; +uint32_t cSubCounters = 0; char localIpAndPort[135]; unsigned long long inReqPayloadMaxSize; unsigned long long outReqMsgMaxSize; From 7d6c3f361371247d33812e80b4f0859a49e238d9 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sun, 10 Sep 2023 12:36:39 +0200 Subject: [PATCH 25/29] Adjusted two functests for different timing in github actions --- .../ngsild_new_subscription-with-mqtt-notification.test | 2 +- test/functionalTest/cases/0000_ngsild/ngsild_pernot.test | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-with-mqtt-notification.test b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-with-mqtt-notification.test index 62c7269f4f..7a6d416c0b 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-with-mqtt-notification.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_new_subscription-with-mqtt-notification.test @@ -259,7 +259,7 @@ bye 03. GET the subscription ======================== HTTP/1.1 200 OK -Content-Length: 584 +Content-Length: REGEX(.*) Content-Type: application/json Date: REGEX(.*) Link: 5;b<9)|a<4", From 0818317b168b973bd2860b7c19cede30c76477ef Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Sun, 10 Sep 2023 14:03:45 +0200 Subject: [PATCH 26/29] Adjusted two functests for different timing in github actions --- test/functionalTest/cases/0000_ngsild/ngsild_pernot.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test index 40514a9d35..601fb3bad4 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_pernot.test @@ -517,7 +517,7 @@ MongoDB server version: REGEX(.*) "ldContext" : "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld", "count" : REGEX((1|2)), "lastNotification" : REGEX(.*), - "noMatch" : REGEX((2|3)) + "noMatch" : REGEX((1|2)) } bye @@ -608,9 +608,9 @@ Link: 5;b<9)|a<4", From bf92106ace12a279b5d61b0eb4432350f4c78a07 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Mon, 11 Sep 2023 12:19:41 +0200 Subject: [PATCH 27/29] Fixed a crash in the subscription cache for subscriptions with empty URL paths in the notification::endpoint::uri field --- CHANGES_NEXT_RELEASE | 1 + .../common/subCacheApiSubscriptionInsert.cpp | 2 +- .../ngsild_subscription_cache_issue.test | 212 ++++++++++++++++++ 3 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 8ff5c2400b..015c1a55d6 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -8,3 +8,4 @@ Fixed issues: * #280 - Better 501 handling of temporal operations * #280 - Highly experimental feature for subscriptions: allowing a wildcard as value for entity type, to not filter on entity type * #280 - Implemented Periodic Notifications (subscriptions with a 'timeInterval') + * #280 - Fixed a crash in the subscription cache for subscriptions with empty URL paths in the notification::endpoint::uri field diff --git a/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp b/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp index 250484691b..d92d2d5fa6 100644 --- a/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp +++ b/src/lib/orionld/common/subCacheApiSubscriptionInsert.cpp @@ -315,7 +315,7 @@ static void subCacheItemFill cSubP->rest)); cSubP->protocolString = strdup(cSubP->protocolString); cSubP->ip = strdup(cSubP->ip); - cSubP->rest = strdup(cSubP->rest); + cSubP->rest = (cSubP->rest != NULL)? strdup(cSubP->rest) : NULL; } if (cSubP->protocol == MQTT) diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue.test b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue.test new file mode 100644 index 0000000000..a656132722 --- /dev/null +++ b/test/functionalTest/cases/0000_ngsild/ngsild_subscription_cache_issue.test @@ -0,0 +1,212 @@ +# Copyright 2023 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Possible error in Subscription Cache + +--SHELL-INIT-- +dbInit CB +orionldStart CB -experimental + +--SHELL-- + +# +# 01. Create a subscription S1 +# 02. Create a subscription S2 +# 03. Restart broker +# 04. GET Subscriptions, make sure all is OK +# + +echo "01. Create a subscription S1" +echo "============================" +payload='{ + "id": "urn:S1", + "type": "Subscription", + "description": "Notify me of all product price changes", + "entities": [ + { + "type": "product" + } + ], + "watchedAttributes": [ "name" ], + "notification": { + "format": "normalized", + "endpoint": { + "uri": "https://ttttt.free.beeceptor.com", + "accept": "application/json", + "receiverInfo": [ + { + "key": "a", + "value": "b" + } + ] + } + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create a subscription S2" +echo "============================" +payload='{ + "id": "urn:S2", + "type": "Subscription", + "description": "Notify me of all product price changes", + "entities": [ + { + "type": "product" + } + ], + "notification": { + "format": "normalized", + "endpoint": { + "uri": "https://ttttt.free.beeceptor.com", + "accept": "application/json", + "receiverInfo": [ + { + "key": "a", + "value": "b" + } + ] + } + } +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" +echo +echo + + +echo "03. Restart the broker" +echo "======================" +brokerStop CB +orionldStart CB -experimental +echo +echo + + + +echo "04. GET Subscriptions, make sure all is OK" +echo "==========================================" +orionCurl --url /ngsi-ld/v1/subscriptions +echo +echo + + +--REGEXPECT-- +01. Create a subscription S1 +============================ +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:S1 + + + +02. Create a subscription S2 +============================ +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:S2 + + + +03. Restart the broker +====================== + + +04. GET Subscriptions, make sure all is OK +========================================== +HTTP/1.1 200 OK +Content-Length: 900 +Content-Type: application/json +Date: REGEX(.*) +Link: Date: Mon, 11 Sep 2023 16:31:45 +0200 Subject: [PATCH 28/29] New 'hidden' CLI: -triggerOperation to make the broker add a field 'trigger: VERB URL-PATH' in notifications --- src/app/orionld/orionld.cpp | 3 +++ src/lib/orionld/common/orionldState.h | 1 + src/lib/orionld/notifications/notificationSend.cpp | 9 +++++++++ .../cases/0000_ngsild/ngsild_showChanges.test | 10 ++++++---- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/app/orionld/orionld.cpp b/src/app/orionld/orionld.cpp index 5e5928b3e9..b8c80c408d 100644 --- a/src/app/orionld/orionld.cpp +++ b/src/app/orionld/orionld.cpp @@ -258,6 +258,7 @@ bool mongocOnly = false; bool debugCurl = false; uint32_t cSubCounters; char coreContextVersion[64]; +bool triggerOperation = false; @@ -337,6 +338,7 @@ char coreContextVersion[64]; #define ID_INDEX_DESC "automatic mongo index on _id.id" #define NOSWAP_DESC "no swapping - for testing only!!!" #define NO_NOTIFY_FALSE_UPDATE_DESC "turn off notifications on non-updates" +#define TRIGGER_OPERATION_DESC "include the operation that triggered the notification" #define EXPERIMENTAL_DESC "enable experimental implementation - use at own risk - see release notes of Orion-LD v1.1.0" #define MONGOCONLY_DESC "enable experimental implementation + turn off mongo legacy driver" #define DBAUTHDB_DESC "database used for authentication" @@ -435,6 +437,7 @@ PaArgument paArgs[] = { "-troePoolSize", &troePoolSize, "TROE_POOL_SIZE", PaInt, PaOpt, 10, 0, 1000, TROE_POOL_DESC }, { "-distributed", &distributed, "DISTRIBUTED", PaBool, PaOpt, false, false, true, DISTRIBUTED_DESC }, { "-noNotifyFalseUpdate", &noNotifyFalseUpdate, "NO_NOTIFY_FALSE_UPDATE", PaBool, PaOpt, false, false, true, NO_NOTIFY_FALSE_UPDATE_DESC }, + { "-triggerOperation", &triggerOperation, "TRIGGER_OPERATION", PaBool, PaHid, false, false, true, TRIGGER_OPERATION_DESC }, { "-experimental", &experimental, "EXPERIMENTAL", PaBool, PaOpt, false, false, true, EXPERIMENTAL_DESC }, { "-mongocOnly", &mongocOnly, "MONGOCONLY", PaBool, PaOpt, false, false, true, MONGOCONLY_DESC }, { "-cSubCounters", &cSubCounters, "CSUB_COUNTERS", PaInt, PaOpt, 20, 0, PaNL, CSUBCOUNTERS_DESC }, diff --git a/src/lib/orionld/common/orionldState.h b/src/lib/orionld/common/orionldState.h index c96384254b..4ae458cf5b 100644 --- a/src/lib/orionld/common/orionldState.h +++ b/src/lib/orionld/common/orionldState.h @@ -582,6 +582,7 @@ extern OrionldPhase orionldPhase; extern bool orionldStartup; // For now, only used inside sub-cache routines extern bool idIndex; // From orionld.cpp extern bool noNotifyFalseUpdate; // From orionld.cpp +extern bool triggerOperation; // From orionld.cpp extern char mongoServerVersion[32]; extern bool experimental; // From orionld.cpp extern bool mongocOnly; // From orionld.cpp diff --git a/src/lib/orionld/notifications/notificationSend.cpp b/src/lib/orionld/notifications/notificationSend.cpp index ac6c5a670c..65d0216700 100644 --- a/src/lib/orionld/notifications/notificationSend.cpp +++ b/src/lib/orionld/notifications/notificationSend.cpp @@ -554,6 +554,15 @@ static KjNode* notificationTree(OrionldAlterationMatch* matchList) kjChildAdd(notificationP, notifiedAtNodeP); kjChildAdd(notificationP, dataNodeP); + // Reason for the notification + if (triggerOperation == true) + { + char trigger[128]; + snprintf(trigger, sizeof(trigger) - 1, "%s %s", orionldState.verbString, orionldState.urlPath); + KjNode* triggerP = kjString(orionldState.kjsonP, "trigger", trigger); + kjChildAdd(notificationP, triggerP); + } + for (OrionldAlterationMatch* matchP = matchList; matchP != NULL; matchP = matchP->next) { KjNode* apiEntityP = (subP->sysAttrs == false)? matchP->altP->finalApiEntityP : matchP->altP->finalApiEntityWithSysAttrsP; diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test b/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test index 8d262b4d2d..2d1de2d7a5 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_showChanges.test @@ -25,7 +25,7 @@ showChanges in Subscription - provoke a notification and see the previousValue --SHELL-INIT-- dbInit CB -orionldStart CB -experimental +orionldStart CB -experimental -triggerOperation accumulatorStart --pretty-print --SHELL-- @@ -130,7 +130,7 @@ echo echo "08. Restart broker" echo "==================" brokerStop CB -orionldStart CB -experimental +orionldStart CB -experimental -triggerOperation echo echo @@ -303,7 +303,7 @@ Date: REGEX(.*) 05. Dump/Reset accumulator, see A1's value and previousValue ============================================================ POST http://REGEX(.*)/notify?subscriptionId=urn:ngsi-ld:subs:S1 -Content-Length: 265 +Content-Length: 320 User-Agent: orionld/REGEX(.*) Host: REGEX(.*) Accept: application/json @@ -326,6 +326,7 @@ Ngsild-Attribute-Format: Normalized "id": "urn:ngsi-ld:Notification:REGEX(.*)", "notifiedAt": "20REGEX(.*)Z", "subscriptionId": "urn:ngsi-ld:subs:S1", + "trigger": "PATCH /ngsi-ld/v1/entities/urn:E1/attrs/A1", "type": "Notification" } ======================================= @@ -483,7 +484,7 @@ Date: REGEX(.*) 12. Dump/Reset accumulator, see A1's value and previousValue ============================================================ POST http://REGEX(.*)/notify?subscriptionId=urn:ngsi-ld:subs:S1 -Content-Length: 266 +Content-Length: 321 User-Agent: orionld/REGEX(.*) Host: REGEX(.*) Accept: application/json @@ -506,6 +507,7 @@ Ngsild-Attribute-Format: Normalized "id": "urn:ngsi-ld:Notification:REGEX(.*)", "notifiedAt": "20REGEX(.*)Z", "subscriptionId": "urn:ngsi-ld:subs:S1", + "trigger": "PATCH /ngsi-ld/v1/entities/urn:E1/attrs/A1", "type": "Notification" } ======================================= From 2423d65315343780e6a8e72d1d5b115e73ca3a6f Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Mon, 11 Sep 2023 16:56:08 +0200 Subject: [PATCH 29/29] New variable needs to be in main of unit test as well ... --- CHANGES_NEXT_RELEASE | 1 + test/unittests/main_UnitTest.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 015c1a55d6..e7f72cfd7a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -9,3 +9,4 @@ Fixed issues: * #280 - Highly experimental feature for subscriptions: allowing a wildcard as value for entity type, to not filter on entity type * #280 - Implemented Periodic Notifications (subscriptions with a 'timeInterval') * #280 - Fixed a crash in the subscription cache for subscriptions with empty URL paths in the notification::endpoint::uri field + * #280 - New CLI (hidden) for extra field in notifications (trigger: "VERB URL PATH"): -triggerOperation diff --git a/test/unittests/main_UnitTest.cpp b/test/unittests/main_UnitTest.cpp index fd909ec06f..3c27ace71b 100644 --- a/test/unittests/main_UnitTest.cpp +++ b/test/unittests/main_UnitTest.cpp @@ -109,6 +109,7 @@ uint32_t cSubCounters = 0; char localIpAndPort[135]; unsigned long long inReqPayloadMaxSize; unsigned long long outReqMsgMaxSize; +bool triggerOperation = false;