From 0005522391830da6b1233a7270a13fd4778c7dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 31 Jan 2024 13:06:50 +0100 Subject: [PATCH 001/104] ADD first version of jexlManager with init method to load Python interpreter --- CMakeLists.txt | 4 +++ src/app/contextBroker/contextBroker.cpp | 2 ++ src/lib/jexl/CMakeLists.txt | 42 +++++++++++++++++++++++ src/lib/jexl/JexlManager.cpp | 45 +++++++++++++++++++++++++ src/lib/jexl/JexlManager.h | 44 ++++++++++++++++++++++++ src/lib/jexl/jexlMgr.cpp | 33 ++++++++++++++++++ src/lib/jexl/jexlMgr.h | 38 +++++++++++++++++++++ src/lib/logMsg/traceLevels.h | 1 + 8 files changed, 209 insertions(+) create mode 100644 src/lib/jexl/CMakeLists.txt create mode 100644 src/lib/jexl/JexlManager.cpp create mode 100644 src/lib/jexl/JexlManager.h create mode 100644 src/lib/jexl/jexlMgr.cpp create mode 100644 src/lib/jexl/jexlMgr.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 42a046ec8d..4cae22ff0b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,6 +204,7 @@ SET (ORION_LIBS common alarmMgr metricsMgr + jexl logSummary lm pa @@ -245,6 +246,7 @@ SET (DYNAMIC_LIBS uuid crypto sasl2 + python3.11 ) # @@ -263,6 +265,7 @@ endif (UNIT_TEST) # Common include # include_directories("/usr/include") +include_directories("/usr/include/python3.11/") # Needed for the new C driver @@ -319,6 +322,7 @@ if (error EQUAL 0) ADD_SUBDIRECTORY(src/lib/cache) ADD_SUBDIRECTORY(src/lib/alarmMgr) ADD_SUBDIRECTORY(src/lib/metricsMgr) + ADD_SUBDIRECTORY(src/lib/jexl) ADD_SUBDIRECTORY(src/lib/logSummary) ADD_SUBDIRECTORY(src/lib/mqtt) ADD_SUBDIRECTORY(src/app/contextBroker) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index bf81abec8f..ccade9df1c 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -105,6 +105,7 @@ #include "alarmMgr/alarmMgr.h" #include "mqtt/mqttMgr.h" #include "metricsMgr/metricsMgr.h" +#include "jexl/jexlMgr.h" #include "logSummary/logSummary.h" #include "contextBroker/orionRestServices.h" @@ -1215,6 +1216,7 @@ int main(int argC, char* argV[]) SemOpType policy = policyGet(reqMutexPolicy); alarmMgr.init(relogAlarms); mqttMgr.init(mqttTimeout); + jexlMgr.init(); orionInit(orionExit, ORION_VERSION, policy, statCounters, statSemWait, statTiming, statNotifQueue, strictIdv1); mongoInit(dbURI, dbHost, rplSet, dbName, user, pwd, authMech, authDb, dbSSL, dbDisableRetryWrites, mtenant, dbTimeout, writeConcern, dbPoolSize, statSemWait); metricsMgr.init(!disableMetrics, statSemWait); diff --git a/src/lib/jexl/CMakeLists.txt b/src/lib/jexl/CMakeLists.txt new file mode 100644 index 0000000000..67a7a10e24 --- /dev/null +++ b/src/lib/jexl/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +CMAKE_MINIMUM_REQUIRED(VERSION 2.6) + +SET (SOURCES + JexlManager.cpp + jexlMgr.cpp +) + +SET (HEADERS + JexlManager.h + jexlMgr.h +) + + + +# Include directories +# ----------------------------------------------------------------- +include_directories("${PROJECT_SOURCE_DIR}/src/lib") + + +# Library declaration +# ----------------------------------------------------------------- +ADD_LIBRARY(jexl STATIC ${SOURCES} ${HEADERS}) diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp new file mode 100644 index 0000000000..61725b309d --- /dev/null +++ b/src/lib/jexl/JexlManager.cpp @@ -0,0 +1,45 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include + +#include "jexl/JexlManager.h" +#include "logMsg/logMsg.h" + +/* **************************************************************************** +* +* JexlManager::init - +*/ +void JexlManager::init(void) +{ + if (sem_init(&sem, 0, 1) == -1) + { + LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); + } + + // FIXME PR: Py_Finalize() has to be be called in the proper place + Py_Initialize(); + LM_T(LmtJexl, ("Python interpreter has been initialized")); +} \ No newline at end of file diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h new file mode 100644 index 0000000000..d22f442392 --- /dev/null +++ b/src/lib/jexl/JexlManager.h @@ -0,0 +1,44 @@ +#ifndef SRC_LIB_JEXL_JEXLMANAGER_H_ +#define SRC_LIB_JEXL_JEXLMANAGER_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include + +/* **************************************************************************** +* +* JexlManager - +*/ +class JexlManager +{ +private: + sem_t sem; + +public: + void init(void); +}; + +#endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file diff --git a/src/lib/jexl/jexlMgr.cpp b/src/lib/jexl/jexlMgr.cpp new file mode 100644 index 0000000000..a5e8e2cd85 --- /dev/null +++ b/src/lib/jexl/jexlMgr.cpp @@ -0,0 +1,33 @@ +/* +* +* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Ken Zangelin +*/ +#include "jexl/JexlManager.h" + + + +/* **************************************************************************** +* +* jexlMgr - +*/ +JexlManager jexlMgr; diff --git a/src/lib/jexl/jexlMgr.h b/src/lib/jexl/jexlMgr.h new file mode 100644 index 0000000000..75b22a66a8 --- /dev/null +++ b/src/lib/jexl/jexlMgr.h @@ -0,0 +1,38 @@ +#ifndef SRC_LIB_JEXL_JEXLMGR_H_ +#define SRC_LIB_JEXL_JEXLMGR_H_ + +/* +* +* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ +#include "jexl/JexlManager.h" + + + +/* **************************************************************************** +* +* metricsMgr - +*/ +extern JexlManager jexlMgr; + +#endif // SRC_LIB_JEXL_JEXLMGR_H_ diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index d0df21e423..8a5ca6a412 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -131,6 +131,7 @@ typedef enum TraceLevels LmtNotImplemented, LmtCurlContext, LmtThreadpool, + LmtJexl, LmtOldInfo = 240, // old INFO traces moved to DEBUG in Orion 2.5.0 From 42f215efc30fc2b5faf1c8ad53051426704c1edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 5 Feb 2024 16:51:59 +0100 Subject: [PATCH 002/104] FIX PoC to measure times --- src/lib/jexl/JexlManager.cpp | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 61725b309d..e14017073c 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -40,6 +40,44 @@ void JexlManager::init(void) } // FIXME PR: Py_Finalize() has to be be called in the proper place + LM_T(LmtJexl, ("Python interpreter is about to be initialized")); Py_Initialize(); LM_T(LmtJexl, ("Python interpreter has been initialized")); + + // FIXME PR: proper Py_XDECREF() has to be called in the proper place + // FIXME PR: capture error and move it to char* to print in LM_X + PyObject* jexl_module = PyImport_ImportModule("pyjexl"); + LM_T(LmtJexl, ("jexl module has been loaded")); + + // Create JEXL engine + PyObject* jexl_engine = PyObject_CallMethod(jexl_module, "JEXL", NULL); + LM_T(LmtJexl, ("jexl engine has been created")); + + // Create expression and context + PyObject* expression = Py_BuildValue("s", "x + y"); + LM_T(LmtJexl, ("jexl expression has been built")); + PyObject* context = Py_BuildValue("{s:i, s:i}", "x", 4, "y", 11); + LM_T(LmtJexl, ("jexl context has been built")); + + // Call evaluate method + PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, context); + LM_T(LmtJexl, ("jexl evaluation is done")); + + // Print result + PyObject* repr = PyObject_Repr(result); + const char* result_str = PyUnicode_AsUTF8(repr); + LM_T(LmtJexl, ("jexl evaluation result is obtainted")); + LM_T(LmtJexl, ("jexl result: %s", result_str)); + + // Free resources + Py_XDECREF(repr); + Py_XDECREF(result); + Py_XDECREF(context); + Py_XDECREF(expression); + Py_XDECREF(jexl_engine); + Py_XDECREF(jexl_module); + LM_T(LmtJexl, ("all has been freed")); + + // Finalize the Python interpreter + Py_Finalize(); } \ No newline at end of file From e696631327e1b0f820bb00064958aef8f32df537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 7 Feb 2024 16:58:22 +0100 Subject: [PATCH 003/104] FIX JexlManager init() and release() methods --- src/app/contextBroker/contextBroker.cpp | 2 + src/lib/jexl/JexlManager.cpp | 106 ++++++++++++++++-------- src/lib/jexl/JexlManager.h | 6 +- 3 files changed, 80 insertions(+), 34 deletions(-) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index ccade9df1c..992e9264ea 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -598,6 +598,8 @@ void exitFunc(void) curl_context_cleanup(); curl_global_cleanup(); + jexlMgr.release(); + #ifdef DEBUG // valgrind pass is done using DEBUG compilation, so we have to take care with // the cache releasing to avoid false positives. In production this is not really diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index e14017073c..00ba1331b6 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -28,56 +28,96 @@ #include "jexl/JexlManager.h" #include "logMsg/logMsg.h" + + +/* **************************************************************************** +* +* capturePythonError - +*/ +static const char* capturePythonError() +{ + if (PyErr_Occurred()) + { + PyObject* ptype; + PyObject* pvalue; + PyObject* ptraceback; + + // Fetch the exception type, value, and traceback + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + + if (pvalue != NULL) + { + PyObject* str_obj = PyObject_Str(pvalue); + const char* error_message = PyUnicode_AsUTF8(str_obj); + + // Release the Python objects + Py_XDECREF(str_obj); + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + + return error_message; + } + } + + PyErr_Clear(); + return ""; +} + + + /* **************************************************************************** * * JexlManager::init - */ void JexlManager::init(void) { + jexl_module = NULL; + jexl_engine = NULL; + if (sem_init(&sem, 0, 1) == -1) { LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); } - // FIXME PR: Py_Finalize() has to be be called in the proper place - LM_T(LmtJexl, ("Python interpreter is about to be initialized")); Py_Initialize(); LM_T(LmtJexl, ("Python interpreter has been initialized")); - // FIXME PR: proper Py_XDECREF() has to be called in the proper place - // FIXME PR: capture error and move it to char* to print in LM_X - PyObject* jexl_module = PyImport_ImportModule("pyjexl"); + jexl_module = PyImport_ImportModule("pyjexl"); + if (jexl_module == NULL) + { + const char* error = capturePythonError(); + LM_X(1, ("Fatal Error (error importing jexl_module: %s)", error)); + } LM_T(LmtJexl, ("jexl module has been loaded")); - // Create JEXL engine - PyObject* jexl_engine = PyObject_CallMethod(jexl_module, "JEXL", NULL); + jexl_engine = PyObject_CallMethod(jexl_module, "JEXL", NULL); + if (jexl_engine == NULL) + { + const char* error = capturePythonError(); + LM_X(1, ("Fatal Error (error creating jexl_engine: %s)", error)); + } LM_T(LmtJexl, ("jexl engine has been created")); +} + +/* **************************************************************************** +* +* JexlManager::release - +*/ +void JexlManager::release(void) +{ + if (jexl_engine != NULL) + { + Py_XDECREF(jexl_engine); + LM_T(LmtJexl, ("jexl engine has been freed")); + } + + if (jexl_module != NULL) + { + Py_XDECREF(jexl_module); + LM_T(LmtJexl, ("jexl module has been freed")); + } - // Create expression and context - PyObject* expression = Py_BuildValue("s", "x + y"); - LM_T(LmtJexl, ("jexl expression has been built")); - PyObject* context = Py_BuildValue("{s:i, s:i}", "x", 4, "y", 11); - LM_T(LmtJexl, ("jexl context has been built")); - - // Call evaluate method - PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, context); - LM_T(LmtJexl, ("jexl evaluation is done")); - - // Print result - PyObject* repr = PyObject_Repr(result); - const char* result_str = PyUnicode_AsUTF8(repr); - LM_T(LmtJexl, ("jexl evaluation result is obtainted")); - LM_T(LmtJexl, ("jexl result: %s", result_str)); - - // Free resources - Py_XDECREF(repr); - Py_XDECREF(result); - Py_XDECREF(context); - Py_XDECREF(expression); - Py_XDECREF(jexl_engine); - Py_XDECREF(jexl_module); - LM_T(LmtJexl, ("all has been freed")); - - // Finalize the Python interpreter Py_Finalize(); + LM_T(LmtJexl, ("Python interpreter has been finalized")); } \ No newline at end of file diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h index d22f442392..6d430154ec 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/jexl/JexlManager.h @@ -26,6 +26,7 @@ * Author: Fermin Galan */ +#include #include /* **************************************************************************** @@ -35,10 +36,13 @@ class JexlManager { private: - sem_t sem; + PyObject* jexl_module; + PyObject* jexl_engine; + sem_t sem; public: void init(void); + void release(void); }; #endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file From 10386226b7fadea1ffa26d57b9fc84546455a597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 18:25:44 +0100 Subject: [PATCH 004/104] FIX replace map by JEXL context (compiles but still wip) --- src/lib/apiTypesV2/Entity.cpp | 9 +- src/lib/apiTypesV2/Entity.h | 5 +- src/lib/common/macroSubstitute.cpp | 32 +++---- src/lib/common/macroSubstitute.h | 8 +- src/lib/jexl/CMakeLists.txt | 4 +- src/lib/jexl/JexlContext.cpp | 83 +++++++++++++++++++ src/lib/jexl/JexlContext.h | 50 +++++++++++ src/lib/ngsi/ContextAttribute.cpp | 6 +- src/lib/ngsi/ContextAttribute.h | 3 +- src/lib/ngsi/ContextElementResponse.cpp | 4 +- src/lib/ngsi/ContextElementResponse.h | 3 +- src/lib/ngsi/ContextElementResponseVector.cpp | 4 +- src/lib/ngsi/ContextElementResponseVector.h | 3 +- src/lib/ngsi10/NotifyContextRequest.cpp | 8 +- src/lib/ngsi10/NotifyContextRequest.h | 2 +- src/lib/ngsiNotify/Notifier.cpp | 37 +++++---- src/lib/parse/CompoundValueNode.cpp | 8 +- src/lib/parse/CompoundValueNode.h | 4 +- 18 files changed, 210 insertions(+), 63 deletions(-) create mode 100644 src/lib/jexl/JexlContext.cpp create mode 100644 src/lib/jexl/JexlContext.h diff --git a/src/lib/apiTypesV2/Entity.cpp b/src/lib/apiTypesV2/Entity.cpp index 7c3fd7abba..5501c18e77 100644 --- a/src/lib/apiTypesV2/Entity.cpp +++ b/src/lib/apiTypesV2/Entity.cpp @@ -41,6 +41,7 @@ #include "rest/OrionError.h" #include "apiTypesV2/Entity.h" +#include "jexl/JexlContext.h" @@ -307,7 +308,7 @@ std::string Entity::toJson bool blacklist, const std::vector& metadataFilter, bool renderNgsiField, - std::map* replacementsP + JexlContext* jexlContextP ) { std::vector orderedAttrs; @@ -326,7 +327,7 @@ std::string Entity::toJson out = toJsonKeyvalues(orderedAttrs); break; default: // NGSI_V2_NORMALIZED - out = toJsonNormalized(orderedAttrs, metadataFilter, renderNgsiField, replacementsP); + out = toJsonNormalized(orderedAttrs, metadataFilter, renderNgsiField, jexlContextP); break; } @@ -443,7 +444,7 @@ std::string Entity::toJsonNormalized const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField, - std::map* replacementsP + JexlContext* jexlContextP ) { JsonObjectHelper jh; @@ -475,7 +476,7 @@ std::string Entity::toJsonNormalized for (unsigned int ix = 0; ix < orderedAttrs.size(); ix++) { ContextAttribute* caP = orderedAttrs[ix]; - jh.addRaw(caP->name, caP->toJson(metadataFilter, renderNgsiField, replacementsP)); + jh.addRaw(caP->name, caP->toJson(metadataFilter, renderNgsiField, jexlContextP)); } return jh.str(); diff --git a/src/lib/apiTypesV2/Entity.h b/src/lib/apiTypesV2/Entity.h index e4906bb998..1813faa0c1 100644 --- a/src/lib/apiTypesV2/Entity.h +++ b/src/lib/apiTypesV2/Entity.h @@ -32,6 +32,7 @@ #include "ngsi/ContextAttributeVector.h" #include "ngsi/EntityId.h" #include "rest/OrionError.h" +#include "jexl/JexlContext.h" @@ -91,7 +92,7 @@ class Entity bool blacklist, const std::vector& metadataFilter, bool renderNgsiField = false, - std::map* replacementsP = NULL); + JexlContext* jexlContext = NULL); std::string toJson(RenderFormat renderFormat, bool renderNgsiField = false); @@ -145,7 +146,7 @@ class Entity std::string toJsonNormalized(const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField = false, - std::map* replacementsP = NULL); + JexlContext* jexlContext = NULL); }; #endif // SRC_LIB_APITYPESV2_ENTITY_H_ diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index b17ac6487c..9ab9ac2131 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -40,13 +40,14 @@ * * Returns the effective string value, taking into account replacements */ -std::string smartStringValue(const std::string stringValue, std::map* replacementsP, const std::string notFoundDefault) +std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault) { // This code is pretty similar to the one in CompoundValueNode::toJson() // The program logic branching is the same, but the result at the end of each if-else // is different, which makes difficult to unify both them - if ((replacementsP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) + if ((jexlContextP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) { + /* FIXME PR: needs adaptation replacementsP -> jexlContextP // "Full replacement" case. In this case, the result is not always a string // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); @@ -59,13 +60,14 @@ std::string smartStringValue(const std::string stringValue, std::mapsecond; - } + }*/ + return ""; } - else if (replacementsP != NULL) + else if (jexlContextP != NULL) { // "Partial replacement" case. In this case, the result is always a string std::string effectiveValue; - if (!macroSubstitute(&effectiveValue, stringValue, replacementsP, "null")) + if (!macroSubstitute(&effectiveValue, stringValue, jexlContextP, "null")) { // error already logged in macroSubstitute, using stringValue itself as failsafe effectiveValue = stringValue; @@ -87,7 +89,7 @@ std::string smartStringValue(const std::string stringValue, std::mapname] = en.attributeVector[ix]->toJsonValue(); } -} +}*/ @@ -114,16 +116,17 @@ void buildReplacementsMap * * stringValueOrNothing - */ -static std::string stringValueOrNothing(std::map* replacementsP, const std::string key, const std::string& notFoundDefault) +static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault) { - std::map::iterator iter = replacementsP->find(key); + // FIXME PR: to from replacementsP to jexlContextP + /*std::map::iterator iter = replacementsP->find(key); if (iter == replacementsP->end()) { return notFoundDefault; } else { - // replacementP contents are prepared for "full replacement" case, so string values use + // replacementsP contents are prepared for "full replacement" case, so string values use // double quotes. But in this case we are in a "partial replacement" case, so we have // to remove them if we find them std::string value = iter->second; @@ -135,7 +138,8 @@ static std::string stringValueOrNothing(std::map* repl { return value; } - } + }*/ + return ""; } @@ -165,7 +169,7 @@ static std::string stringValueOrNothing(std::map* repl * Date: Mon Jun 19 16:33:29 2017 +0200 * */ -bool macroSubstitute(std::string* to, const std::string& from, std::map* replacementsP, const std::string& notFoundDefault) +bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexlContextP, const std::string& notFoundDefault) { // Initial size check: is the string to convert too big? // @@ -228,7 +232,7 @@ bool macroSubstitute(std::string* to, const std::string& from, std::map outReqMsgMaxSize) @@ -246,7 +250,7 @@ bool macroSubstitute(std::string* to, const std::string& from, std::mapsecond; std::string macro = "${" + macroName + "}"; - std::string value = stringValueOrNothing(replacementsP, macroName, notFoundDefault); + std::string value = stringValueOrNothing(jexlContextP, macroName, notFoundDefault); // We have to do the replace operation as many times as macro occurrences for (unsigned int ix = 0; ix < times; ix++) diff --git a/src/lib/common/macroSubstitute.h b/src/lib/common/macroSubstitute.h index 288c25540b..0483fc23ae 100644 --- a/src/lib/common/macroSubstitute.h +++ b/src/lib/common/macroSubstitute.h @@ -34,7 +34,7 @@ * smartStringValue - * */ -extern std::string smartStringValue(const std::string stringValue, std::map* replacementsP, const std::string notFoundDefault); +extern std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault); @@ -43,13 +43,13 @@ extern std::string smartStringValue(const std::string stringValue, std::map* replacementsP -); +);*/ @@ -58,6 +58,6 @@ extern void buildReplacementsMap * macroSubstitute - * */ -extern bool macroSubstitute(std::string* sP, const std::string& in, std::map* replacementsP, const std::string& notFoundDefault); +extern bool macroSubstitute(std::string* sP, const std::string& in, JexlContext* jexlContextP, const std::string& notFoundDefault); #endif // SRC_LIB_COMMON_MACROSUBSTITUTE_H_ diff --git a/src/lib/jexl/CMakeLists.txt b/src/lib/jexl/CMakeLists.txt index 67a7a10e24..abacbeb3ca 100644 --- a/src/lib/jexl/CMakeLists.txt +++ b/src/lib/jexl/CMakeLists.txt @@ -22,11 +22,13 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES JexlManager.cpp - jexlMgr.cpp + JexlContext.cpp + jexlMgr.cpp ) SET (HEADERS JexlManager.h + JexlContext.h jexlMgr.h ) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp new file mode 100644 index 0000000000..33dd5e877d --- /dev/null +++ b/src/lib/jexl/JexlContext.cpp @@ -0,0 +1,83 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "jexl/JexlContext.h" + +#include + +/* **************************************************************************** +* +* JexlContext::JexlContext - +*/ +JexlContext::JexlContext(const std::string& id, const std::string& type, const std::string& service, const std::string& servicePath, const std::string& token) +{ + jexl_context = PyDict_New(); + + if (jexl_context == NULL) + { + // FIXME PR: error control + } + + PyObject* value; + + value = Py_BuildValue("s", id.c_str()); + PyDict_SetItemString(jexl_context, "id", value); + Py_DECREF(value); + + value = Py_BuildValue("s", type.c_str()); + PyDict_SetItemString(jexl_context, "type", value); + Py_DECREF(value); + + value = Py_BuildValue("s", service.c_str()); + PyDict_SetItemString(jexl_context, "service", value); + Py_DECREF(value); + + value = Py_BuildValue("s", servicePath.c_str()); + PyDict_SetItemString(jexl_context, "servicePath", value); + Py_DECREF(value); + + value = Py_BuildValue("s", token.c_str()); + PyDict_SetItemString(jexl_context, "authToken", value); + Py_DECREF(value); + + /*for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) + { + // FIXME PR: the first argument in Py_BuildValue would depend on the attribute type, "s" is only for string type + value = Py_BuildValue("s", en.attributeVector[ix]->toJsonValue().c_str()); + PyDict_SetItemString(jexl_context, en.attributeVector[ix]->name.c_str(), value); + Py_DECREF(value); + }*/ +} + + + +/* **************************************************************************** +* +* JexlContext::~JexlContext - +*/ +JexlContext::~JexlContext() +{ + Py_XDECREF(jexl_context); +} diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h new file mode 100644 index 0000000000..bc5bbd27dd --- /dev/null +++ b/src/lib/jexl/JexlContext.h @@ -0,0 +1,50 @@ +#ifndef SRC_LIB_JEXL_JEXLCONTEXT_H_ +#define SRC_LIB_JEXL_JEXLCONTEXT_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include + +#include + +/* **************************************************************************** +* +* JexlContext - +*/ +class JexlContext +{ +private: + PyObject* jexl_context; + +public: + ~JexlContext(); + + JexlContext(const std::string& id, const std::string& type, const std::string& service, const std::string& servicePath, const std::string& token); +}; + + + +#endif // #define SRC_LIB_JEXL_JEXLCONTEXT_H_ diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 88681d879a..f4407a89fc 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1052,7 +1052,7 @@ void ContextAttribute::filterAndOrderMetadata * renderNgsiField true is used in custom notification payloads, which have some small differences * with regards to conventional rendering */ -std::string ContextAttribute::toJson(const std::vector& metadataFilter, bool renderNgsiField, std::map* replacementsP) +std::string ContextAttribute::toJson(const std::vector& metadataFilter, bool renderNgsiField, JexlContext* jexlContextP) { JsonObjectHelper jh; @@ -1085,7 +1085,7 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi // of DB entities) may lead to NULL, so the check is needed if (childToRenderP != NULL) { - jh.addRaw("value", childToRenderP->toJson(replacementsP)); + jh.addRaw("value", childToRenderP->toJson(jexlContextP)); } } else if (valueType == orion::ValueTypeNumber) @@ -1101,7 +1101,7 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi } else if (valueType == orion::ValueTypeString) { - jh.addRaw("value", smartStringValue(stringValue, replacementsP, "null")); + jh.addRaw("value", smartStringValue(stringValue, jexlContextP, "null")); } else if (valueType == orion::ValueTypeBoolean) { diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index 58d7ed39f1..1ca58a0a98 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -37,6 +37,7 @@ #include "parse/CompoundValueNode.h" #include "rest/HttpStatusCode.h" #include "mongoDriver/BSONObjBuilder.h" +#include "jexl/JexlContext.h" @@ -107,7 +108,7 @@ typedef struct ContextAttribute std::string toJsonV1AsNameString(bool comma); - std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, std::map* replacementsP = NULL); + std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, JexlContext* jexlContextP = NULL); std::string toJsonValue(void); diff --git a/src/lib/ngsi/ContextElementResponse.cpp b/src/lib/ngsi/ContextElementResponse.cpp index ba9d3ae0b2..262d0f2a4c 100644 --- a/src/lib/ngsi/ContextElementResponse.cpp +++ b/src/lib/ngsi/ContextElementResponse.cpp @@ -215,12 +215,12 @@ std::string ContextElementResponse::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP + JexlContext* jexlContextP ) { std::string out; - out = entity.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, false, replacementsP); + out = entity.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, false, jexlContextP); return out; } diff --git a/src/lib/ngsi/ContextElementResponse.h b/src/lib/ngsi/ContextElementResponse.h index 87b3bc17b1..471a8e66e6 100644 --- a/src/lib/ngsi/ContextElementResponse.h +++ b/src/lib/ngsi/ContextElementResponse.h @@ -33,6 +33,7 @@ #include "ngsi/StringList.h" #include "ngsi/ContextAttribute.h" #include "apiTypesV2/Entity.h" +#include "jexl/JexlContext.h" #include "mongoDriver/BSONObj.h" @@ -82,7 +83,7 @@ typedef struct ContextElementResponse const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP); + JexlContext* jexlContext); void applyUpdateOperators(void); diff --git a/src/lib/ngsi/ContextElementResponseVector.cpp b/src/lib/ngsi/ContextElementResponseVector.cpp index e044e9779d..03fd35741a 100644 --- a/src/lib/ngsi/ContextElementResponseVector.cpp +++ b/src/lib/ngsi/ContextElementResponseVector.cpp @@ -119,14 +119,14 @@ std::string ContextElementResponseVector::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP + JexlContext* jexlContextP ) { JsonVectorHelper jvh; for (unsigned int ix = 0; ix < vec.size(); ++ix) { - jvh.addRaw(vec[ix]->toJson(renderFormat, attrsFilter, blacklist, metadataFilter, replacementsP)); + jvh.addRaw(vec[ix]->toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP)); } return jvh.str(); diff --git a/src/lib/ngsi/ContextElementResponseVector.h b/src/lib/ngsi/ContextElementResponseVector.h index 92f74e5ff6..16cb7afb7c 100644 --- a/src/lib/ngsi/ContextElementResponseVector.h +++ b/src/lib/ngsi/ContextElementResponseVector.h @@ -31,6 +31,7 @@ #include "ngsi/ContextElementResponse.h" #include "apiTypesV2/EntityVector.h" #include "common/RenderFormat.h" +#include "jexl/JexlContext.h" @@ -60,7 +61,7 @@ typedef struct ContextElementResponseVector const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP); + JexlContext* jexlContextP); void push_back(ContextElementResponse* item); unsigned int size(void) const; ContextElementResponse* lookup(Entity* eP, HttpStatusCode code = SccNone); diff --git a/src/lib/ngsi10/NotifyContextRequest.cpp b/src/lib/ngsi10/NotifyContextRequest.cpp index b9b4ac5af7..30122bfcb0 100644 --- a/src/lib/ngsi10/NotifyContextRequest.cpp +++ b/src/lib/ngsi10/NotifyContextRequest.cpp @@ -80,7 +80,7 @@ std::string NotifyContextRequest::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP + JexlContext* jexlContextP ) { if ((renderFormat != NGSI_V2_NORMALIZED) && (renderFormat != NGSI_V2_KEYVALUES) && (renderFormat != NGSI_V2_VALUES) && (renderFormat != NGSI_V2_SIMPLIFIEDKEYVALUES) && (renderFormat != NGSI_V2_SIMPLIFIEDNORMALIZED)) @@ -100,7 +100,7 @@ std::string NotifyContextRequest::toJson else { std::string out; - out += contextElementResponseVector[0]->toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, replacementsP); + out += contextElementResponseVector[0]->toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, jexlContextP); return out; } } @@ -114,7 +114,7 @@ std::string NotifyContextRequest::toJson else { std::string out; - out += contextElementResponseVector[0]->toJson(NGSI_V2_KEYVALUES, attrsFilter, blacklist, metadataFilter, replacementsP); + out += contextElementResponseVector[0]->toJson(NGSI_V2_KEYVALUES, attrsFilter, blacklist, metadataFilter, jexlContextP); return out; } } @@ -123,7 +123,7 @@ std::string NotifyContextRequest::toJson JsonObjectHelper jh; jh.addString("subscriptionId", subscriptionId.get()); - jh.addRaw("data", contextElementResponseVector.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, replacementsP)); + jh.addRaw("data", contextElementResponseVector.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP)); return jh.str(); } } diff --git a/src/lib/ngsi10/NotifyContextRequest.h b/src/lib/ngsi10/NotifyContextRequest.h index c4043704d5..80e4a19c8d 100644 --- a/src/lib/ngsi10/NotifyContextRequest.h +++ b/src/lib/ngsi10/NotifyContextRequest.h @@ -53,7 +53,7 @@ typedef struct NotifyContextRequest const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP = NULL); + JexlContext* jexlContextP = NULL); std::string check(ApiVersion apiVersion, const std::string& predetectedError); void release(void); NotifyContextRequest* clone(void); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index aae1ecb34f..32db3e2adc 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -121,7 +121,7 @@ static bool setPayload const std::string& notifPayload, const SubscriptionId& subscriptionId, Entity& en, - std::map* replacementsP, + JexlContext* jexlContextP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -170,7 +170,7 @@ static bool setPayload } else { - if (!macroSubstitute(payloadP, notifPayload, replacementsP, "")) + if (!macroSubstitute(payloadP, notifPayload, jexlContextP, "")) { return false; } @@ -194,12 +194,12 @@ static bool setPayload static bool setJsonPayload ( orion::CompoundValueNode* json, - std::map* replacementsP, + JexlContext* jexlContextP, std::string* payloadP, std::string* mimeTypeP ) { - *payloadP = json->toJson(replacementsP); + *payloadP = json->toJson(jexlContextP); *mimeTypeP = "application/json"; // this can be overriden by headers field return true; } @@ -237,7 +237,7 @@ static bool setNgsiPayload const Entity& ngsi, const SubscriptionId& subscriptionId, Entity& en, - std::map* replacementsP, + JexlContext* jexlContextP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -256,7 +256,7 @@ static bool setNgsiPayload else { // If id is not found in the replacements macro, we use en.id. - effectiveId = removeQuotes(smartStringValue(ngsi.id, replacementsP, '"' + en.id + '"')); + effectiveId = removeQuotes(smartStringValue(ngsi.id, jexlContextP, '"' + en.id + '"')); } std::string effectiveType; @@ -267,7 +267,7 @@ static bool setNgsiPayload else { // If type is not found in the replacements macro, we use en.type. - effectiveType = removeQuotes(smartStringValue(ngsi.type, replacementsP, '"' + en.type + '"')); + effectiveType = removeQuotes(smartStringValue(ngsi.type, jexlContextP, '"' + en.type + '"')); } cer.entity.fill(effectiveId, effectiveType, en.isPattern, en.servicePath); @@ -293,11 +293,11 @@ static bool setNgsiPayload if ((renderFormat == NGSI_V2_SIMPLIFIEDNORMALIZED) || (renderFormat == NGSI_V2_SIMPLIFIEDKEYVALUES)) { - *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, replacementsP); + *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP); } else { - *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, replacementsP); + *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, jexlContextP); } return true; @@ -336,8 +336,9 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - std::map replacements; - buildReplacementsMap(en, tenant, xauthToken, &replacements); + //std::map replacements; + //buildReplacementsMap(en, tenant, xauthToken, &replacements); + JexlContext jexlContext(en.id, en.type, tenant, en.servicePath, xauthToken); // // 1. Verb/Method @@ -363,7 +364,7 @@ static SenderThreadParams* buildSenderParamsCustom // 2. URL // std::string notifUrl = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.url : notification.mqttInfo.url); - if (macroSubstitute(&url, notifUrl, &replacements, "") == false) + if (macroSubstitute(&url, notifUrl, &jexlContext, "") == false) { // Warning already logged in macroSubstitute() return NULL; @@ -379,7 +380,7 @@ static SenderThreadParams* buildSenderParamsCustom { bool includePayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.includePayload : notification.mqttInfo.includePayload); std::string notifPayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.payload : notification.mqttInfo.payload); - if (!setPayload(includePayload, notifPayload, subscriptionId, en, &replacements, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) + if (!setPayload(includePayload, notifPayload, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) { // Warning already logged in macroSubstitute() return NULL; @@ -388,14 +389,14 @@ static SenderThreadParams* buildSenderParamsCustom else if (customPayloadType == ngsiv2::CustomPayloadType::Json) { orion::CompoundValueNode* json = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.json : notification.mqttInfo.json); - setJsonPayload(json, &replacements, &payload, &mimeType); + setJsonPayload(json, &jexlContext, &payload, &mimeType); renderFormat = NGSI_V2_CUSTOM; } else // customPayloadType == ngsiv2::CustomPayloadType::Ngsi { // Important to use const& for Entity here. Otherwise problems may occur in the object release logic const Entity& ngsi = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.ngsi : notification.mqttInfo.ngsi); - if (!setNgsiPayload(ngsi, subscriptionId, en, &replacements, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) + if (!setNgsiPayload(ngsi, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) { // Warning already logged in macroSubstitute() return NULL; @@ -414,7 +415,7 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &replacements, "") == false) || (macroSubstitute(&value, it->second, &replacements, "") == false)) + if ((macroSubstitute(&key, it->first, &jexlContext, "") == false) || (macroSubstitute(&value, it->second, &jexlContext, "") == false)) { // Warning already logged in macroSubstitute() return NULL; @@ -440,7 +441,7 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &replacements, "") == false) || (macroSubstitute(&value, it->second, &replacements, "") == false)) + if ((macroSubstitute(&key, it->first, &jexlContext, "") == false) || (macroSubstitute(&value, it->second, &jexlContext, "") == false)) { // Warning already logged in macroSubstitute() return NULL; @@ -505,7 +506,7 @@ static SenderThreadParams* buildSenderParamsCustom // 8. Topic (only in the case of MQTT notifications) if (notification.type == ngsiv2::MqttNotification) { - if (macroSubstitute(&topic, notification.mqttInfo.topic, &replacements, "") == false) + if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "") == false) { // Warning already logged in macroSubstitute() return NULL; diff --git a/src/lib/parse/CompoundValueNode.cpp b/src/lib/parse/CompoundValueNode.cpp index 454c239383..6dfb6f59d4 100644 --- a/src/lib/parse/CompoundValueNode.cpp +++ b/src/lib/parse/CompoundValueNode.cpp @@ -612,7 +612,7 @@ bool CompoundValueNode::equal(const orion::BSONElement& be) * CompoundValueNode:toJson * */ -std::string CompoundValueNode::toJson(std::map* replacementsP) +std::string CompoundValueNode::toJson(JexlContext* jexlContextP) { std::string out; JsonVectorHelper jvh; @@ -621,7 +621,7 @@ std::string CompoundValueNode::toJson(std::map* replac switch (valueType) { case orion::ValueTypeString: - return smartStringValue(stringValue, replacementsP, "null"); + return smartStringValue(stringValue, jexlContextP, "null"); case orion::ValueTypeNumber: return double2string(numberValue); @@ -635,14 +635,14 @@ std::string CompoundValueNode::toJson(std::map* replac case orion::ValueTypeVector: for (unsigned int ix = 0; ix < childV.size(); ix++) { - jvh.addRaw(childV[ix]->toJson(replacementsP)); + jvh.addRaw(childV[ix]->toJson(jexlContextP)); } return jvh.str(); case orion::ValueTypeObject: for (unsigned int ix = 0; ix < childV.size(); ix++) { - joh.addRaw(childV[ix]->name, childV[ix]->toJson(replacementsP)); + joh.addRaw(childV[ix]->name, childV[ix]->toJson(jexlContextP)); } return joh.str(); diff --git a/src/lib/parse/CompoundValueNode.h b/src/lib/parse/CompoundValueNode.h index b46aa2c461..12fd070a31 100644 --- a/src/lib/parse/CompoundValueNode.h +++ b/src/lib/parse/CompoundValueNode.h @@ -35,6 +35,8 @@ #include "mongoDriver/BSONElement.h" +#include "jexl/JexlContext.h" + namespace orion { @@ -117,7 +119,7 @@ class CompoundValueNode bool equal(const orion::BSONElement& be); std::string finish(void); - std::string toJson(std::map* replacementsP = NULL); + std::string toJson(JexlContext* jexlContextP = NULL); void shortShow(const std::string& indent); void show(const std::string& indent); From cda7164eae804cb35dc5354f4ed49600cfb0ca91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 18:46:25 +0100 Subject: [PATCH 005/104] FIX improve JexlContext class --- src/lib/jexl/JexlContext.cpp | 28 ++++++++++++++++++++-------- src/lib/jexl/JexlContext.h | 12 +++++++++++- src/lib/ngsiNotify/Notifier.cpp | 4 ++++ 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index 33dd5e877d..5e8d76441c 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -31,7 +31,14 @@ * * JexlContext::JexlContext - */ -JexlContext::JexlContext(const std::string& id, const std::string& type, const std::string& service, const std::string& servicePath, const std::string& token) +JexlContext::JexlContext +( + const std::string& id, + const std::string& type, + const std::string& service, + const std::string& servicePath, + const std::string& token +) { jexl_context = PyDict_New(); @@ -61,14 +68,19 @@ JexlContext::JexlContext(const std::string& id, const std::string& type, const s value = Py_BuildValue("s", token.c_str()); PyDict_SetItemString(jexl_context, "authToken", value); Py_DECREF(value); +} - /*for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) - { - // FIXME PR: the first argument in Py_BuildValue would depend on the attribute type, "s" is only for string type - value = Py_BuildValue("s", en.attributeVector[ix]->toJsonValue().c_str()); - PyDict_SetItemString(jexl_context, en.attributeVector[ix]->name.c_str(), value); - Py_DECREF(value); - }*/ + + +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, const std::string& _value) +{ + PyObject* value = Py_BuildValue("s", _value.c_str()); + PyDict_SetItemString(jexl_context, key.c_str(), value); + Py_DECREF(value); } diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index bc5bbd27dd..88ef424896 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -30,6 +30,7 @@ #include + /* **************************************************************************** * * JexlContext - @@ -42,7 +43,16 @@ class JexlContext public: ~JexlContext(); - JexlContext(const std::string& id, const std::string& type, const std::string& service, const std::string& servicePath, const std::string& token); + JexlContext + ( + const std::string& id, + const std::string& type, + const std::string& service, + const std::string& servicePath, + const std::string& token + ); + + void add(const std::string& key, const std::string& value); }; diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 32db3e2adc..f44d4addf8 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -339,6 +339,10 @@ static SenderThreadParams* buildSenderParamsCustom //std::map replacements; //buildReplacementsMap(en, tenant, xauthToken, &replacements); JexlContext jexlContext(en.id, en.type, tenant, en.servicePath, xauthToken); + for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) + { + jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); + } // // 1. Verb/Method From e1389f6e0373f7aa7546be4ad874beca6dba7698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 19:26:43 +0100 Subject: [PATCH 006/104] FIX improve JexlManager and JexlContext classes --- src/lib/jexl/JexlContext.cpp | 28 +++++++++++++++++++++++++++- src/lib/jexl/JexlContext.h | 4 +++- src/lib/jexl/JexlManager.cpp | 35 +++++++++++++++++++++++++++++++++++ src/lib/jexl/JexlManager.h | 7 +++++-- 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index 5e8d76441c..b9e5968cbe 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -71,6 +71,16 @@ JexlContext::JexlContext } +/* **************************************************************************** +* +* JexlContext::get - +*/ +PyObject* JexlContext::get(void) +{ + return jexl_context; +} + + /* **************************************************************************** * @@ -85,6 +95,22 @@ void JexlContext::add(const std::string& key, const std::string& _value) +/* **************************************************************************** +* +* JexlContext::hasKey - +*/ +bool JexlContext::hasKey(const std::string& key) +{ + // Check if the key exists in the jexl_context dictionary + PyObject* keyObject = PyUnicode_FromString(key.c_str()); + int result = PyDict_Contains(jexl_context, keyObject); + Py_DECREF(keyObject); + + return result == 1; // Return true if key exists, false otherwise +} + + + /* **************************************************************************** * * JexlContext::~JexlContext - @@ -92,4 +118,4 @@ void JexlContext::add(const std::string& key, const std::string& _value) JexlContext::~JexlContext() { Py_XDECREF(jexl_context); -} +} \ No newline at end of file diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index 88ef424896..77d6083c07 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -52,7 +52,9 @@ class JexlContext const std::string& token ); - void add(const std::string& key, const std::string& value); + PyObject* get(void); + void add(const std::string& key, const std::string& value); + bool hasKey(const std::string& key); }; diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 00ba1331b6..4ad612d391 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -100,6 +100,41 @@ void JexlManager::init(void) LM_T(LmtJexl, ("jexl engine has been created")); } + + +/* **************************************************************************** +* +* JexlManager::evaluate - +*/ +std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) +{ + PyObject* expression = Py_BuildValue("s", _expression); + if (expression == NULL) + { + // FIXME PR: manage error + return "null"; + } + + PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, jexlContextP->get()); + if (result == NULL) + { + // FIXME PR: manage error + return "null"; + } + + PyObject* repr = PyObject_Repr(result); + if (repr == NULL) + { + // FIXME PR: manage error + return "null"; + } + + const char* result_str = PyUnicode_AsUTF8(repr); + return std::string(result_str); +} + + + /* **************************************************************************** * * JexlManager::release - diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h index 6d430154ec..89cadcf534 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/jexl/JexlManager.h @@ -29,6 +29,8 @@ #include #include +#include "jexl/JexlContext.h" + /* **************************************************************************** * * JexlManager - @@ -41,8 +43,9 @@ class JexlManager sem_t sem; public: - void init(void); - void release(void); + void init(void); + std::string evaluate(JexlContext* jexlContextP, const std::string& expression); + void release(void); }; #endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file From cbd523b3cfffd8b459f8a2caa4688fe658134eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 19:33:00 +0100 Subject: [PATCH 007/104] FIX use JEXL evaluate to evaluate macros --- src/lib/common/macroSubstitute.cpp | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 9ab9ac2131..5c1a98c74d 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -32,6 +32,7 @@ #include "common/JsonHelper.h" #include "common/macroSubstitute.h" +#include "jexl/jexlMgr.h" /* **************************************************************************** @@ -39,6 +40,9 @@ * smartStringValue - * * Returns the effective string value, taking into account replacements +* +* FIXME PR: notFoundDefault is not longer used? +* */ std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault) { @@ -47,21 +51,11 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // is different, which makes difficult to unify both them if ((jexlContextP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) { - /* FIXME PR: needs adaptation replacementsP -> jexlContextP // "Full replacement" case. In this case, the result is not always a string // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - std::map::iterator iter = replacementsP->find(macroName); - if (iter == replacementsP->end()) - { - // macro doesn't exist in the replacement map, so we use null as failsafe - return notFoundDefault; - } - else - { - return iter->second; - }*/ - return ""; + + return jexlMgr.evaluate(jexlContextP, macroName); } else if (jexlContextP != NULL) { @@ -115,10 +109,12 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon /* **************************************************************************** * * stringValueOrNothing - +* +* FIXME PR: notFoundDefault no longer used? */ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault) { - // FIXME PR: to from replacementsP to jexlContextP + return jexlMgr.evaluate(jexlContextP, key); /*std::map::iterator iter = replacementsP->find(key); if (iter == replacementsP->end()) { @@ -139,7 +135,6 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st return value; } }*/ - return ""; } From 590af6e7d4126e7eff4d29f6dd8ab184e6d35d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 19:54:24 +0100 Subject: [PATCH 008/104] FIX increase debug log tracing in jexl library --- src/lib/jexl/JexlContext.cpp | 11 ++++++++++- src/lib/jexl/JexlManager.cpp | 22 ++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index b9e5968cbe..a34f6962c4 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -23,9 +23,12 @@ * Author: Fermin Galan */ +#include + +#include "logMsg/logMsg.h" #include "jexl/JexlContext.h" -#include + /* **************************************************************************** * @@ -49,22 +52,27 @@ JexlContext::JexlContext PyObject* value; + LM_T(LmtJexl, ("adding to JEXL context: id=%s", id.c_str())); value = Py_BuildValue("s", id.c_str()); PyDict_SetItemString(jexl_context, "id", value); Py_DECREF(value); + LM_T(LmtJexl, ("adding to JEXL context: type=%s", type.c_str())); value = Py_BuildValue("s", type.c_str()); PyDict_SetItemString(jexl_context, "type", value); Py_DECREF(value); + LM_T(LmtJexl, ("adding to JEXL context: service=%s", service.c_str())); value = Py_BuildValue("s", service.c_str()); PyDict_SetItemString(jexl_context, "service", value); Py_DECREF(value); + LM_T(LmtJexl, ("adding to JEXL context: servicePath=%s", servicePath.c_str())); value = Py_BuildValue("s", servicePath.c_str()); PyDict_SetItemString(jexl_context, "servicePath", value); Py_DECREF(value); + LM_T(LmtJexl, ("adding to JEXL context: token=%s", token.c_str())); value = Py_BuildValue("s", token.c_str()); PyDict_SetItemString(jexl_context, "authToken", value); Py_DECREF(value); @@ -88,6 +96,7 @@ PyObject* JexlContext::get(void) */ void JexlContext::add(const std::string& key, const std::string& _value) { + LM_T(LmtJexl, ("adding to JEXL context: %s=%s", key.c_str(), _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); PyDict_SetItemString(jexl_context, key.c_str(), value); Py_DECREF(value); diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 4ad612d391..20dd12864f 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -108,29 +108,39 @@ void JexlManager::init(void) */ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { - PyObject* expression = Py_BuildValue("s", _expression); + LM_T(LmtJexl, ("evaluating JEXL expresion: %s", _expression.c_str())); + + PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { - // FIXME PR: manage error + // FIXME PR: grab error message from Python stack + LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, jexlContextP->get()); + Py_XDECREF(expression); if (result == NULL) { - // FIXME PR: manage error + // FIXME PR: grab error message from Python stack + LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } PyObject* repr = PyObject_Repr(result); + Py_XDECREF(result); if (repr == NULL) { - // FIXME PR: manage error + // FIXME PR: grab error message from Python stack + LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } - const char* result_str = PyUnicode_AsUTF8(repr); - return std::string(result_str); + std::string result_str = std::string(PyUnicode_AsUTF8(repr)); + Py_XDECREF(repr); + + LM_T(LmtJexl, ("JEXL expression evaluation result: %s", result_str.c_str())); + return result_str; } From 9dfdccc5085b38c98bee39de4687035acb7adcb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 12 Feb 2024 08:54:04 +0100 Subject: [PATCH 009/104] FIX jexl implementation --- src/lib/common/macroSubstitute.cpp | 39 +++++++++++++++++++++++------- src/lib/common/macroSubstitute.h | 2 +- src/lib/jexl/JexlContext.cpp | 18 +++++++------- src/lib/jexl/JexlManager.cpp | 22 +++++++++++++---- src/lib/ngsiNotify/Notifier.cpp | 13 ++++++---- 5 files changed, 65 insertions(+), 29 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 5c1a98c74d..83526a319e 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -41,8 +41,6 @@ * * Returns the effective string value, taking into account replacements * -* FIXME PR: notFoundDefault is not longer used? -* */ std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault) { @@ -55,13 +53,18 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); + if (!jexlContextP->hasKey(macroName)) + { + return notFoundDefault; + } + return jexlMgr.evaluate(jexlContextP, macroName); } else if (jexlContextP != NULL) { // "Partial replacement" case. In this case, the result is always a string std::string effectiveValue; - if (!macroSubstitute(&effectiveValue, stringValue, jexlContextP, "null")) + if (!macroSubstitute(&effectiveValue, stringValue, jexlContextP, "null", true)) { // error already logged in macroSubstitute, using stringValue itself as failsafe effectiveValue = stringValue; @@ -110,11 +113,28 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon * * stringValueOrNothing - * -* FIXME PR: notFoundDefault no longer used? */ -static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault) +// FIXME PR: inTheMiddle -> raw ? +static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault, bool inTheMiddle) { - return jexlMgr.evaluate(jexlContextP, key); + if (!jexlContextP->hasKey(key)) + { + return notFoundDefault; + } + + std::string s = jexlMgr.evaluate(jexlContextP, key); + + if (inTheMiddle) + { + // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), + // so double quotes have to be be removed + if (s[0] == '"') + { + s = s.substr(1, s.size()-2); + } + } + + return s; /*std::map::iterator iter = replacementsP->find(key); if (iter == replacementsP->end()) { @@ -164,7 +184,8 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st * Date: Mon Jun 19 16:33:29 2017 +0200 * */ -bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexlContextP, const std::string& notFoundDefault) +// FIXME PR: inTheMiddle -> raw ? +bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexlContextP, const std::string& notFoundDefault, bool inTheMiddle) { // Initial size check: is the string to convert too big? // @@ -227,7 +248,7 @@ bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexl // The +3 is due to "${" and "}" toReduce += (macroName.length() + 3) * times; - toAdd += stringValueOrNothing(jexlContextP, macroName, notFoundDefault).length() * times; + toAdd += stringValueOrNothing(jexlContextP, macroName, notFoundDefault, inTheMiddle).length() * times; } if (from.length() + toAdd - toReduce > outReqMsgMaxSize) @@ -245,7 +266,7 @@ bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexl unsigned int times = it->second; std::string macro = "${" + macroName + "}"; - std::string value = stringValueOrNothing(jexlContextP, macroName, notFoundDefault); + std::string value = stringValueOrNothing(jexlContextP, macroName, notFoundDefault, inTheMiddle); // We have to do the replace operation as many times as macro occurrences for (unsigned int ix = 0; ix < times; ix++) diff --git a/src/lib/common/macroSubstitute.h b/src/lib/common/macroSubstitute.h index 0483fc23ae..cdb66ceaf7 100644 --- a/src/lib/common/macroSubstitute.h +++ b/src/lib/common/macroSubstitute.h @@ -58,6 +58,6 @@ extern std::string smartStringValue(const std::string stringValue, JexlContext* * macroSubstitute - * */ -extern bool macroSubstitute(std::string* sP, const std::string& in, JexlContext* jexlContextP, const std::string& notFoundDefault); +extern bool macroSubstitute(std::string* sP, const std::string& in, JexlContext* jexlContextP, const std::string& notFoundDefault, bool inTheMiddle = false); #endif // SRC_LIB_COMMON_MACROSUBSTITUTE_H_ diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index a34f6962c4..5e01dc951f 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -53,27 +53,27 @@ JexlContext::JexlContext PyObject* value; LM_T(LmtJexl, ("adding to JEXL context: id=%s", id.c_str())); - value = Py_BuildValue("s", id.c_str()); + value = Py_BuildValue("s", ("\"" + id + "\"").c_str()); PyDict_SetItemString(jexl_context, "id", value); Py_DECREF(value); - LM_T(LmtJexl, ("adding to JEXL context: type=%s", type.c_str())); - value = Py_BuildValue("s", type.c_str()); + LM_T(LmtJexl, ("adding to JEXL context: type=\"%s\"", type.c_str())); + value = Py_BuildValue("s", ("\"" + type + "\"").c_str()); PyDict_SetItemString(jexl_context, "type", value); Py_DECREF(value); - LM_T(LmtJexl, ("adding to JEXL context: service=%s", service.c_str())); - value = Py_BuildValue("s", service.c_str()); + LM_T(LmtJexl, ("adding to JEXL context: service=\"%s\"", service.c_str())); + value = Py_BuildValue("s", ("\"" + service + "\"").c_str()); PyDict_SetItemString(jexl_context, "service", value); Py_DECREF(value); - LM_T(LmtJexl, ("adding to JEXL context: servicePath=%s", servicePath.c_str())); - value = Py_BuildValue("s", servicePath.c_str()); + LM_T(LmtJexl, ("adding to JEXL context: servicePath=\"%s\"", servicePath.c_str())); + value = Py_BuildValue("s", ("\"" + servicePath + "\"").c_str()); PyDict_SetItemString(jexl_context, "servicePath", value); Py_DECREF(value); - LM_T(LmtJexl, ("adding to JEXL context: token=%s", token.c_str())); - value = Py_BuildValue("s", token.c_str()); + LM_T(LmtJexl, ("adding to JEXL context: token=\"%s\"", token.c_str())); + value = Py_BuildValue("s", ("\"" + token + "\"").c_str()); PyDict_SetItemString(jexl_context, "authToken", value); Py_DECREF(value); } diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 20dd12864f..462e192c30 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -108,12 +108,12 @@ void JexlManager::init(void) */ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { - LM_T(LmtJexl, ("evaluating JEXL expresion: %s", _expression.c_str())); + LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { - // FIXME PR: grab error message from Python stack + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } @@ -122,7 +122,7 @@ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& Py_XDECREF(expression); if (result == NULL) { - // FIXME PR: grab error message from Python stack + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } @@ -131,7 +131,7 @@ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& Py_XDECREF(result); if (repr == NULL) { - // FIXME PR: grab error message from Python stack + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } @@ -139,7 +139,19 @@ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& std::string result_str = std::string(PyUnicode_AsUTF8(repr)); Py_XDECREF(repr); - LM_T(LmtJexl, ("JEXL expression evaluation result: %s", result_str.c_str())); + // Check if the string has quotes + /*if ((result_str.front() == '\'') && (result_str.back() == '\'')) + { + // Erase the first and last characters (quotes) + result_str.erase(0, 1); // Erase the first character + result_str.erase(result_str.size() - 1); // Erase the last character + }*/ + if (result_str[0] == '\'') + { + result_str = result_str.substr(1, result_str.size()-2); + } + + LM_T(LmtJexl, ("JEXL expression evaluation result: <%s>", result_str.c_str())); return result_str; } diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index f44d4addf8..1a8f19a58f 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -170,7 +170,7 @@ static bool setPayload } else { - if (!macroSubstitute(payloadP, notifPayload, jexlContextP, "")) + if (!macroSubstitute(payloadP, notifPayload, jexlContextP, "", true)) { return false; } @@ -211,6 +211,8 @@ static bool setJsonPayload * * Entity id and type are special. Different from a attribute, they are always * strings and cannot take a number, boolean, etc. as value. +* +* FIXME PR: duplicated logic! (both " and ' in different points of the code) */ inline std::string removeQuotes(std::string s) { @@ -341,6 +343,7 @@ static SenderThreadParams* buildSenderParamsCustom JexlContext jexlContext(en.id, en.type, tenant, en.servicePath, xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { + // FIXME PR: this works with every attribute type? (number, bool, etc.) jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); } @@ -368,7 +371,7 @@ static SenderThreadParams* buildSenderParamsCustom // 2. URL // std::string notifUrl = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.url : notification.mqttInfo.url); - if (macroSubstitute(&url, notifUrl, &jexlContext, "") == false) + if (macroSubstitute(&url, notifUrl, &jexlContext, "", true) == false) { // Warning already logged in macroSubstitute() return NULL; @@ -419,7 +422,7 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &jexlContext, "") == false) || (macroSubstitute(&value, it->second, &jexlContext, "") == false)) + if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) { // Warning already logged in macroSubstitute() return NULL; @@ -445,7 +448,7 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &jexlContext, "") == false) || (macroSubstitute(&value, it->second, &jexlContext, "") == false)) + if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) { // Warning already logged in macroSubstitute() return NULL; @@ -510,7 +513,7 @@ static SenderThreadParams* buildSenderParamsCustom // 8. Topic (only in the case of MQTT notifications) if (notification.type == ngsiv2::MqttNotification) { - if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "") == false) + if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "", true) == false) { // Warning already logged in macroSubstitute() return NULL; From 726dc9c17aa8906a682f4af3dfb4a7f8a7e9991a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 12 Feb 2024 10:16:58 +0100 Subject: [PATCH 010/104] FIX add pyjexl to CI image --- ci/deb/build-dep.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/deb/build-dep.sh b/ci/deb/build-dep.sh index ea8531172a..4d35fe450e 100755 --- a/ci/deb/build-dep.sh +++ b/ci/deb/build-dep.sh @@ -55,6 +55,7 @@ echo "INSTALL: MongoDB shell" \ && apt-get -y update \ && apt-get -y install mongodb-mongosh +# pyjexl is needed by Context Broker functionality, the others are for ftest suite echo "INSTALL: python special dependencies" \ && cd /opt \ && python3 -m venv /opt/ft_env \ @@ -63,6 +64,7 @@ echo "INSTALL: python special dependencies" \ && pip install Werkzeug==2.0.2 \ && pip install paho-mqtt==1.6.1 \ && pip install amqtt==0.11.0b1 \ +&& pip install pyjexl==0.3.0 \ && deactivate # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0 From 904c9652c1424bd63a2b5ba7b80a38938ffd4aad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 09:32:23 +0100 Subject: [PATCH 011/104] FIX comments in Dockerfiles --- docker/Dockerfile | 1 + docker/Dockerfile.alpine | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5b80157d4..a54f4844f4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -88,6 +88,7 @@ RUN \ cd cmake-build && \ # Different from ci/deb/build-dep.sh and build from source documentation, we add here also # the MONGOC_TEST_USE_CRYPT_SHARED=FALSE. It needs Python and in this tiny image we don't have it + # FIXME PR: now that Python is going to be include for JEXL functionalty, maybe this should be changed... cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DMONGOC_TEST_USE_CRYPT_SHARED=FALSE .. && \ make && \ make install && \ diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 524335f677..b871d42a5d 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -93,6 +93,7 @@ RUN \ cd cmake-build && \ # Different from ci/deb/build-dep.sh and build from source documentation, we add here also # the MONGOC_TEST_USE_CRYPT_SHARED=FALSE. It needs Python and in this tiny image we don't have it + # FIXME PR: now that Python is going to be include for JEXL functionalty, maybe this should be changed... cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DMONGOC_TEST_USE_CRYPT_SHARED=FALSE .. && \ make && \ make install && \ From ae88c6e93694e2a2ab0e76fcc114252aa5479d80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 09:34:52 +0100 Subject: [PATCH 012/104] ADD JexlResult class (still wip) --- src/lib/common/macroSubstitute.cpp | 20 +++--- src/lib/jexl/CMakeLists.txt | 2 + src/lib/jexl/JexlContext.cpp | 81 +++++++++++++++-------- src/lib/jexl/JexlContext.h | 3 + src/lib/jexl/JexlManager.cpp | 101 ++++++++++++++++++++++------- src/lib/jexl/JexlManager.h | 7 +- src/lib/jexl/JexlResult.cpp | 63 ++++++++++++++++++ src/lib/jexl/JexlResult.h | 52 +++++++++++++++ src/lib/ngsi/ContextAttribute.cpp | 48 ++++++++++++++ src/lib/ngsi/ContextAttribute.h | 2 + src/lib/ngsiNotify/Notifier.cpp | 3 +- 11 files changed, 319 insertions(+), 63 deletions(-) create mode 100644 src/lib/jexl/JexlResult.cpp create mode 100644 src/lib/jexl/JexlResult.h diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 83526a319e..530b7f4b3a 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -53,12 +53,12 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - if (!jexlContextP->hasKey(macroName)) - { - return notFoundDefault; - } + //if (!jexlContextP->hasKey(macroName)) + //{ + // return notFoundDefault; + //} - return jexlMgr.evaluate(jexlContextP, macroName); + return jexlMgr.evaluate(jexlContextP, macroName).toString(); } else if (jexlContextP != NULL) { @@ -117,12 +117,12 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // FIXME PR: inTheMiddle -> raw ? static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault, bool inTheMiddle) { - if (!jexlContextP->hasKey(key)) - { - return notFoundDefault; - } + //if (!jexlContextP->hasKey(key)) + //{ + // return notFoundDefault; + //} - std::string s = jexlMgr.evaluate(jexlContextP, key); + std::string s = jexlMgr.evaluate(jexlContextP, key).toString(); if (inTheMiddle) { diff --git a/src/lib/jexl/CMakeLists.txt b/src/lib/jexl/CMakeLists.txt index abacbeb3ca..a2ea7eb6e2 100644 --- a/src/lib/jexl/CMakeLists.txt +++ b/src/lib/jexl/CMakeLists.txt @@ -23,12 +23,14 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES JexlManager.cpp JexlContext.cpp + JexlResult.cpp jexlMgr.cpp ) SET (HEADERS JexlManager.h JexlContext.h + JexlResult.h jexlMgr.h ) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index 5e01dc951f..d0c1b6f51f 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -50,35 +50,15 @@ JexlContext::JexlContext // FIXME PR: error control } - PyObject* value; - - LM_T(LmtJexl, ("adding to JEXL context: id=%s", id.c_str())); - value = Py_BuildValue("s", ("\"" + id + "\"").c_str()); - PyDict_SetItemString(jexl_context, "id", value); - Py_DECREF(value); - - LM_T(LmtJexl, ("adding to JEXL context: type=\"%s\"", type.c_str())); - value = Py_BuildValue("s", ("\"" + type + "\"").c_str()); - PyDict_SetItemString(jexl_context, "type", value); - Py_DECREF(value); - - LM_T(LmtJexl, ("adding to JEXL context: service=\"%s\"", service.c_str())); - value = Py_BuildValue("s", ("\"" + service + "\"").c_str()); - PyDict_SetItemString(jexl_context, "service", value); - Py_DECREF(value); - - LM_T(LmtJexl, ("adding to JEXL context: servicePath=\"%s\"", servicePath.c_str())); - value = Py_BuildValue("s", ("\"" + servicePath + "\"").c_str()); - PyDict_SetItemString(jexl_context, "servicePath", value); - Py_DECREF(value); - - LM_T(LmtJexl, ("adding to JEXL context: token=\"%s\"", token.c_str())); - value = Py_BuildValue("s", ("\"" + token + "\"").c_str()); - PyDict_SetItemString(jexl_context, "authToken", value); - Py_DECREF(value); + add("id", id); + add("type", type); + add("service", service); + add("servicePath", servicePath); + add("authToken", token); } + /* **************************************************************************** * * JexlContext::get - @@ -96,7 +76,7 @@ PyObject* JexlContext::get(void) */ void JexlContext::add(const std::string& key, const std::string& _value) { - LM_T(LmtJexl, ("adding to JEXL context: %s=%s", key.c_str(), _value.c_str())); + LM_T(LmtJexl, ("adding to JEXL context (string): %s=%s", key.c_str(), _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); PyDict_SetItemString(jexl_context, key.c_str(), value); Py_DECREF(value); @@ -104,6 +84,52 @@ void JexlContext::add(const std::string& key, const std::string& _value) +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, double _value) +{ + LM_T(LmtJexl, ("adding to JEXL context (double): %s=%f", key.c_str(), _value)); + PyObject* value = Py_BuildValue("d", _value); + PyDict_SetItemString(jexl_context, key.c_str(), value); + Py_DECREF(value); +} + + + + +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, bool _value) +{ + LM_T(LmtJexl, ("adding to JEXL context (bool): %s=%s", key.c_str(), _value? "true" : "false")); + if (_value) + { + PyDict_SetItemString(jexl_context, key.c_str(), Py_True); + } + else + { + PyDict_SetItemString(jexl_context, key.c_str(), Py_False); + } +} + + + +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key) +{ + LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyDict_SetItemString(jexl_context, key.c_str(), Py_None); +} + + + /* **************************************************************************** * * JexlContext::hasKey - @@ -126,5 +152,6 @@ bool JexlContext::hasKey(const std::string& key) */ JexlContext::~JexlContext() { + // FIXME PR: this is not correct. Recursively release of the dict object Py_XDECREF(jexl_context); } \ No newline at end of file diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index 77d6083c07..0eb1f987da 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -54,6 +54,9 @@ class JexlContext PyObject* get(void); void add(const std::string& key, const std::string& value); + void add(const std::string& key, double value); + void add(const std::string& key, bool value); + void add(const std::string& key); bool hasKey(const std::string& key); }; diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 462e192c30..9de0ee88c5 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -26,8 +26,11 @@ #include #include "jexl/JexlManager.h" +#include "jexl/JexlResult.h" #include "logMsg/logMsg.h" +#include "orionTypes/OrionValueType.h" + /* **************************************************************************** @@ -65,6 +68,36 @@ static const char* capturePythonError() } +/* **************************************************************************** +* +* getPyObjectType - +* +* FIXME PR: ValueTypeVector and ValueTypeObject should be taken into account +*/ +static orion::ValueType getPyObjectType(PyObject* obj) +{ + // PyBool_Check() has to be donde before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be + // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 + if (PyBool_Check(obj)) + { + return orion::ValueTypeBoolean; + } + else if (PyLong_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyFloat_Check(obj)) + { + return orion::ValueTypeNumber; + } + else + { + // For other types (including dict and list) we use string as failsafe + return orion::ValueTypeString; + } +} + + /* **************************************************************************** * @@ -106,16 +139,21 @@ void JexlManager::init(void) * * JexlManager::evaluate - */ -std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) +JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); + JexlResult r; + + // If nothing changes, the returned value is null + r.valueType = orion::ValueTypeNull; + PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); - return "null"; + return r; } PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, jexlContextP->get()); @@ -124,35 +162,54 @@ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& { // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); - return "null"; + return r; } - PyObject* repr = PyObject_Repr(result); - Py_XDECREF(result); - if (repr == NULL) + // Special case: expresion evalutes to None + if (result == Py_None) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); - return "null"; + Py_XDECREF(result); + r.valueType = orion::ValueTypeNull; + LM_T(LmtJexl, ("JEXL evaluation result is null")); + return r; } - std::string result_str = std::string(PyUnicode_AsUTF8(repr)); - Py_XDECREF(repr); - - // Check if the string has quotes - /*if ((result_str.front() == '\'') && (result_str.back() == '\'')) + // Other not null types + r.valueType = getPyObjectType(result); + if (r.valueType == orion::ValueTypeNumber) { - // Erase the first and last characters (quotes) - result_str.erase(0, 1); // Erase the first character - result_str.erase(result_str.size() - 1); // Erase the last character - }*/ - if (result_str[0] == '\'') + r.numberValue = PyFloat_AsDouble(result); + LM_T(LmtJexl, ("JEXL evaluation result (double): %f", r.numberValue)); + Py_XDECREF(result); + } + else if (r.valueType == orion::ValueTypeBoolean) { - result_str = result_str.substr(1, result_str.size()-2); + r.boolValue = PyObject_IsTrue(result); + LM_T(LmtJexl, ("JEXL evaluation result (bool): %s", r.boolValue ? "true": "false")); + Py_XDECREF(result); } + else if (r.valueType == orion::ValueTypeString) + { + const char* repr = PyUnicode_AsUTF8(result); + Py_XDECREF(result); + if (repr == NULL) + { + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? + LM_T(LmtJexl, ("error evaluating expression, result is null")); + } + else + { + r.stringValue = std::string(repr); + LM_T(LmtJexl, ("JEXL evaluation result (string): %s", r.stringValue.c_str())); + } + } + + //if (result_str[0] == '\'') + //{ + // result_str = result_str.substr(1, result_str.size()-2); + //} - LM_T(LmtJexl, ("JEXL expression evaluation result: <%s>", result_str.c_str())); - return result_str; + return r; } diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h index 89cadcf534..ad83fd56ea 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/jexl/JexlManager.h @@ -30,6 +30,7 @@ #include #include "jexl/JexlContext.h" +#include "jexl/JexlResult.h" /* **************************************************************************** * @@ -43,9 +44,9 @@ class JexlManager sem_t sem; public: - void init(void); - std::string evaluate(JexlContext* jexlContextP, const std::string& expression); - void release(void); + void init(void); + JexlResult evaluate(JexlContext* jexlContextP, const std::string& expression); + void release(void); }; #endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp new file mode 100644 index 0000000000..654c16582b --- /dev/null +++ b/src/lib/jexl/JexlResult.cpp @@ -0,0 +1,63 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "jexl/JexlResult.h" + +#include "common/string.h" +#include "common/JsonHelper.h" +#include "logMsg/logMsg.h" + +/* **************************************************************************** +* +* toString - +* +* Pretty similar to ContextAttribute::toJsonValue() +* +* FIXME PR: ValueTypeVector and ValueTypeObject should be taken into account +*/ +std::string JexlResult::toString(void) +{ + if (valueType == orion::ValueTypeNumber) + { + return double2string(numberValue); + } + else if (valueType == orion::ValueTypeBoolean) + { + return boolValue ? "true" : "false"; + } + else if (valueType == orion::ValueTypeString) + { + return "\"" + toJsonString(stringValue) + "\""; + } + else if (valueType == orion::ValueTypeNull) + { + return "null"; + } + else + { + LM_E(("Runtime Error (not allowed type in JexlResult: %s)", valueTypeName(valueType))); + return ""; + } +} \ No newline at end of file diff --git a/src/lib/jexl/JexlResult.h b/src/lib/jexl/JexlResult.h new file mode 100644 index 0000000000..a3e0709820 --- /dev/null +++ b/src/lib/jexl/JexlResult.h @@ -0,0 +1,52 @@ +#ifndef SRC_LIB_JEXL_JEXLRESULT_H_ +#define SRC_LIB_JEXL_JEXLRESULT_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "orionTypes/OrionValueType.h" + +#include + +/* **************************************************************************** +* +* JexlResult - +*/ +class JexlResult +{ +public: + // Similar to the fields used in ContextAttribute.h + + orion::ValueType valueType; // Type of value + std::string stringValue; // "value" as a String + double numberValue; // "value" as a Number + bool boolValue; // "value" as a Boolean + + std::string toString(void); +}; + + + +#endif // #define SRC_LIB_JEXL_JEXLRESULT_H_ diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index f4407a89fc..534a1f93de 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1290,6 +1290,54 @@ std::string ContextAttribute::toJsonAsValue +/* **************************************************************************** +* +* addToContext - +* +* Pretty similar in structure to toJsonValue +*/ +void ContextAttribute::addToContext(JexlContext* jexlContextP) +{ + if (compoundValueP != NULL) + { + // FIXME PR: not sure if this has the proper effect... + jexlContextP->add(name, compoundValueP->toJson()); + } + else if (valueType == orion::ValueTypeNumber) + { + if ((type == DATE_TYPE) || (type == DATE_TYPE_ALT)) + { + jexlContextP->add(name, toJsonString(isodate2str(numberValue))); + } + else // regular number + { + jexlContextP->add(name, numberValue); + } + } + else if (valueType == orion::ValueTypeString) + { + jexlContextP->add(name, toJsonString(stringValue)); + } + else if (valueType == orion::ValueTypeBoolean) + { + jexlContextP->add(name, boolValue); + } + else if (valueType == orion::ValueTypeNull) + { + jexlContextP->add(name); + } + else if (valueType == orion::ValueTypeNotGiven) + { + LM_E(("Runtime Error (value not given for attribute %s)", name.c_str())); + } + else + { + LM_E(("Runtime Error (invalid value type %s for attribute %s)", valueTypeName(valueType), name.c_str())); + } +} + + + /* **************************************************************************** * * ContextAttribute::check - diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index 1ca58a0a98..6b65827d29 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -119,6 +119,8 @@ typedef struct ContextAttribute MimeType* outMimeTypeP, HttpStatusCode* scP); + void addToContext(JexlContext* jexlContextP); + void release(void); std::string getName(void); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 1a8f19a58f..a7fc2ce48b 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -344,7 +344,8 @@ static SenderThreadParams* buildSenderParamsCustom for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { // FIXME PR: this works with every attribute type? (number, bool, etc.) - jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); + //jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); + en.attributeVector[ix]->addToContext(&jexlContext); } // From fb5939d268710023e493982a87a4ac46f04d16f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 11:07:01 +0100 Subject: [PATCH 013/104] ADD new JexlContextList --- src/lib/jexl/JexlContext.cpp | 169 +++++++++++++++++++++++++++++--- src/lib/jexl/JexlContext.h | 34 +++++-- src/lib/jexl/JexlManager.cpp | 42 +++++++- src/lib/jexl/JexlResult.cpp | 4 + src/lib/ngsiNotify/Notifier.cpp | 7 +- 5 files changed, 233 insertions(+), 23 deletions(-) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index d0c1b6f51f..ba5975e6a1 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -23,6 +23,9 @@ * Author: Fermin Galan */ +// FIXME PR: use better names than JexlContext and JexlContextList. Also in LM_T +// FIXME PR: add methods should check error in Py_BuildValue() for NULL + #include #include "logMsg/logMsg.h" @@ -35,26 +38,15 @@ * JexlContext::JexlContext - */ JexlContext::JexlContext -( - const std::string& id, - const std::string& type, - const std::string& service, - const std::string& servicePath, - const std::string& token -) +() { jexl_context = PyDict_New(); if (jexl_context == NULL) { // FIXME PR: error control + // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) } - - add("id", id); - add("type", type); - add("service", service); - add("servicePath", servicePath); - add("authToken", token); } @@ -130,6 +122,32 @@ void JexlContext::add(const std::string& key) +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, JexlContext _jexlContext) +{ + // FIXME PR: implement a toString() method in JexlContext to be used here + //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyDict_SetItemString(jexl_context, key.c_str(), _jexlContext.get()); +} + + + +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, JexlContextList jexlContextList) +{ + // FIXME PR: implement a toString() method in JexlContextList to be used here + //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyDict_SetItemString(jexl_context, key.c_str(), jexlContextList.get()); +} + + + /* **************************************************************************** * * JexlContext::hasKey - @@ -154,4 +172,129 @@ JexlContext::~JexlContext() { // FIXME PR: this is not correct. Recursively release of the dict object Py_XDECREF(jexl_context); +} + + +/* **************************************************************************** +* +* JexlContextList::JexlContextList - +*/ +JexlContextList::JexlContextList() +{ + jexl_context = PyList_New(0); + + if (jexl_context == NULL) + { + // FIXME PR: error control + // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + } +} + + + +/* **************************************************************************** +* +* JexlContextList::get - +*/ +PyObject* JexlContextList::get(void) +{ + return jexl_context; +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(const std::string& _value) +{ + LM_T(LmtJexl, ("adding to JEXL context list (string): %s=", _value.c_str())); + PyObject* value = Py_BuildValue("s", _value.c_str()); + PyList_Append(jexl_context, value); + Py_DECREF(value); +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(double _value) +{ + LM_T(LmtJexl, ("adding to JEXL context list (double): %f", _value)); + PyObject* value = Py_BuildValue("d", _value); + PyList_Append(jexl_context, value); + Py_DECREF(value); +} + + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(bool _value) +{ + LM_T(LmtJexl, ("adding to JEXL context list (bool): %s", _value? "true" : "false")); + if (_value) + { + PyList_Append(jexl_context, Py_True); + } + else + { + PyList_Append(jexl_context, Py_False); + } +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(void) +{ + LM_T(LmtJexl, ("adding to JEXL context list (none)")); + PyList_Append(jexl_context, Py_None); +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(JexlContext _jexlContextP) +{ + // FIXME PR: implement a toString() method in JexlContext to be used here + //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyList_Append(jexl_context, _jexlContextP.get()); +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(JexlContextList _jexlContextList) +{ + // FIXME PR: implement a toString() method in JexlContextList to be used here + //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyList_Append(jexl_context, _jexlContextList.get()); +} + + + +/* **************************************************************************** +* +* JexlContextList::~JexlContextList - +*/ +JexlContextList::~JexlContextList() +{ + // FIXME PR: this is not correct. Recursively release of the list object + Py_XDECREF(jexl_context); } \ No newline at end of file diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index 0eb1f987da..bf343b4e7e 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -26,10 +26,15 @@ * Author: Fermin Galan */ +// FIXME PR: rename this to a better name (ExprContextDict?) + #include #include +//#include "jexl/JexlContextList.h" + +class JexlContextList; // forward declaration /* **************************************************************************** * @@ -43,23 +48,36 @@ class JexlContext public: ~JexlContext(); - JexlContext - ( - const std::string& id, - const std::string& type, - const std::string& service, - const std::string& servicePath, - const std::string& token - ); + JexlContext(); PyObject* get(void); void add(const std::string& key, const std::string& value); void add(const std::string& key, double value); void add(const std::string& key, bool value); void add(const std::string& key); + void add(const std::string& key, JexlContext jexlContext); + void add(const std::string& key, JexlContextList jexlContextList); bool hasKey(const std::string& key); }; +class JexlContextList +{ +private: + PyObject* jexl_context; + +public: + ~JexlContextList(); + + JexlContextList(); + + PyObject* get(void); + void add(const std::string& value); + void add(double value); + void add(bool value); + void add(void); + void add(JexlContext jexlContext); + void add(JexlContextList jexlContextList); +}; #endif // #define SRC_LIB_JEXL_JEXLCONTEXT_H_ diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 9de0ee88c5..e8bd7bf5fb 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -90,9 +90,17 @@ static orion::ValueType getPyObjectType(PyObject* obj) { return orion::ValueTypeNumber; } + else if (PyDict_Check(obj)) + { + return orion::ValueTypeObject; + } + else if (PyList_Check(obj)) + { + return orion::ValueTypeVector; + } else { - // For other types (including dict and list) we use string as failsafe + // For other types we use string (this is also a failsafe for types not being strings) return orion::ValueTypeString; } } @@ -188,6 +196,38 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ LM_T(LmtJexl, ("JEXL evaluation result (bool): %s", r.boolValue ? "true": "false")); Py_XDECREF(result); } + else if (r.valueType == orion::ValueTypeObject) + { + // FIXME PR: same processing than string? Unify? + const char* repr = PyUnicode_AsUTF8(result); + Py_XDECREF(result); + if (repr == NULL) + { + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? + LM_T(LmtJexl, ("error evaluating expression, result is null")); + } + else + { + r.stringValue = std::string(repr); + LM_T(LmtJexl, ("JEXL evaluation result (object): %s", r.stringValue.c_str())); + } + } + else if (r.valueType == orion::ValueTypeVector) + { + // FIXME PR: same processing than string? Unify? + const char* repr = PyUnicode_AsUTF8(result); + Py_XDECREF(result); + if (repr == NULL) + { + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? + LM_T(LmtJexl, ("error evaluating expression, result is null")); + } + else + { + r.stringValue = std::string(repr); + LM_T(LmtJexl, ("JEXL evaluation result (list): %s", r.stringValue.c_str())); + } + } else if (r.valueType == orion::ValueTypeString) { const char* repr = PyUnicode_AsUTF8(result); diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 654c16582b..1197b1491a 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -51,6 +51,10 @@ std::string JexlResult::toString(void) { return "\"" + toJsonString(stringValue) + "\""; } + else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) + { + return toJsonString(stringValue); + } else if (valueType == orion::ValueTypeNull) { return "null"; diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index a7fc2ce48b..24c6953ef0 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -340,7 +340,12 @@ static SenderThreadParams* buildSenderParamsCustom // Used by several macroSubstitute() calls along this function //std::map replacements; //buildReplacementsMap(en, tenant, xauthToken, &replacements); - JexlContext jexlContext(en.id, en.type, tenant, en.servicePath, xauthToken); + JexlContext jexlContext; + jexlContext.add("id", en.id); + jexlContext.add("type", en.type); + jexlContext.add("service", tenant); + jexlContext.add("servicePath", en.servicePath); + jexlContext.add("authToken", xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { // FIXME PR: this works with every attribute type? (number, bool, etc.) From 8a6c762cdb408a9c8800db382932243fcf59254c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 16:48:39 +0100 Subject: [PATCH 014/104] FIX expression evaluation logic --- src/lib/common/macroSubstitute.cpp | 58 ++-- src/lib/jexl/JexlContext.cpp | 10 +- src/lib/jexl/JexlContext.h | 6 +- src/lib/jexl/JexlManager.cpp | 172 +++++++---- src/lib/jexl/JexlManager.h | 6 +- src/lib/jexl/JexlResult.cpp | 2 +- src/lib/ngsi/ContextAttribute.cpp | 10 +- src/lib/ngsiNotify/Notifier.cpp | 12 +- src/lib/parse/CompoundValueNode.cpp | 100 +++++++ src/lib/parse/CompoundValueNode.h | 3 + ...fication_templates_many_notifications.test | 7 +- .../jexl_basic.test | 279 ++++++++++++++++++ 12 files changed, 549 insertions(+), 116 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 530b7f4b3a..10c5ec0251 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -53,12 +53,15 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - //if (!jexlContextP->hasKey(macroName)) - //{ - // return notFoundDefault; - //} - - return jexlMgr.evaluate(jexlContextP, macroName).toString(); + JexlResult r = jexlMgr.evaluate(jexlContextP, macroName); + if (r.valueType == orion::ValueTypeNull) + { + return notFoundDefault; + } + else + { + return r.toString(); + } } else if (jexlContextP != NULL) { @@ -117,44 +120,27 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // FIXME PR: inTheMiddle -> raw ? static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault, bool inTheMiddle) { - //if (!jexlContextP->hasKey(key)) - //{ - // return notFoundDefault; - //} - - std::string s = jexlMgr.evaluate(jexlContextP, key).toString(); + JexlResult r = jexlMgr.evaluate(jexlContextP, key); - if (inTheMiddle) - { - // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), - // so double quotes have to be be removed - if (s[0] == '"') - { - s = s.substr(1, s.size()-2); - } - } - - return s; - /*std::map::iterator iter = replacementsP->find(key); - if (iter == replacementsP->end()) + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } else { - // replacementsP contents are prepared for "full replacement" case, so string values use - // double quotes. But in this case we are in a "partial replacement" case, so we have - // to remove them if we find them - std::string value = iter->second; - if (value[0] == '"') - { - return value.substr(1, value.size()-2); - } - else + std::string s = r.toString(); + if (inTheMiddle) { - return value; + // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), + // so double quotes have to be be removed + if (s[0] == '"') + { + s = s.substr(1, s.size()-2); + } } - }*/ + + return s; + } } diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index ba5975e6a1..82ab5e3dd4 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -166,12 +166,12 @@ bool JexlContext::hasKey(const std::string& key) /* **************************************************************************** * -* JexlContext::~JexlContext - +* JexlContext::release - */ -JexlContext::~JexlContext() +void JexlContext::release(void) { // FIXME PR: this is not correct. Recursively release of the dict object - Py_XDECREF(jexl_context); + //Py_XDECREF(jexl_context); } @@ -291,9 +291,9 @@ void JexlContextList::add(JexlContextList _jexlContextList) /* **************************************************************************** * -* JexlContextList::~JexlContextList - +* JexlContextList::relesase - */ -JexlContextList::~JexlContextList() +void JexlContextList::release(void) { // FIXME PR: this is not correct. Recursively release of the list object Py_XDECREF(jexl_context); diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index bf343b4e7e..4c91fc95a6 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -46,8 +46,6 @@ class JexlContext PyObject* jexl_context; public: - ~JexlContext(); - JexlContext(); PyObject* get(void); @@ -58,6 +56,7 @@ class JexlContext void add(const std::string& key, JexlContext jexlContext); void add(const std::string& key, JexlContextList jexlContextList); bool hasKey(const std::string& key); + void release(void); }; class JexlContextList @@ -66,8 +65,6 @@ class JexlContextList PyObject* jexl_context; public: - ~JexlContextList(); - JexlContextList(); PyObject* get(void); @@ -77,6 +74,7 @@ class JexlContextList void add(void); void add(JexlContext jexlContext); void add(JexlContextList jexlContextList); + void release(void); }; diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index e8bd7bf5fb..f82f1f9e2b 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -71,8 +71,6 @@ static const char* capturePythonError() /* **************************************************************************** * * getPyObjectType - -* -* FIXME PR: ValueTypeVector and ValueTypeObject should be taken into account */ static orion::ValueType getPyObjectType(PyObject* obj) { @@ -107,14 +105,46 @@ static orion::ValueType getPyObjectType(PyObject* obj) +#if 0 +/* **************************************************************************** +* +* customJsonSerializerFunction - +* +* To be used in the json.dumps() step in JexlManager::evaluate() in order to get +* integer numbers when possible (i.e. "2" instead of "2.0"), this way behaving as +* the pre-JEXL macro replacement logic +*/ +static PyObject* customJsonSerializerFunction(PyObject* obj) +{ + // Check if the object is a float + if (PyFloat_Check(obj)) + { + // Convert float to integer if it represents a whole number + double value = PyFloat_AsDouble(obj); + if (value == (int)value) + { + return PyLong_FromDouble(value); + } + } + + // Return the original object + Py_INCREF(obj); + return obj; +} +#endif + + + /* **************************************************************************** * * JexlManager::init - */ void JexlManager::init(void) { - jexl_module = NULL; - jexl_engine = NULL; + pyjexlModule = NULL; + jsonModule = NULL; + jexlEngine = NULL; + //customJsonSerializer = NULL; if (sem_init(&sem, 0, 1) == -1) { @@ -124,19 +154,37 @@ void JexlManager::init(void) Py_Initialize(); LM_T(LmtJexl, ("Python interpreter has been initialized")); - jexl_module = PyImport_ImportModule("pyjexl"); - if (jexl_module == NULL) + pyjexlModule = PyImport_ImportModule("pyjexl"); + if (pyjexlModule == NULL) + { + const char* error = capturePythonError(); + LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", error)); + } + LM_T(LmtJexl, ("pyjexl module has been loaded")); + + jsonModule = PyImport_ImportModule("json"); + if (jsonModule == NULL) + { + const char* error = capturePythonError(); + LM_X(1, ("Fatal Error (error importing json module: %s)", error)); + } + LM_T(LmtJexl, ("json module has been loaded")); + +#if 0 + customJsonSerializer = PyCapsule_New((void*)customJsonSerializerFunction, NULL, NULL); + if (customJsonSerializer == NULL) { const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error importing jexl_module: %s)", error)); + LM_X(1, ("Fatal Error (error creating json custom serializer function: %s)", error)); } - LM_T(LmtJexl, ("jexl module has been loaded")); + LM_T(LmtJexl, ("json custom serializer has been createdmodule has been loaded")); +#endif - jexl_engine = PyObject_CallMethod(jexl_module, "JEXL", NULL); - if (jexl_engine == NULL) + jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); + if (jexlEngine == NULL) { const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error creating jexl_engine: %s)", error)); + LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", error)); } LM_T(LmtJexl, ("jexl engine has been created")); } @@ -153,32 +201,32 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ JexlResult r; - // If nothing changes, the returned value is null + // If nothing changes, the returned value would be null (failsafe) r.valueType = orion::ValueTypeNull; PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error building expression: %s", capturePythonError())); return r; } - PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, jexlContextP->get()); + PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, jexlContextP->get()); Py_XDECREF(expression); if (result == NULL) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error evaluating expression: %s", capturePythonError())); return r; } // Special case: expresion evalutes to None if (result == Py_None) { - Py_XDECREF(result); - r.valueType = orion::ValueTypeNull; LM_T(LmtJexl, ("JEXL evaluation result is null")); + r.valueType = orion::ValueTypeNull; + Py_XDECREF(result); return r; } @@ -196,59 +244,53 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ LM_T(LmtJexl, ("JEXL evaluation result (bool): %s", r.boolValue ? "true": "false")); Py_XDECREF(result); } - else if (r.valueType == orion::ValueTypeObject) - { - // FIXME PR: same processing than string? Unify? - const char* repr = PyUnicode_AsUTF8(result); - Py_XDECREF(result); - if (repr == NULL) - { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); - } - else - { - r.stringValue = std::string(repr); - LM_T(LmtJexl, ("JEXL evaluation result (object): %s", r.stringValue.c_str())); - } - } - else if (r.valueType == orion::ValueTypeVector) + else if ((r.valueType == orion::ValueTypeObject) || (r.valueType == orion::ValueTypeVector)) { - // FIXME PR: same processing than string? Unify? - const char* repr = PyUnicode_AsUTF8(result); + // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot + // be used, as it produce string with ' instead of " in the resulting JSON string. + // FIXME PR: is the customJsonSerializer going to be used at the end? + PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); Py_XDECREF(result); if (repr == NULL) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); + r.valueType = orion::ValueTypeNull; } else { - r.stringValue = std::string(repr); - LM_T(LmtJexl, ("JEXL evaluation result (list): %s", r.stringValue.c_str())); + const char* str = PyUnicode_AsUTF8(repr); + Py_XDECREF(repr); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); + r.valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtJexl, ("JEXL evaluation result (object or vector): %s", str)); + r.stringValue = std::string(str); + } } } else if (r.valueType == orion::ValueTypeString) { - const char* repr = PyUnicode_AsUTF8(result); + const char* str = PyUnicode_AsUTF8(result); Py_XDECREF(result); - if (repr == NULL) + if (str == NULL) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + r.valueType = orion::ValueTypeNull; } else { - r.stringValue = std::string(repr); - LM_T(LmtJexl, ("JEXL evaluation result (string): %s", r.stringValue.c_str())); + LM_T(LmtJexl, ("JEXL evaluation result (string): %s", str)); + r.stringValue = std::string(str); } } - //if (result_str[0] == '\'') - //{ - // result_str = result_str.substr(1, result_str.size()-2); - //} - return r; } @@ -260,16 +302,30 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ */ void JexlManager::release(void) { - if (jexl_engine != NULL) + if (jexlEngine != NULL) { - Py_XDECREF(jexl_engine); + Py_XDECREF(jexlEngine); LM_T(LmtJexl, ("jexl engine has been freed")); } - if (jexl_module != NULL) + if (pyjexlModule != NULL) + { + Py_XDECREF(pyjexlModule); + LM_T(LmtJexl, ("pyjexl module has been freed")); + } + +#if 0 + if (customJsonSerializer != NULL) + { + Py_XDECREF(customJsonSerializer); + LM_T(LmtJexl, ("json custom serializer module has been freed")); + } +#endif + + if (jsonModule != NULL) { - Py_XDECREF(jexl_module); - LM_T(LmtJexl, ("jexl module has been freed")); + Py_XDECREF(jsonModule); + LM_T(LmtJexl, ("json module has been freed")); } Py_Finalize(); diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h index ad83fd56ea..adce908c7c 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/jexl/JexlManager.h @@ -39,8 +39,10 @@ class JexlManager { private: - PyObject* jexl_module; - PyObject* jexl_engine; + PyObject* pyjexlModule; + PyObject* jexlEngine; + PyObject* jsonModule; + //PyObject* customJsonSerializer; sem_t sem; public: diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 1197b1491a..2769844403 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -53,7 +53,7 @@ std::string JexlResult::toString(void) } else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) { - return toJsonString(stringValue); + return stringValue; } else if (valueType == orion::ValueTypeNull) { diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 534a1f93de..725e159ed9 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1300,8 +1300,14 @@ void ContextAttribute::addToContext(JexlContext* jexlContextP) { if (compoundValueP != NULL) { - // FIXME PR: not sure if this has the proper effect... - jexlContextP->add(name, compoundValueP->toJson()); + if (valueType == orion::ValueTypeObject) + { + jexlContextP->add(name, compoundValueP->toJexlContext()); + } + else // valueType == orion::ValueTypeVector + { + jexlContextP->add(name, compoundValueP->toJexlContextList()); + } } else if (valueType == orion::ValueTypeNumber) { diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 24c6953ef0..4ab2b5262f 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -338,8 +338,6 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - //std::map replacements; - //buildReplacementsMap(en, tenant, xauthToken, &replacements); JexlContext jexlContext; jexlContext.add("id", en.id); jexlContext.add("type", en.type); @@ -348,8 +346,6 @@ static SenderThreadParams* buildSenderParamsCustom jexlContext.add("authToken", xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - // FIXME PR: this works with every attribute type? (number, bool, etc.) - //jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); en.attributeVector[ix]->addToContext(&jexlContext); } @@ -380,6 +376,7 @@ static SenderThreadParams* buildSenderParamsCustom if (macroSubstitute(&url, notifUrl, &jexlContext, "", true) == false) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } @@ -396,6 +393,7 @@ static SenderThreadParams* buildSenderParamsCustom if (!setPayload(includePayload, notifPayload, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } } @@ -412,6 +410,7 @@ static SenderThreadParams* buildSenderParamsCustom if (!setNgsiPayload(ngsi, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } mimeType = "application/json"; @@ -431,6 +430,7 @@ static SenderThreadParams* buildSenderParamsCustom if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } @@ -457,6 +457,7 @@ static SenderThreadParams* buildSenderParamsCustom if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } @@ -488,6 +489,7 @@ static SenderThreadParams* buildSenderParamsCustom if (!parseUrl(url, host, port, uriPath, protocol)) { LM_E(("Runtime Error (not sending notification: malformed URL: '%s')", url.c_str())); + jexlContext.release(); return NULL; } @@ -522,6 +524,7 @@ static SenderThreadParams* buildSenderParamsCustom if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "", true) == false) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } } @@ -557,6 +560,7 @@ static SenderThreadParams* buildSenderParamsCustom snprintf(suffix, sizeof(suffix), "%u", correlatorCounter); paramsP->fiwareCorrelator = fiwareCorrelator + "; cbnotif=" + suffix; + jexlContext.release(); return paramsP; } diff --git a/src/lib/parse/CompoundValueNode.cpp b/src/lib/parse/CompoundValueNode.cpp index 6dfb6f59d4..f9e45c1555 100644 --- a/src/lib/parse/CompoundValueNode.cpp +++ b/src/lib/parse/CompoundValueNode.cpp @@ -658,6 +658,106 @@ std::string CompoundValueNode::toJson(JexlContext* jexlContextP) +/* **************************************************************************** +* +* CompoundValueNode:toJexlContext +* +*/ +JexlContext CompoundValueNode::toJexlContext(void) +{ + JexlContext jc; + for (uint64_t ix = 0; ix < childV.size(); ++ix) + { + CompoundValueNode* child = childV[ix]; + switch (child->valueType) + { + case orion::ValueTypeString: + jc.add(child->name, child->stringValue); + break; + + case orion::ValueTypeNumber: + jc.add(child->name, child->numberValue); + break; + + case orion::ValueTypeBoolean: + jc.add(child->name, child->boolValue); + break; + + case orion::ValueTypeNull: + jc.add(child->name); + break; + + case orion::ValueTypeVector: + jc.add(child->name, child->toJexlContextList()); + break; + + case orion::ValueTypeObject: + jc.add(child->name, child->toJexlContext()); + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given (%s))", name.c_str())); + break; + + default: + LM_E(("Runtime Error (value type unknown (%s))", name.c_str())); + } + } + return jc; +} + + + +/* **************************************************************************** +* +* CompoundValueNode:toJexlContextList +* +*/ +JexlContextList CompoundValueNode::toJexlContextList(void) +{ + JexlContextList jcl; + for (uint64_t ix = 0; ix < childV.size(); ++ix) + { + CompoundValueNode* child = childV[ix]; + switch (child->valueType) + { + case orion::ValueTypeString: + jcl.add(child->stringValue); + break; + + case orion::ValueTypeNumber: + jcl.add(child->numberValue); + break; + + case orion::ValueTypeBoolean: + jcl.add(child->boolValue); + break; + + case orion::ValueTypeNull: + jcl.add(); + break; + + case orion::ValueTypeVector: + jcl.add(child->toJexlContextList()); + break; + + case orion::ValueTypeObject: + jcl.add(child->toJexlContext()); + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given)")); + break; + + default: + LM_E(("Runtime Error (value type unknown)")); + } + } + return jcl; +} + + + /* **************************************************************************** * * clone - diff --git a/src/lib/parse/CompoundValueNode.h b/src/lib/parse/CompoundValueNode.h index 12fd070a31..a77eb1a393 100644 --- a/src/lib/parse/CompoundValueNode.h +++ b/src/lib/parse/CompoundValueNode.h @@ -121,6 +121,9 @@ class CompoundValueNode std::string toJson(JexlContext* jexlContextP = NULL); + JexlContext toJexlContext(void); + JexlContextList toJexlContextList(void); + void shortShow(const std::string& indent); void show(const std::string& indent); diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test index 67f39b8667..341c55b7b9 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test @@ -348,22 +348,21 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a2=null&id=E1&type=T2 +PUT http://127.0.0.1:REGEX(\d+)/notify?id=E1&type=T2 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 32 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json -A2: null Entity-Type: T2 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { "A1": "", - "A2": "null", + "A2": "", "A3": "" } #SORT_END diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test new file mode 100644 index 0000000000..209b61474c --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test @@ -0,0 +1,279 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression sum: A+B +# 02. Create entity E1 with A=1 and B=2 +# 03. Update entity E1 with A=foo and B=bar +# 04. Update entity E1 with A=2.1 and B=-3.8 +# 05. Dump accumulator and see two notifications (sum: 3 and sum: foobar) +# + + +echo "01. Create custom sub with custom expression sum: A+B" +echo "=====================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "sum": { + "value": "${A+B}", + "type": "Calculated" + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1 and B=2" +echo "=====================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A=foo and B=bar" +echo "=========================================" +payload='{ + "A": { + "value": "foo", + "type": "Text" + }, + "B": { + "value": "bar", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Update entity E1 with A=2.1 and B=-3.8" +echo "==========================================" +payload='{ + "A": { + "value": 2.1, + "type": "Number" + }, + "B": { + "value": -3.8, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1,7)" +echo "=================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: A+B +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 and B=2 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A=foo and B=bar +========================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Update entity E1 with A=2.1 and B=-3.8 +========================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1,7) +================================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 221 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 232 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Text", + "value": "foo" + }, + "B": { + "metadata": {}, + "type": "Text", + "value": "bar" + }, + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": "foobar" + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 229 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2.1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": -3.8 + }, + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": -1.7 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 7231ad45a0f942aee4f2c1aab9648c36bef786bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 18:58:27 +0100 Subject: [PATCH 015/104] FIX value not given processing logic --- src/lib/common/macroSubstitute.cpp | 4 +-- src/lib/jexl/JexlManager.cpp | 4 +-- ...fication_templates_many_notifications.test | 33 +++++++++++-------- .../covered_custom_notification.test | 4 +-- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 10c5ec0251..fd6efea913 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -54,7 +54,7 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon std::string macroName = stringValue.substr(2, stringValue.size() - 3); JexlResult r = jexlMgr.evaluate(jexlContextP, macroName); - if (r.valueType == orion::ValueTypeNull) + if (r.valueType == orion::ValueTypeNotGiven) { return notFoundDefault; } @@ -122,7 +122,7 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st { JexlResult r = jexlMgr.evaluate(jexlContextP, key); - if (r.valueType == orion::ValueTypeNull) + if (r.valueType == orion::ValueTypeNotGiven) { return notFoundDefault; } diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index f82f1f9e2b..286b3e6260 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -201,8 +201,8 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ JexlResult r; - // If nothing changes, the returned value would be null (failsafe) - r.valueType = orion::ValueTypeNull; + // If nothing changes, the returned value would be ValueTypeNotGiven + r.valueType = orion::ValueTypeNotGiven; PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test index 341c55b7b9..774a462d13 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test @@ -277,23 +277,25 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&a2=null&a3=null&id=E1&type=T1 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 44 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json A1: true +A2: null +A3: null Entity-Type: T1 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { "A1": "true", - "A2": "", - "A3": "" + "A2": "null", + "A3": "null" } #SORT_END #SORT_START @@ -348,22 +350,25 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?id=E1&type=T2 +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=null&a2=null&a3=null&id=E1&type=T2 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 32 +Content-Length: 44 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json +A1: null +A2: null +A3: null Entity-Type: T2 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "", - "A2": "", - "A3": "" + "A1": "null", + "A2": "null", + "A3": "null" } #SORT_END #SORT_START @@ -418,11 +423,13 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=null&a2=null&a3=3&id=E1&type=T3 Fiware-Servicepath: / +A1: null +A2: null A3: 3 Entity-Id: E1 -Content-Length: 33 +Content-Length: 41 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -432,8 +439,8 @@ Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "", - "A2": "", + "A1": "null", + "A2": "null", "A3": "3" } ======================================= diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test index 460ffe731a..33c6efe8b2 100644 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test @@ -199,7 +199,7 @@ Content-Length: 0 ======================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 31 +Content-Length: 35 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -207,7 +207,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": }======================================= +{ "A1-value": 1, "A2-value": null }======================================= --TEARDOWN-- From 06e0d98c95c33cb76e7c907ef3522f9c14d80d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 20:02:30 +0100 Subject: [PATCH 016/104] FIX ftest --- .../notification_templates.test | 7 ++++--- .../http_info_for_sub_update.test | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates.test b/test/functionalTest/cases/2015_notification_templates/notification_templates.test index 23df76a2fe..7ff8e92606 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates.test @@ -536,10 +536,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) "A3": "13" } ======================================= -POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&id=E1&step=07&type=Thing +POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&a4=null&id=E1&step=07&type=Thing Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 40 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Step: 07 Ngsiv2-Attrsformat: custom @@ -547,6 +547,7 @@ Host: 127.0.0.1:REGEX(\d+) Accept: application/json A2: a2 A1: a1 +A4: null Entity-Type: Thing Content-Type: user-specified Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) @@ -554,7 +555,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) { "A1": "a1", "A2": "a2", - "A4": "" + "A4": "null" } ======================================= #SORT_END diff --git a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test index 7a4956a172..00a402dfd1 100644 --- a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test +++ b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test @@ -297,7 +297,7 @@ Content-Length: 0 07. Dump accumulator ==================== -PATCH http://localhost:REGEX(\d+)/notify +PATCH http://localhost:REGEX(\d+)/notify?Q2=null Fiware-Servicepath: / Content-Length: 36 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) @@ -306,6 +306,7 @@ Host: localhost:REGEX(\d+) Accept: application/json Content-Type: application/json Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 +H2: null { "What": "PAYLOAD2", From 444b6e241b3a8fda1f0446e6bf32629eb6e8d88a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 15 Feb 2024 11:48:18 +0100 Subject: [PATCH 017/104] Revert "FIX value not given processing logic" This reverts commit 7231ad45a0f942aee4f2c1aab9648c36bef786bb. --- src/lib/common/macroSubstitute.cpp | 4 +-- src/lib/jexl/JexlManager.cpp | 4 +-- ...fication_templates_many_notifications.test | 33 ++++++++----------- .../covered_custom_notification.test | 4 +-- 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index fd6efea913..10c5ec0251 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -54,7 +54,7 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon std::string macroName = stringValue.substr(2, stringValue.size() - 3); JexlResult r = jexlMgr.evaluate(jexlContextP, macroName); - if (r.valueType == orion::ValueTypeNotGiven) + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } @@ -122,7 +122,7 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st { JexlResult r = jexlMgr.evaluate(jexlContextP, key); - if (r.valueType == orion::ValueTypeNotGiven) + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 286b3e6260..f82f1f9e2b 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -201,8 +201,8 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ JexlResult r; - // If nothing changes, the returned value would be ValueTypeNotGiven - r.valueType = orion::ValueTypeNotGiven; + // If nothing changes, the returned value would be null (failsafe) + r.valueType = orion::ValueTypeNull; PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test index 774a462d13..341c55b7b9 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test @@ -277,25 +277,23 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&a2=null&a3=null&id=E1&type=T1 +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 44 +Content-Length: 36 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json A1: true -A2: null -A3: null Entity-Type: T1 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { "A1": "true", - "A2": "null", - "A3": "null" + "A2": "", + "A3": "" } #SORT_END #SORT_START @@ -350,25 +348,22 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=null&a2=null&a3=null&id=E1&type=T2 +PUT http://127.0.0.1:REGEX(\d+)/notify?id=E1&type=T2 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 44 +Content-Length: 32 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json -A1: null -A2: null -A3: null Entity-Type: T2 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "null", - "A2": "null", - "A3": "null" + "A1": "", + "A2": "", + "A3": "" } #SORT_END #SORT_START @@ -423,13 +418,11 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=null&a2=null&a3=3&id=E1&type=T3 +PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 Fiware-Servicepath: / -A1: null -A2: null A3: 3 Entity-Id: E1 -Content-Length: 41 +Content-Length: 33 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -439,8 +432,8 @@ Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "null", - "A2": "null", + "A1": "", + "A2": "", "A3": "3" } ======================================= diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test index 33c6efe8b2..460ffe731a 100644 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test @@ -199,7 +199,7 @@ Content-Length: 0 ======================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 35 +Content-Length: 31 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -207,7 +207,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": null }======================================= +{ "A1-value": 1, "A2-value": }======================================= --TEARDOWN-- From e20b1e3adf0236c6941a1b2ec0fb7e771b41c488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 15 Feb 2024 11:51:47 +0100 Subject: [PATCH 018/104] Revert "FIX ftest" This reverts commit 06e0d98c95c33cb76e7c907ef3522f9c14d80d96. --- .../notification_templates.test | 7 +++---- .../http_info_for_sub_update.test | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates.test b/test/functionalTest/cases/2015_notification_templates/notification_templates.test index 7ff8e92606..23df76a2fe 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates.test @@ -536,10 +536,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) "A3": "13" } ======================================= -POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&a4=null&id=E1&step=07&type=Thing +POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&id=E1&step=07&type=Thing Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 40 +Content-Length: 36 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Step: 07 Ngsiv2-Attrsformat: custom @@ -547,7 +547,6 @@ Host: 127.0.0.1:REGEX(\d+) Accept: application/json A2: a2 A1: a1 -A4: null Entity-Type: Thing Content-Type: user-specified Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) @@ -555,7 +554,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) { "A1": "a1", "A2": "a2", - "A4": "null" + "A4": "" } ======================================= #SORT_END diff --git a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test index 00a402dfd1..7a4956a172 100644 --- a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test +++ b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test @@ -297,7 +297,7 @@ Content-Length: 0 07. Dump accumulator ==================== -PATCH http://localhost:REGEX(\d+)/notify?Q2=null +PATCH http://localhost:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 36 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) @@ -306,7 +306,6 @@ Host: localhost:REGEX(\d+) Accept: application/json Content-Type: application/json Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -H2: null { "What": "PAYLOAD2", From ae3d83b7df3f71d0572b802e4e63ebcbec4054e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 15 Feb 2024 12:00:15 +0100 Subject: [PATCH 019/104] FIX ftest --- .../covered_custom_notification.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test index 460ffe731a..6d40e8e28b 100644 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test @@ -174,7 +174,7 @@ Content-Length: 0 ====================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 35 +Content-Length: 31 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -182,7 +182,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": null }======================================= +{ "A1-value": 1, "A2-value": }======================================= 05. Create E2 with attribute A1=1 From a57599d45a6d32c1f8e84b17c117389c5850fc9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 16 Feb 2024 12:36:00 +0100 Subject: [PATCH 020/104] FIX add CompoundValue to JexlResult --- src/lib/jexl/JexlResult.cpp | 26 ++++++++++++++++++++++++-- src/lib/jexl/JexlResult.h | 8 +++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 2769844403..f4cf5b044d 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -35,7 +35,6 @@ * * Pretty similar to ContextAttribute::toJsonValue() * -* FIXME PR: ValueTypeVector and ValueTypeObject should be taken into account */ std::string JexlResult::toString(void) { @@ -53,7 +52,15 @@ std::string JexlResult::toString(void) } else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) { - return stringValue; + if (compoundValueP != NULL) + { + return compoundValueP->toJson(); + } + else + { + LM_E(("Runtime Error (result is vector/object but compountValue is NULL)")); + return ""; + } } else if (valueType == orion::ValueTypeNull) { @@ -64,4 +71,19 @@ std::string JexlResult::toString(void) LM_E(("Runtime Error (not allowed type in JexlResult: %s)", valueTypeName(valueType))); return ""; } +} + + + +/* **************************************************************************** +* +* JexlResult::release - +*/ +void JexlResult::release(void) +{ + if (compoundValueP != NULL) + { + delete compoundValueP; + compoundValueP = NULL; + } } \ No newline at end of file diff --git a/src/lib/jexl/JexlResult.h b/src/lib/jexl/JexlResult.h index a3e0709820..34c8effdc7 100644 --- a/src/lib/jexl/JexlResult.h +++ b/src/lib/jexl/JexlResult.h @@ -28,6 +28,8 @@ #include "orionTypes/OrionValueType.h" +#include "parse/CompoundValueNode.h" + #include /* **************************************************************************** @@ -44,7 +46,11 @@ class JexlResult double numberValue; // "value" as a Number bool boolValue; // "value" as a Boolean - std::string toString(void); + // Use only when valueType is object or vector + orion::CompoundValueNode* compoundValueP; + + std::string toString(void); + void release(void); }; From 9ea1c8291247dc44467c4c92913fa55c1e2f32a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 16 Feb 2024 16:39:33 +0100 Subject: [PATCH 021/104] FIX JexlResult improvements (wip) --- src/lib/jexl/JexlManager.cpp | 48 ++------- src/lib/jexl/JexlResult.cpp | 202 +++++++++++++++++++++++++++++++++++ src/lib/jexl/JexlResult.h | 6 ++ 3 files changed, 214 insertions(+), 42 deletions(-) diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index f82f1f9e2b..bfb261f944 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -68,42 +68,6 @@ static const char* capturePythonError() } -/* **************************************************************************** -* -* getPyObjectType - -*/ -static orion::ValueType getPyObjectType(PyObject* obj) -{ - // PyBool_Check() has to be donde before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be - // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 - if (PyBool_Check(obj)) - { - return orion::ValueTypeBoolean; - } - else if (PyLong_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyFloat_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyDict_Check(obj)) - { - return orion::ValueTypeObject; - } - else if (PyList_Check(obj)) - { - return orion::ValueTypeVector; - } - else - { - // For other types we use string (this is also a failsafe for types not being strings) - return orion::ValueTypeString; - } -} - - #if 0 /* **************************************************************************** @@ -197,12 +161,9 @@ void JexlManager::init(void) */ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { - LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - JexlResult r; - // If nothing changes, the returned value would be null (failsafe) - r.valueType = orion::ValueTypeNull; + LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) @@ -221,8 +182,11 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ return r; } + r.fill(result); + Py_XDECREF(result); + // Special case: expresion evalutes to None - if (result == Py_None) + /*if (result == Py_None) { LM_T(LmtJexl, ("JEXL evaluation result is null")); r.valueType = orion::ValueTypeNull; @@ -289,7 +253,7 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ LM_T(LmtJexl, ("JEXL evaluation result (string): %s", str)); r.stringValue = std::string(str); } - } + }*/ return r; } diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index f4cf5b044d..d103b4edff 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -29,6 +29,208 @@ #include "common/JsonHelper.h" #include "logMsg/logMsg.h" + + +/* **************************************************************************** +* +* capturePythonError - +* +* FIXME PR: duplicate code. Unify +*/ +static const char* capturePythonError() +{ + if (PyErr_Occurred()) + { + PyObject* ptype; + PyObject* pvalue; + PyObject* ptraceback; + + // Fetch the exception type, value, and traceback + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + + if (pvalue != NULL) + { + PyObject* str_obj = PyObject_Str(pvalue); + const char* error_message = PyUnicode_AsUTF8(str_obj); + + // Release the Python objects + Py_XDECREF(str_obj); + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + + return error_message; + } + } + + PyErr_Clear(); + return ""; +} + + + +/* **************************************************************************** +* +* getPyObjectType - +*/ +static orion::ValueType getPyObjectType(PyObject* obj) +{ + // PyBool_Check() has to be done before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be + // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 + if (PyBool_Check(obj)) + { + return orion::ValueTypeBoolean; + } + else if (PyLong_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyFloat_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyDict_Check(obj)) + { + return orion::ValueTypeObject; + } + else if (PyList_Check(obj)) + { + return orion::ValueTypeVector; + } + else + { + // For other types we use string (this is also a failsafe for types not being strings) + return orion::ValueTypeString; + } +} + + + +/* **************************************************************************** +* +* fill - +* +*/ +void JexlResult::fill(PyObject* result) +{ + // If nothing changes, the returned value would be null (failsafe) + valueType = orion::ValueTypeNull; + + // Special case: expresion evalutes to None + if (result == Py_None) + { + LM_T(LmtJexl, ("JexlResult is null")); + valueType = orion::ValueTypeNull; + return; + } + + // Other not null types + // FIXME PR: integrate getPyObjetType function into this one + valueType = getPyObjectType(result); + if (valueType == orion::ValueTypeNumber) + { + numberValue = PyFloat_AsDouble(result); + LM_T(LmtJexl, ("JexlResult (double): %f", numberValue)); + } + else if (valueType == orion::ValueTypeBoolean) + { + boolValue = PyObject_IsTrue(result); + LM_T(LmtJexl, ("JexlResult (bool): %s", boolValue ? "true": "false")); + } + else if (valueType == orion::ValueTypeObject) + { + processDict(compoundValueP, result); + // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot + // be used, as it produce string with ' instead of " in the resulting JSON string. + // FIXME PR: is the customJsonSerializer going to be used at the end? + /*PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); + if (repr == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + const char* str = PyUnicode_AsUTF8(repr); + Py_XDECREF(repr); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtJexl, ("JexlResult (object or vector): %s", str)); + stringValue = std::string(str); + } + }*/ + } + else if (valueType == orion::ValueTypeVector) + { + processList(compoundValueP, result); + // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot + // be used, as it produce string with ' instead of " in the resulting JSON string. + // FIXME PR: is the customJsonSerializer going to be used at the end? + /*PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); + if (repr == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + const char* str = PyUnicode_AsUTF8(repr); + Py_XDECREF(repr); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtJexl, ("JexlResult (object or vector): %s", str)); + stringValue = std::string(str); + } + }*/ + } + else if (valueType == orion::ValueTypeString) + { + const char* str = PyUnicode_AsUTF8(result); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtJexl, ("JexlResult (string): %s", str)); + stringValue = std::string(str); + } + } +} + + + + void JexlResult::processList(orion::CompoundValueNode* node, PyObject* result) + { + + } + + + + + void JexlResult::processDict(orion::CompoundValueNode* node, PyObject* result) + { + + } + + + /* **************************************************************************** * * toString - diff --git a/src/lib/jexl/JexlResult.h b/src/lib/jexl/JexlResult.h index 34c8effdc7..86144b8f07 100644 --- a/src/lib/jexl/JexlResult.h +++ b/src/lib/jexl/JexlResult.h @@ -30,6 +30,7 @@ #include "parse/CompoundValueNode.h" +#include #include /* **************************************************************************** @@ -49,6 +50,11 @@ class JexlResult // Use only when valueType is object or vector orion::CompoundValueNode* compoundValueP; + void fill(PyObject* result); + + void processList(orion::CompoundValueNode* node, PyObject* result); + void processDict(orion::CompoundValueNode* node, PyObject* result); + std::string toString(void); void release(void); }; From f9f9ed6bd954ae509751cbf61d2da488aecc0f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 09:36:49 +0100 Subject: [PATCH 022/104] FIX JexlResult improvements (wip) --- src/lib/jexl/JexlResult.cpp | 146 ++++++++++++++++++++++++++++++++++-- src/lib/jexl/JexlResult.h | 4 +- 2 files changed, 140 insertions(+), 10 deletions(-) diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index d103b4edff..4923fafb6b 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -125,7 +125,6 @@ void JexlResult::fill(PyObject* result) } // Other not null types - // FIXME PR: integrate getPyObjetType function into this one valueType = getPyObjectType(result); if (valueType == orion::ValueTypeNumber) { @@ -139,7 +138,13 @@ void JexlResult::fill(PyObject* result) } else if (valueType == orion::ValueTypeObject) { - processDict(compoundValueP, result); + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(result, &pos, &key, &value)) + { + processDictItem(compoundValueP, key, value); + } // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot // be used, as it produce string with ' instead of " in the resulting JSON string. // FIXME PR: is the customJsonSerializer going to be used at the end? @@ -169,7 +174,12 @@ void JexlResult::fill(PyObject* result) } else if (valueType == orion::ValueTypeVector) { - processList(compoundValueP, result); + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); + Py_ssize_t size = PyList_Size(result); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + processListItem(compoundValueP, PyList_GetItem(result, ix)); + } // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot // be used, as it produce string with ' instead of " in the resulting JSON string. // FIXME PR: is the customJsonSerializer going to be used at the end? @@ -216,18 +226,138 @@ void JexlResult::fill(PyObject* result) - void JexlResult::processList(orion::CompoundValueNode* node, PyObject* result) +/* **************************************************************************** +* +* processListItem - +* +* FIXME PR: maybe this should be static function out of the class? +*/ +void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* item) +{ + orion::CompoundValueNode* nodeP; + const char* str; + double d; + bool b; + switch (getPyObjectType(item)) { - + case orion::ValueTypeString: + str = PyUnicode_AsUTF8(item); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + } + else + { + LM_T(LmtJexl, ("processListITem (string): %s", str)); + nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); + parentP->add(nodeP); + } + break; + + case orion::ValueTypeNumber: + d = PyFloat_AsDouble(item); + LM_T(LmtJexl, ("processList (double): %f", d)); + nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); + parentP->add(nodeP); + break; + + case orion::ValueTypeBoolean: + b = PyObject_IsTrue(item); + LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); + nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); + parentP->add(nodeP); + break; + + case orion::ValueTypeNull: + nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeNull); + parentP->add(nodeP); + break; + + case orion::ValueTypeVector: + // TBD + break; + + case orion::ValueTypeObject: + // TBD + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given))")); + break; + + default: + LM_E(("Runtime Error (value type unknown))")); } +} - - void JexlResult::processDict(orion::CompoundValueNode* node, PyObject* result) +/* **************************************************************************** +* +* processDictItem - +* +* FIXME PR: maybe this should be static function out of the class? +*/ +void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +{ + std::string keyStr = ""; + orion::CompoundValueNode* nodeP; + const char* str; + double d; + bool b; + switch (getPyObjectType(value)) { - + case orion::ValueTypeString: + str = PyUnicode_AsUTF8(value); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + } + else + { + LM_T(LmtJexl, ("processListITem (string): %s", str)); + nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); + parentP->add(nodeP); + } + break; + + case orion::ValueTypeNumber: + d = PyFloat_AsDouble(value); + LM_T(LmtJexl, ("processList (double): %f", d)); + nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); + parentP->add(nodeP); + break; + + case orion::ValueTypeBoolean: + b = PyObject_IsTrue(value); + LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); + nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); + parentP->add(nodeP); + break; + + case orion::ValueTypeNull: + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeNull); + parentP->add(nodeP); + break; + + case orion::ValueTypeVector: + // TBD + break; + + case orion::ValueTypeObject: + // TBD + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given))")); + break; + + default: + LM_E(("Runtime Error (value type unknown))")); } +} diff --git a/src/lib/jexl/JexlResult.h b/src/lib/jexl/JexlResult.h index 86144b8f07..cdf853c24a 100644 --- a/src/lib/jexl/JexlResult.h +++ b/src/lib/jexl/JexlResult.h @@ -52,8 +52,8 @@ class JexlResult void fill(PyObject* result); - void processList(orion::CompoundValueNode* node, PyObject* result); - void processDict(orion::CompoundValueNode* node, PyObject* result); + void processListItem(orion::CompoundValueNode* parentP, PyObject* item); + void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); std::string toString(void); void release(void); From d7b3a66dcbd4df20ccdc7c27336bbe32998fd70d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 10:31:57 +0100 Subject: [PATCH 023/104] FIX JexlResult improvements --- src/lib/jexl/JexlResult.cpp | 62 +++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 4923fafb6b..6a77126be7 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -97,6 +97,10 @@ static orion::ValueType getPyObjectType(PyObject* obj) { return orion::ValueTypeVector; } + else if (obj == Py_None) + { + return orion::ValueTypeNull; + } else { // For other types we use string (this is also a failsafe for types not being strings) @@ -232,16 +236,20 @@ void JexlResult::fill(PyObject* result) * * FIXME PR: maybe this should be static function out of the class? */ -void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* item) +void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* value) { orion::CompoundValueNode* nodeP; + const char* str; double d; bool b; - switch (getPyObjectType(item)) + PyObject *keyAux, *valueAux; + Py_ssize_t pos, size; + + switch (getPyObjectType(value)) { case orion::ValueTypeString: - str = PyUnicode_AsUTF8(item); + str = PyUnicode_AsUTF8(value); if (str == NULL) { // FIXME PR: use LM_E/LM_W? @@ -256,14 +264,14 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* it break; case orion::ValueTypeNumber: - d = PyFloat_AsDouble(item); + d = PyFloat_AsDouble(value); LM_T(LmtJexl, ("processList (double): %f", d)); nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); parentP->add(nodeP); break; case orion::ValueTypeBoolean: - b = PyObject_IsTrue(item); + b = PyObject_IsTrue(value); LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); parentP->add(nodeP); @@ -275,11 +283,23 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* it break; case orion::ValueTypeVector: - // TBD + nodeP = new orion::CompoundValueNode(orion::ValueTypeVector); + size = PyList_Size(value); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + processListItem(nodeP, PyList_GetItem(value, ix)); + } + parentP->add(nodeP); break; case orion::ValueTypeObject: - // TBD + nodeP = new orion::CompoundValueNode(orion::ValueTypeObject); + pos = 0; + while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + { + processDictItem(nodeP, keyAux, valueAux); + } + parentP->add(nodeP); break; case orion::ValueTypeNotGiven: @@ -301,11 +321,21 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* it */ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) { - std::string keyStr = ""; + const char * keyStr = PyUnicode_AsUTF8(key); + if (keyStr == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + return; + } + orion::CompoundValueNode* nodeP; const char* str; double d; bool b; + PyObject *keyAux, *valueAux; + Py_ssize_t pos, size; + switch (getPyObjectType(value)) { case orion::ValueTypeString: @@ -343,11 +373,23 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke break; case orion::ValueTypeVector: - // TBD + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeVector); + size = PyList_Size(value); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + processListItem(nodeP, PyList_GetItem(value, ix)); + } + parentP->add(nodeP); break; case orion::ValueTypeObject: - // TBD + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeObject); + pos = 0; + while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + { + processDictItem(nodeP, keyAux, valueAux); + } + parentP->add(nodeP); break; case orion::ValueTypeNotGiven: From 4a5d3e321e6b0de5d2c49c9c7921020805056459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 11:06:37 +0100 Subject: [PATCH 024/104] FIX JexlResult improvements --- src/lib/jexl/JexlManager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index bfb261f944..c0545b87a1 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -162,6 +162,7 @@ void JexlManager::init(void) JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { JexlResult r; + r.valueType = orion::ValueTypeNull; LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); From 9827b46e685fb9aa468998cac42236dbb7e94ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 12:30:15 +0100 Subject: [PATCH 025/104] FIX cleanup --- src/lib/common/macroSubstitute.cpp | 28 ------- src/lib/common/macroSubstitute.h | 15 ---- src/lib/jexl/JexlManager.cpp | 119 ----------------------------- src/lib/jexl/JexlResult.cpp | 60 ++------------- 4 files changed, 7 insertions(+), 215 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 10c5ec0251..b9e6571ceb 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -84,34 +84,6 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon -/* **************************************************************************** -* -* buildReplacementMap - -* -*/ -/*void buildReplacementsMap -( - const Entity& en, - const std::string& service, - const std::string& token, - std::map* replacementsP -) -{ - replacementsP->insert(std::pair("id", "\"" + en.id + "\"")); - replacementsP->insert(std::pair("type", "\"" + en.type + "\"")); - replacementsP->insert(std::pair("service", "\"" + service + "\"")); - replacementsP->insert(std::pair("servicePath", "\"" + en.servicePath + "\"")); - replacementsP->insert(std::pair("authToken", "\"" + token + "\"")); - for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) - { - // Note that if some attribute is named service, servicePath or authToken (although it would be - // an anti-pattern), the attribute takes precedence - (*replacementsP)[en.attributeVector[ix]->name] = en.attributeVector[ix]->toJsonValue(); - } -}*/ - - - /* **************************************************************************** * * stringValueOrNothing - diff --git a/src/lib/common/macroSubstitute.h b/src/lib/common/macroSubstitute.h index cdb66ceaf7..a4ef0d89e3 100644 --- a/src/lib/common/macroSubstitute.h +++ b/src/lib/common/macroSubstitute.h @@ -38,21 +38,6 @@ extern std::string smartStringValue(const std::string stringValue, JexlContext* -/* **************************************************************************** -* -* buildReplacementMap - -* -*/ -/*extern void buildReplacementsMap -( - const Entity& en, - const std::string& service, - const std::string& token, - std::map* replacementsP -);*/ - - - /* **************************************************************************** * * macroSubstitute - diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index c0545b87a1..1d70241401 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -69,36 +69,6 @@ static const char* capturePythonError() -#if 0 -/* **************************************************************************** -* -* customJsonSerializerFunction - -* -* To be used in the json.dumps() step in JexlManager::evaluate() in order to get -* integer numbers when possible (i.e. "2" instead of "2.0"), this way behaving as -* the pre-JEXL macro replacement logic -*/ -static PyObject* customJsonSerializerFunction(PyObject* obj) -{ - // Check if the object is a float - if (PyFloat_Check(obj)) - { - // Convert float to integer if it represents a whole number - double value = PyFloat_AsDouble(obj); - if (value == (int)value) - { - return PyLong_FromDouble(value); - } - } - - // Return the original object - Py_INCREF(obj); - return obj; -} -#endif - - - /* **************************************************************************** * * JexlManager::init - @@ -108,7 +78,6 @@ void JexlManager::init(void) pyjexlModule = NULL; jsonModule = NULL; jexlEngine = NULL; - //customJsonSerializer = NULL; if (sem_init(&sem, 0, 1) == -1) { @@ -134,16 +103,6 @@ void JexlManager::init(void) } LM_T(LmtJexl, ("json module has been loaded")); -#if 0 - customJsonSerializer = PyCapsule_New((void*)customJsonSerializerFunction, NULL, NULL); - if (customJsonSerializer == NULL) - { - const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error creating json custom serializer function: %s)", error)); - } - LM_T(LmtJexl, ("json custom serializer has been createdmodule has been loaded")); -#endif - jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); if (jexlEngine == NULL) { @@ -186,76 +145,6 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ r.fill(result); Py_XDECREF(result); - // Special case: expresion evalutes to None - /*if (result == Py_None) - { - LM_T(LmtJexl, ("JEXL evaluation result is null")); - r.valueType = orion::ValueTypeNull; - Py_XDECREF(result); - return r; - } - - // Other not null types - r.valueType = getPyObjectType(result); - if (r.valueType == orion::ValueTypeNumber) - { - r.numberValue = PyFloat_AsDouble(result); - LM_T(LmtJexl, ("JEXL evaluation result (double): %f", r.numberValue)); - Py_XDECREF(result); - } - else if (r.valueType == orion::ValueTypeBoolean) - { - r.boolValue = PyObject_IsTrue(result); - LM_T(LmtJexl, ("JEXL evaluation result (bool): %s", r.boolValue ? "true": "false")); - Py_XDECREF(result); - } - else if ((r.valueType == orion::ValueTypeObject) || (r.valueType == orion::ValueTypeVector)) - { - // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot - // be used, as it produce string with ' instead of " in the resulting JSON string. - // FIXME PR: is the customJsonSerializer going to be used at the end? - PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); - Py_XDECREF(result); - if (repr == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); - r.valueType = orion::ValueTypeNull; - } - else - { - const char* str = PyUnicode_AsUTF8(repr); - Py_XDECREF(repr); - if (str == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); - r.valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtJexl, ("JEXL evaluation result (object or vector): %s", str)); - r.stringValue = std::string(str); - } - } - } - else if (r.valueType == orion::ValueTypeString) - { - const char* str = PyUnicode_AsUTF8(result); - Py_XDECREF(result); - if (str == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); - r.valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtJexl, ("JEXL evaluation result (string): %s", str)); - r.stringValue = std::string(str); - } - }*/ - return r; } @@ -279,14 +168,6 @@ void JexlManager::release(void) LM_T(LmtJexl, ("pyjexl module has been freed")); } -#if 0 - if (customJsonSerializer != NULL) - { - Py_XDECREF(customJsonSerializer); - LM_T(LmtJexl, ("json custom serializer module has been freed")); - } -#endif - if (jsonModule != NULL) { Py_XDECREF(jsonModule); diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 6a77126be7..19f4060f6f 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -147,34 +147,9 @@ void JexlResult::fill(PyObject* result) Py_ssize_t pos = 0; while (PyDict_Next(result, &pos, &key, &value)) { + // FIXME PR: memory should be freed? processDictItem(compoundValueP, key, value); } - // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot - // be used, as it produce string with ' instead of " in the resulting JSON string. - // FIXME PR: is the customJsonSerializer going to be used at the end? - /*PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); - if (repr == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - const char* str = PyUnicode_AsUTF8(repr); - Py_XDECREF(repr); - if (str == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtJexl, ("JexlResult (object or vector): %s", str)); - stringValue = std::string(str); - } - }*/ } else if (valueType == orion::ValueTypeVector) { @@ -182,34 +157,9 @@ void JexlResult::fill(PyObject* result) Py_ssize_t size = PyList_Size(result); for (Py_ssize_t ix = 0; ix < size; ++ix) { - processListItem(compoundValueP, PyList_GetItem(result, ix)); + // FIXME PR: memory should be freed? + processListItem(compoundValueP, PyList_GetItem(result, ix)); } - // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot - // be used, as it produce string with ' instead of " in the resulting JSON string. - // FIXME PR: is the customJsonSerializer going to be used at the end? - /*PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); - if (repr == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - const char* str = PyUnicode_AsUTF8(repr); - Py_XDECREF(repr); - if (str == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtJexl, ("JexlResult (object or vector): %s", str)); - stringValue = std::string(str); - } - }*/ } else if (valueType == orion::ValueTypeString) { @@ -288,6 +238,7 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va for (Py_ssize_t ix = 0; ix < size; ++ix) { processListItem(nodeP, PyList_GetItem(value, ix)); + // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -298,6 +249,7 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va while (PyDict_Next(value, &pos, &keyAux, &valueAux)) { processDictItem(nodeP, keyAux, valueAux); + // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -378,6 +330,7 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke for (Py_ssize_t ix = 0; ix < size; ++ix) { processListItem(nodeP, PyList_GetItem(value, ix)); + // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -388,6 +341,7 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke while (PyDict_Next(value, &pos, &keyAux, &valueAux)) { processDictItem(nodeP, keyAux, valueAux); + // FIXME PR: memory should be freed? } parentP->add(nodeP); break; From 8e3bdd35acb67867417db8d9ad5ad2461abad162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 14:27:27 +0100 Subject: [PATCH 026/104] FIX rename jexl to expression in class names and so --- CMakeLists.txt | 4 +- src/app/contextBroker/contextBroker.cpp | 6 +- src/lib/apiTypesV2/Entity.cpp | 10 +- src/lib/apiTypesV2/Entity.h | 6 +- src/lib/common/macroSubstitute.cpp | 37 +-- src/lib/common/macroSubstitute.h | 4 +- src/lib/common/string.h | 21 ++ src/lib/{jexl => expressions}/CMakeLists.txt | 18 +- src/lib/expressions/ExprContext.cpp | 299 +++++++++++++++++ .../ExprContext.h} | 35 +- .../ExprManager.cpp} | 44 +-- .../ExprManager.h} | 16 +- .../ExprResult.cpp} | 66 ++-- .../JexlResult.h => expressions/ExprResult.h} | 10 +- .../jexlMgr.cpp => expressions/exprMgr.cpp} | 10 +- .../{jexl/jexlMgr.h => expressions/exprMgr.h} | 12 +- src/lib/jexl/JexlContext.cpp | 300 ------------------ src/lib/logMsg/traceLevels.h | 2 +- src/lib/ngsi/ContextAttribute.cpp | 22 +- src/lib/ngsi/ContextAttribute.h | 6 +- src/lib/ngsi/ContextElementResponse.cpp | 4 +- src/lib/ngsi/ContextElementResponse.h | 4 +- src/lib/ngsi/ContextElementResponseVector.cpp | 4 +- src/lib/ngsi/ContextElementResponseVector.h | 4 +- src/lib/ngsi10/NotifyContextRequest.cpp | 8 +- src/lib/ngsi10/NotifyContextRequest.h | 2 +- src/lib/ngsiNotify/Notifier.cpp | 84 ++--- src/lib/parse/CompoundValueNode.cpp | 48 +-- src/lib/parse/CompoundValueNode.h | 8 +- 29 files changed, 546 insertions(+), 548 deletions(-) rename src/lib/{jexl => expressions}/CMakeLists.txt (85%) create mode 100644 src/lib/expressions/ExprContext.cpp rename src/lib/{jexl/JexlContext.h => expressions/ExprContext.h} (71%) rename src/lib/{jexl/JexlManager.cpp => expressions/ExprManager.cpp} (76%) rename src/lib/{jexl/JexlManager.h => expressions/ExprManager.h} (80%) rename src/lib/{jexl/JexlResult.cpp => expressions/ExprResult.cpp} (81%) rename src/lib/{jexl/JexlResult.h => expressions/ExprResult.h} (92%) rename src/lib/{jexl/jexlMgr.cpp => expressions/exprMgr.cpp} (85%) rename src/lib/{jexl/jexlMgr.h => expressions/exprMgr.h} (80%) delete mode 100644 src/lib/jexl/JexlContext.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4cae22ff0b..03bfaab443 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,7 +204,7 @@ SET (ORION_LIBS common alarmMgr metricsMgr - jexl + expressions logSummary lm pa @@ -322,7 +322,7 @@ if (error EQUAL 0) ADD_SUBDIRECTORY(src/lib/cache) ADD_SUBDIRECTORY(src/lib/alarmMgr) ADD_SUBDIRECTORY(src/lib/metricsMgr) - ADD_SUBDIRECTORY(src/lib/jexl) + ADD_SUBDIRECTORY(src/lib/expressions) ADD_SUBDIRECTORY(src/lib/logSummary) ADD_SUBDIRECTORY(src/lib/mqtt) ADD_SUBDIRECTORY(src/app/contextBroker) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index b665a01555..ec994937fa 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -105,7 +105,7 @@ #include "alarmMgr/alarmMgr.h" #include "mqtt/mqttMgr.h" #include "metricsMgr/metricsMgr.h" -#include "jexl/jexlMgr.h" +#include "expressions/exprMgr.h" #include "logSummary/logSummary.h" #include "contextBroker/orionRestServices.h" @@ -598,7 +598,7 @@ void exitFunc(void) curl_context_cleanup(); curl_global_cleanup(); - jexlMgr.release(); + exprMgr.release(); #ifdef DEBUG // valgrind pass is done using DEBUG compilation, so we have to take care with @@ -1222,7 +1222,7 @@ int main(int argC, char* argV[]) SemOpType policy = policyGet(reqMutexPolicy); alarmMgr.init(relogAlarms); mqttMgr.init(mqttTimeout); - jexlMgr.init(); + exprMgr.init(); orionInit(orionExit, ORION_VERSION, policy, statCounters, statSemWait, statTiming, statNotifQueue, strictIdv1); mongoInit(dbURI, dbHost, rplSet, dbName, user, pwd, authMech, authDb, dbSSL, dbDisableRetryWrites, mtenant, dbTimeout, writeConcern, dbPoolSize, statSemWait); metricsMgr.init(!disableMetrics, statSemWait); diff --git a/src/lib/apiTypesV2/Entity.cpp b/src/lib/apiTypesV2/Entity.cpp index 5501c18e77..f864e0bcb2 100644 --- a/src/lib/apiTypesV2/Entity.cpp +++ b/src/lib/apiTypesV2/Entity.cpp @@ -41,7 +41,7 @@ #include "rest/OrionError.h" #include "apiTypesV2/Entity.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" @@ -308,7 +308,7 @@ std::string Entity::toJson bool blacklist, const std::vector& metadataFilter, bool renderNgsiField, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { std::vector orderedAttrs; @@ -327,7 +327,7 @@ std::string Entity::toJson out = toJsonKeyvalues(orderedAttrs); break; default: // NGSI_V2_NORMALIZED - out = toJsonNormalized(orderedAttrs, metadataFilter, renderNgsiField, jexlContextP); + out = toJsonNormalized(orderedAttrs, metadataFilter, renderNgsiField, exprContextObjectP); break; } @@ -444,7 +444,7 @@ std::string Entity::toJsonNormalized const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { JsonObjectHelper jh; @@ -476,7 +476,7 @@ std::string Entity::toJsonNormalized for (unsigned int ix = 0; ix < orderedAttrs.size(); ix++) { ContextAttribute* caP = orderedAttrs[ix]; - jh.addRaw(caP->name, caP->toJson(metadataFilter, renderNgsiField, jexlContextP)); + jh.addRaw(caP->name, caP->toJson(metadataFilter, renderNgsiField, exprContextObjectP)); } return jh.str(); diff --git a/src/lib/apiTypesV2/Entity.h b/src/lib/apiTypesV2/Entity.h index 1813faa0c1..f8acc3af97 100644 --- a/src/lib/apiTypesV2/Entity.h +++ b/src/lib/apiTypesV2/Entity.h @@ -32,7 +32,7 @@ #include "ngsi/ContextAttributeVector.h" #include "ngsi/EntityId.h" #include "rest/OrionError.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" @@ -92,7 +92,7 @@ class Entity bool blacklist, const std::vector& metadataFilter, bool renderNgsiField = false, - JexlContext* jexlContext = NULL); + ExprContextObject* exprContext = NULL); std::string toJson(RenderFormat renderFormat, bool renderNgsiField = false); @@ -146,7 +146,7 @@ class Entity std::string toJsonNormalized(const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField = false, - JexlContext* jexlContext = NULL); + ExprContextObject* exprContext = NULL); }; #endif // SRC_LIB_APITYPESV2_ENTITY_H_ diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index b9e6571ceb..3c0fc7e105 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -32,7 +32,7 @@ #include "common/JsonHelper.h" #include "common/macroSubstitute.h" -#include "jexl/jexlMgr.h" +#include "expressions/exprMgr.h" /* **************************************************************************** @@ -42,18 +42,18 @@ * Returns the effective string value, taking into account replacements * */ -std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault) +std::string smartStringValue(const std::string stringValue, ExprContextObject* exprContextObjectP, const std::string notFoundDefault) { // This code is pretty similar to the one in CompoundValueNode::toJson() // The program logic branching is the same, but the result at the end of each if-else // is different, which makes difficult to unify both them - if ((jexlContextP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) + if ((exprContextObjectP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) { // "Full replacement" case. In this case, the result is not always a string // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - JexlResult r = jexlMgr.evaluate(jexlContextP, macroName); + ExprResult r = exprMgr.evaluate(exprContextObjectP, macroName); if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; @@ -63,11 +63,11 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon return r.toString(); } } - else if (jexlContextP != NULL) + else if (exprContextObjectP != NULL) { // "Partial replacement" case. In this case, the result is always a string std::string effectiveValue; - if (!macroSubstitute(&effectiveValue, stringValue, jexlContextP, "null", true)) + if (!macroSubstitute(&effectiveValue, stringValue, exprContextObjectP, "null", true)) { // error already logged in macroSubstitute, using stringValue itself as failsafe effectiveValue = stringValue; @@ -89,10 +89,9 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon * stringValueOrNothing - * */ -// FIXME PR: inTheMiddle -> raw ? -static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault, bool inTheMiddle) +static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, const std::string key, const std::string& notFoundDefault, bool raw) { - JexlResult r = jexlMgr.evaluate(jexlContextP, key); + ExprResult r = exprMgr.evaluate(exprContextObjectP, key); if (r.valueType == orion::ValueTypeNull) { @@ -101,17 +100,16 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st else { std::string s = r.toString(); - if (inTheMiddle) + if (raw) { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), // so double quotes have to be be removed - if (s[0] == '"') - { - s = s.substr(1, s.size()-2); - } + return removeQuotes(s); + } + else + { + return s; } - - return s; } } @@ -142,8 +140,7 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st * Date: Mon Jun 19 16:33:29 2017 +0200 * */ -// FIXME PR: inTheMiddle -> raw ? -bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexlContextP, const std::string& notFoundDefault, bool inTheMiddle) +bool macroSubstitute(std::string* to, const std::string& from, ExprContextObject* exprContextObjectP, const std::string& notFoundDefault, bool raw) { // Initial size check: is the string to convert too big? // @@ -206,7 +203,7 @@ bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexl // The +3 is due to "${" and "}" toReduce += (macroName.length() + 3) * times; - toAdd += stringValueOrNothing(jexlContextP, macroName, notFoundDefault, inTheMiddle).length() * times; + toAdd += stringValueOrNothing(exprContextObjectP, macroName, notFoundDefault, raw).length() * times; } if (from.length() + toAdd - toReduce > outReqMsgMaxSize) @@ -224,7 +221,7 @@ bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexl unsigned int times = it->second; std::string macro = "${" + macroName + "}"; - std::string value = stringValueOrNothing(jexlContextP, macroName, notFoundDefault, inTheMiddle); + std::string value = stringValueOrNothing(exprContextObjectP, macroName, notFoundDefault, raw); // We have to do the replace operation as many times as macro occurrences for (unsigned int ix = 0; ix < times; ix++) diff --git a/src/lib/common/macroSubstitute.h b/src/lib/common/macroSubstitute.h index a4ef0d89e3..58f2aac2e5 100644 --- a/src/lib/common/macroSubstitute.h +++ b/src/lib/common/macroSubstitute.h @@ -34,7 +34,7 @@ * smartStringValue - * */ -extern std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault); +extern std::string smartStringValue(const std::string stringValue, ExprContextObject* exprContextObjectP, const std::string notFoundDefault); @@ -43,6 +43,6 @@ extern std::string smartStringValue(const std::string stringValue, JexlContext* * macroSubstitute - * */ -extern bool macroSubstitute(std::string* sP, const std::string& in, JexlContext* jexlContextP, const std::string& notFoundDefault, bool inTheMiddle = false); +extern bool macroSubstitute(std::string* sP, const std::string& in, ExprContextObject* exprContextObjectP, const std::string& notFoundDefault, bool raw = false); #endif // SRC_LIB_COMMON_MACROSUBSTITUTE_H_ diff --git a/src/lib/common/string.h b/src/lib/common/string.h index 886362d709..f3abcee416 100644 --- a/src/lib/common/string.h +++ b/src/lib/common/string.h @@ -216,4 +216,25 @@ extern std::string offuscatePassword(const std::string& uri, const std::string& */ extern bool regComp(regex_t* re, const char* pattern, int flags); + + +/* **************************************************************************** +* +* removeQuotes - +* +*/ +inline std::string removeQuotes(std::string s) +{ + if (s[0] == '"') + { + return s.substr(1, s.size()-2); + } + else + { + return s; + } +} + + + #endif // SRC_LIB_COMMON_STRING_H_ diff --git a/src/lib/jexl/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt similarity index 85% rename from src/lib/jexl/CMakeLists.txt rename to src/lib/expressions/CMakeLists.txt index a2ea7eb6e2..4f938e3634 100644 --- a/src/lib/jexl/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -21,17 +21,17 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES - JexlManager.cpp - JexlContext.cpp - JexlResult.cpp - jexlMgr.cpp + ExprManager.cpp + ExprContext.cpp + ExprResult.cpp + exprMgr.cpp ) SET (HEADERS - JexlManager.h - JexlContext.h - JexlResult.h - jexlMgr.h + ExprManager.h + ExprContext.h + ExprResult.h + exprMgr.h ) @@ -43,4 +43,4 @@ include_directories("${PROJECT_SOURCE_DIR}/src/lib") # Library declaration # ----------------------------------------------------------------- -ADD_LIBRARY(jexl STATIC ${SOURCES} ${HEADERS}) +ADD_LIBRARY(expressions STATIC ${SOURCES} ${HEADERS}) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp new file mode 100644 index 0000000000..8c05fd5764 --- /dev/null +++ b/src/lib/expressions/ExprContext.cpp @@ -0,0 +1,299 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +// FIXME PR: add methods should check error in Py_BuildValue() for NULL + +#include + +#include "logMsg/logMsg.h" +#include "expressions/ExprContext.h" + + + +/* **************************************************************************** +* +* ExprContextObject::ExprContextObject - +*/ +ExprContextObject::ExprContextObject +() +{ + jexlContext = PyDict_New(); + + if (jexlContext == NULL) + { + // FIXME PR: error control + // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + } +} + + + +/* **************************************************************************** +* +* ExprContextObject::get - +*/ +PyObject* ExprContextObject::get(void) +{ + return jexlContext; +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, const std::string& _value) +{ + LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); + PyObject* value = Py_BuildValue("s", _value.c_str()); + PyDict_SetItemString(jexlContext, key.c_str(), value); + Py_DECREF(value); +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, double _value) +{ + LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); + PyObject* value = Py_BuildValue("d", _value); + PyDict_SetItemString(jexlContext, key.c_str(), value); + Py_DECREF(value); +} + + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, bool _value) +{ + LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value? "true" : "false")); + if (_value) + { + PyDict_SetItemString(jexlContext, key.c_str(), Py_True); + } + else + { + PyDict_SetItemString(jexlContext, key.c_str(), Py_False); + } +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key) +{ + LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), Py_None); +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, ExprContextObject exprContextObject) +{ + // FIXME PR: implement a toString() method in ExprContextObject to be used here + //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.get()); +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, ExprContextList exprContextList) +{ + // FIXME PR: implement a toString() method in ExprContextList to be used here + //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); +} + + + +/* **************************************************************************** +* +* ExprContextObject::hasKey - +*/ +bool ExprContextObject::hasKey(const std::string& key) +{ + // Check if the key exists in the jexlContext dictionary + PyObject* keyObject = PyUnicode_FromString(key.c_str()); + int result = PyDict_Contains(jexlContext, keyObject); + Py_DECREF(keyObject); + + return result == 1; // Return true if key exists, false otherwise +} + + + +/* **************************************************************************** +* +* ExprContextObject::release - +*/ +void ExprContextObject::release(void) +{ + // FIXME PR: this is not correct. Recursively release of the dict object + //Py_XDECREF(jexlContext); +} + + +/* **************************************************************************** +* +* ExprContextList::ExprContextList - +*/ +ExprContextList::ExprContextList() +{ + jexlContext = PyList_New(0); + + if (jexlContext == NULL) + { + // FIXME PR: error control + // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + } +} + + + +/* **************************************************************************** +* +* ExprContextList::get - +*/ +PyObject* ExprContextList::get(void) +{ + return jexlContext; +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(const std::string& _value) +{ + LM_T(LmtExpr, ("adding to expression context list (string): %s=", _value.c_str())); + PyObject* value = Py_BuildValue("s", _value.c_str()); + PyList_Append(jexlContext, value); + Py_DECREF(value); +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(double _value) +{ + LM_T(LmtExpr, ("adding to expression context list (double): %f", _value)); + PyObject* value = Py_BuildValue("d", _value); + PyList_Append(jexlContext, value); + Py_DECREF(value); +} + + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(bool _value) +{ + LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value? "true" : "false")); + if (_value) + { + PyList_Append(jexlContext, Py_True); + } + else + { + PyList_Append(jexlContext, Py_False); + } +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(void) +{ + LM_T(LmtExpr, ("adding to expression context list (none)")); + PyList_Append(jexlContext, Py_None); +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(ExprContextObject exprContextObject) +{ + // FIXME PR: implement a toString() method in ExprContext to be used here + //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + PyList_Append(jexlContext, exprContextObject.get()); +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(ExprContextList exprContextList) +{ + // FIXME PR: implement a toString() method in ExprContextList to be used here + //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + PyList_Append(jexlContext, exprContextList.get()); +} + + + +/* **************************************************************************** +* +* ExprContextList::relesase - +*/ +void ExprContextList::release(void) +{ + // FIXME PR: this is not correct. Recursively release of the list object + Py_XDECREF(jexlContext); +} \ No newline at end of file diff --git a/src/lib/jexl/JexlContext.h b/src/lib/expressions/ExprContext.h similarity index 71% rename from src/lib/jexl/JexlContext.h rename to src/lib/expressions/ExprContext.h index 4c91fc95a6..4db34f562c 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/expressions/ExprContext.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_JEXLCONTEXT_H_ -#define SRC_LIB_JEXL_JEXLCONTEXT_H_ +#ifndef SRC_LIB_JEXL_EXPRCONTEXT_H_ +#define SRC_LIB_JEXL_EXPRCONTEXT_H_ /* * @@ -26,56 +26,51 @@ * Author: Fermin Galan */ -// FIXME PR: rename this to a better name (ExprContextDict?) - #include - #include -//#include "jexl/JexlContextList.h" - -class JexlContextList; // forward declaration +class ExprContextList; // forward declaration /* **************************************************************************** * -* JexlContext - +* ExprContext - */ -class JexlContext +class ExprContextObject { private: - PyObject* jexl_context; + PyObject* jexlContext; public: - JexlContext(); + ExprContextObject(); PyObject* get(void); void add(const std::string& key, const std::string& value); void add(const std::string& key, double value); void add(const std::string& key, bool value); void add(const std::string& key); - void add(const std::string& key, JexlContext jexlContext); - void add(const std::string& key, JexlContextList jexlContextList); + void add(const std::string& key, ExprContextObject exprContextObject); + void add(const std::string& key, ExprContextList exprContextList); bool hasKey(const std::string& key); void release(void); }; -class JexlContextList +class ExprContextList { private: - PyObject* jexl_context; + PyObject* jexlContext; public: - JexlContextList(); + ExprContextList(); PyObject* get(void); void add(const std::string& value); void add(double value); void add(bool value); void add(void); - void add(JexlContext jexlContext); - void add(JexlContextList jexlContextList); + void add(ExprContextObject exprContextObject); + void add(ExprContextList exprContextList); void release(void); }; -#endif // #define SRC_LIB_JEXL_JEXLCONTEXT_H_ +#endif // #define SRC_LIB_JEXL_EXPRCONTEXT_H_ diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/expressions/ExprManager.cpp similarity index 76% rename from src/lib/jexl/JexlManager.cpp rename to src/lib/expressions/ExprManager.cpp index 1d70241401..10739d6a1d 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -25,8 +25,8 @@ #include -#include "jexl/JexlManager.h" -#include "jexl/JexlResult.h" +#include "expressions/ExprManager.h" +#include "expressions/ExprResult.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" @@ -71,9 +71,9 @@ static const char* capturePythonError() /* **************************************************************************** * -* JexlManager::init - +* ExprManager::init - */ -void JexlManager::init(void) +void ExprManager::init(void) { pyjexlModule = NULL; jsonModule = NULL; @@ -85,7 +85,7 @@ void JexlManager::init(void) } Py_Initialize(); - LM_T(LmtJexl, ("Python interpreter has been initialized")); + LM_T(LmtExpr, ("Python interpreter has been initialized")); pyjexlModule = PyImport_ImportModule("pyjexl"); if (pyjexlModule == NULL) @@ -93,7 +93,7 @@ void JexlManager::init(void) const char* error = capturePythonError(); LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", error)); } - LM_T(LmtJexl, ("pyjexl module has been loaded")); + LM_T(LmtExpr, ("pyjexl module has been loaded")); jsonModule = PyImport_ImportModule("json"); if (jsonModule == NULL) @@ -101,7 +101,7 @@ void JexlManager::init(void) const char* error = capturePythonError(); LM_X(1, ("Fatal Error (error importing json module: %s)", error)); } - LM_T(LmtJexl, ("json module has been loaded")); + LM_T(LmtExpr, ("json module has been loaded")); jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); if (jexlEngine == NULL) @@ -109,40 +109,42 @@ void JexlManager::init(void) const char* error = capturePythonError(); LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", error)); } - LM_T(LmtJexl, ("jexl engine has been created")); + LM_T(LmtExpr, ("jexl engine has been created")); } /* **************************************************************************** * -* JexlManager::evaluate - +* ExprManager::evaluate - */ -JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) +ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) { - JexlResult r; + ExprResult r; r.valueType = orion::ValueTypeNull; - LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); + LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error building expression: %s", capturePythonError())); + LM_T(LmtExpr, ("error building expression: %s", capturePythonError())); return r; } - PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, jexlContextP->get()); + PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, exprContextObjectP->get()); Py_XDECREF(expression); if (result == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression: %s", capturePythonError())); + LM_T(LmtExpr, ("error evaluating expression: %s", capturePythonError())); return r; } r.fill(result); + + // FIXME PR: does this Py_XDECREF() recursively in the case of dicts or lists? Py_XDECREF(result); return r; @@ -152,28 +154,28 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ /* **************************************************************************** * -* JexlManager::release - +* ExprManager::release - */ -void JexlManager::release(void) +void ExprManager::release(void) { if (jexlEngine != NULL) { Py_XDECREF(jexlEngine); - LM_T(LmtJexl, ("jexl engine has been freed")); + LM_T(LmtExpr, ("jexl engine has been freed")); } if (pyjexlModule != NULL) { Py_XDECREF(pyjexlModule); - LM_T(LmtJexl, ("pyjexl module has been freed")); + LM_T(LmtExpr, ("pyjexl module has been freed")); } if (jsonModule != NULL) { Py_XDECREF(jsonModule); - LM_T(LmtJexl, ("json module has been freed")); + LM_T(LmtExpr, ("json module has been freed")); } Py_Finalize(); - LM_T(LmtJexl, ("Python interpreter has been finalized")); + LM_T(LmtExpr, ("Python interpreter has been finalized")); } \ No newline at end of file diff --git a/src/lib/jexl/JexlManager.h b/src/lib/expressions/ExprManager.h similarity index 80% rename from src/lib/jexl/JexlManager.h rename to src/lib/expressions/ExprManager.h index adce908c7c..7466372b29 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/expressions/ExprManager.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_JEXLMANAGER_H_ -#define SRC_LIB_JEXL_JEXLMANAGER_H_ +#ifndef SRC_LIB_JEXL_EXPRMANAGER_H_ +#define SRC_LIB_JEXL_EXPRMANAGER_H_ /* * @@ -29,14 +29,14 @@ #include #include -#include "jexl/JexlContext.h" -#include "jexl/JexlResult.h" +#include "expressions/ExprContext.h" +#include "expressions/ExprResult.h" /* **************************************************************************** * -* JexlManager - +* ExprManager - */ -class JexlManager +class ExprManager { private: PyObject* pyjexlModule; @@ -47,8 +47,8 @@ class JexlManager public: void init(void); - JexlResult evaluate(JexlContext* jexlContextP, const std::string& expression); + ExprResult evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); void release(void); }; -#endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file +#endif // SRC_LIB_JEXL_EXPRMANAGER_H_ \ No newline at end of file diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/expressions/ExprResult.cpp similarity index 81% rename from src/lib/jexl/JexlResult.cpp rename to src/lib/expressions/ExprResult.cpp index 19f4060f6f..036651fdc4 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -23,7 +23,7 @@ * Author: Fermin Galan */ -#include "jexl/JexlResult.h" +#include "expressions/ExprResult.h" #include "common/string.h" #include "common/JsonHelper.h" @@ -115,7 +115,7 @@ static orion::ValueType getPyObjectType(PyObject* obj) * fill - * */ -void JexlResult::fill(PyObject* result) +void ExprResult::fill(PyObject* result) { // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; @@ -123,7 +123,7 @@ void JexlResult::fill(PyObject* result) // Special case: expresion evalutes to None if (result == Py_None) { - LM_T(LmtJexl, ("JexlResult is null")); + LM_T(LmtExpr, ("ExprResult is null")); valueType = orion::ValueTypeNull; return; } @@ -133,12 +133,12 @@ void JexlResult::fill(PyObject* result) if (valueType == orion::ValueTypeNumber) { numberValue = PyFloat_AsDouble(result); - LM_T(LmtJexl, ("JexlResult (double): %f", numberValue)); + LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); } else if (valueType == orion::ValueTypeBoolean) { boolValue = PyObject_IsTrue(result); - LM_T(LmtJexl, ("JexlResult (bool): %s", boolValue ? "true": "false")); + LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); } else if (valueType == orion::ValueTypeObject) { @@ -147,7 +147,8 @@ void JexlResult::fill(PyObject* result) Py_ssize_t pos = 0; while (PyDict_Next(result, &pos, &key, &value)) { - // FIXME PR: memory should be freed? + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() processDictItem(compoundValueP, key, value); } } @@ -157,7 +158,8 @@ void JexlResult::fill(PyObject* result) Py_ssize_t size = PyList_Size(result); for (Py_ssize_t ix = 0; ix < size; ++ix) { - // FIXME PR: memory should be freed? + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() processListItem(compoundValueP, PyList_GetItem(result, ix)); } } @@ -167,12 +169,12 @@ void JexlResult::fill(PyObject* result) if (str == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); valueType = orion::ValueTypeNull; } else { - LM_T(LmtJexl, ("JexlResult (string): %s", str)); + LM_T(LmtExpr, ("ExprResult (string): %s", str)); stringValue = std::string(str); } } @@ -186,7 +188,7 @@ void JexlResult::fill(PyObject* result) * * FIXME PR: maybe this should be static function out of the class? */ -void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* value) +void ExprResult::processListItem(orion::CompoundValueNode* parentP, PyObject* value) { orion::CompoundValueNode* nodeP; @@ -203,11 +205,11 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va if (str == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); } else { - LM_T(LmtJexl, ("processListITem (string): %s", str)); + LM_T(LmtExpr, ("processListITem (string): %s", str)); nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); parentP->add(nodeP); } @@ -215,14 +217,14 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va case orion::ValueTypeNumber: d = PyFloat_AsDouble(value); - LM_T(LmtJexl, ("processList (double): %f", d)); + LM_T(LmtExpr, ("processList (double): %f", d)); nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); parentP->add(nodeP); break; case orion::ValueTypeBoolean: b = PyObject_IsTrue(value); - LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); + LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); parentP->add(nodeP); break; @@ -237,8 +239,9 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va size = PyList_Size(value); for (Py_ssize_t ix = 0; ix < size; ++ix) { - processListItem(nodeP, PyList_GetItem(value, ix)); - // FIXME PR: memory should be freed? + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(nodeP, PyList_GetItem(value, ix)); } parentP->add(nodeP); break; @@ -248,8 +251,9 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va pos = 0; while (PyDict_Next(value, &pos, &keyAux, &valueAux)) { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() processDictItem(nodeP, keyAux, valueAux); - // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -271,13 +275,13 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va * * FIXME PR: maybe this should be static function out of the class? */ -void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) { const char * keyStr = PyUnicode_AsUTF8(key); if (keyStr == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); return; } @@ -295,11 +299,11 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke if (str == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); } else { - LM_T(LmtJexl, ("processListITem (string): %s", str)); + LM_T(LmtExpr, ("processListITem (string): %s", str)); nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); parentP->add(nodeP); } @@ -307,14 +311,14 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke case orion::ValueTypeNumber: d = PyFloat_AsDouble(value); - LM_T(LmtJexl, ("processList (double): %f", d)); + LM_T(LmtExpr, ("processList (double): %f", d)); nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); parentP->add(nodeP); break; case orion::ValueTypeBoolean: b = PyObject_IsTrue(value); - LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); + LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); parentP->add(nodeP); break; @@ -329,8 +333,9 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke size = PyList_Size(value); for (Py_ssize_t ix = 0; ix < size; ++ix) { - processListItem(nodeP, PyList_GetItem(value, ix)); - // FIXME PR: memory should be freed? + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(nodeP, PyList_GetItem(value, ix)); } parentP->add(nodeP); break; @@ -340,8 +345,9 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke pos = 0; while (PyDict_Next(value, &pos, &keyAux, &valueAux)) { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() processDictItem(nodeP, keyAux, valueAux); - // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -364,7 +370,7 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke * Pretty similar to ContextAttribute::toJsonValue() * */ -std::string JexlResult::toString(void) +std::string ExprResult::toString(void) { if (valueType == orion::ValueTypeNumber) { @@ -396,7 +402,7 @@ std::string JexlResult::toString(void) } else { - LM_E(("Runtime Error (not allowed type in JexlResult: %s)", valueTypeName(valueType))); + LM_E(("Runtime Error (not allowed type in ExprResult: %s)", valueTypeName(valueType))); return ""; } } @@ -405,9 +411,9 @@ std::string JexlResult::toString(void) /* **************************************************************************** * -* JexlResult::release - +* ExprResult::release - */ -void JexlResult::release(void) +void ExprResult::release(void) { if (compoundValueP != NULL) { diff --git a/src/lib/jexl/JexlResult.h b/src/lib/expressions/ExprResult.h similarity index 92% rename from src/lib/jexl/JexlResult.h rename to src/lib/expressions/ExprResult.h index cdf853c24a..bfa4bcbbb7 100644 --- a/src/lib/jexl/JexlResult.h +++ b/src/lib/expressions/ExprResult.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_JEXLRESULT_H_ -#define SRC_LIB_JEXL_JEXLRESULT_H_ +#ifndef SRC_LIB_JEXL_EXPRRESULT_H_ +#define SRC_LIB_JEXL_EXPRRESULT_H_ /* * @@ -35,9 +35,9 @@ /* **************************************************************************** * -* JexlResult - +* ExprResult - */ -class JexlResult +class ExprResult { public: // Similar to the fields used in ContextAttribute.h @@ -61,4 +61,4 @@ class JexlResult -#endif // #define SRC_LIB_JEXL_JEXLRESULT_H_ +#endif // #define SRC_LIB_JEXL_EXPRRESULT_H_ diff --git a/src/lib/jexl/jexlMgr.cpp b/src/lib/expressions/exprMgr.cpp similarity index 85% rename from src/lib/jexl/jexlMgr.cpp rename to src/lib/expressions/exprMgr.cpp index a5e8e2cd85..f56d51706b 100644 --- a/src/lib/jexl/jexlMgr.cpp +++ b/src/lib/expressions/exprMgr.cpp @@ -1,6 +1,6 @@ /* * -* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U * * This file is part of Orion Context Broker. * @@ -20,14 +20,14 @@ * For those usages not covered by this license please contact with * iot_support at tid dot es * -* Author: Ken Zangelin +* Author: Fermin Galan */ -#include "jexl/JexlManager.h" +#include "expressions/ExprManager.h" /* **************************************************************************** * -* jexlMgr - +* exprMgr - */ -JexlManager jexlMgr; +ExprManager exprMgr; diff --git a/src/lib/jexl/jexlMgr.h b/src/lib/expressions/exprMgr.h similarity index 80% rename from src/lib/jexl/jexlMgr.h rename to src/lib/expressions/exprMgr.h index 75b22a66a8..852e396e24 100644 --- a/src/lib/jexl/jexlMgr.h +++ b/src/lib/expressions/exprMgr.h @@ -1,9 +1,9 @@ -#ifndef SRC_LIB_JEXL_JEXLMGR_H_ -#define SRC_LIB_JEXL_JEXLMGR_H_ +#ifndef SRC_LIB_JEXL_EXPRMGR_H_ +#define SRC_LIB_JEXL_EXPRMGR_H_ /* * -* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U * * This file is part of Orion Context Broker. * @@ -25,7 +25,7 @@ * * Author: Fermin Galan */ -#include "jexl/JexlManager.h" +#include "expressions/ExprManager.h" @@ -33,6 +33,6 @@ * * metricsMgr - */ -extern JexlManager jexlMgr; +extern ExprManager exprMgr; -#endif // SRC_LIB_JEXL_JEXLMGR_H_ +#endif // SRC_LIB_JEXL_EXPRMGR_H_ diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp deleted file mode 100644 index 82ab5e3dd4..0000000000 --- a/src/lib/jexl/JexlContext.cpp +++ /dev/null @@ -1,300 +0,0 @@ -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -// FIXME PR: use better names than JexlContext and JexlContextList. Also in LM_T -// FIXME PR: add methods should check error in Py_BuildValue() for NULL - -#include - -#include "logMsg/logMsg.h" -#include "jexl/JexlContext.h" - - - -/* **************************************************************************** -* -* JexlContext::JexlContext - -*/ -JexlContext::JexlContext -() -{ - jexl_context = PyDict_New(); - - if (jexl_context == NULL) - { - // FIXME PR: error control - // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) - } -} - - - -/* **************************************************************************** -* -* JexlContext::get - -*/ -PyObject* JexlContext::get(void) -{ - return jexl_context; -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, const std::string& _value) -{ - LM_T(LmtJexl, ("adding to JEXL context (string): %s=%s", key.c_str(), _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - PyDict_SetItemString(jexl_context, key.c_str(), value); - Py_DECREF(value); -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, double _value) -{ - LM_T(LmtJexl, ("adding to JEXL context (double): %s=%f", key.c_str(), _value)); - PyObject* value = Py_BuildValue("d", _value); - PyDict_SetItemString(jexl_context, key.c_str(), value); - Py_DECREF(value); -} - - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, bool _value) -{ - LM_T(LmtJexl, ("adding to JEXL context (bool): %s=%s", key.c_str(), _value? "true" : "false")); - if (_value) - { - PyDict_SetItemString(jexl_context, key.c_str(), Py_True); - } - else - { - PyDict_SetItemString(jexl_context, key.c_str(), Py_False); - } -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key) -{ - LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyDict_SetItemString(jexl_context, key.c_str(), Py_None); -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, JexlContext _jexlContext) -{ - // FIXME PR: implement a toString() method in JexlContext to be used here - //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyDict_SetItemString(jexl_context, key.c_str(), _jexlContext.get()); -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, JexlContextList jexlContextList) -{ - // FIXME PR: implement a toString() method in JexlContextList to be used here - //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyDict_SetItemString(jexl_context, key.c_str(), jexlContextList.get()); -} - - - -/* **************************************************************************** -* -* JexlContext::hasKey - -*/ -bool JexlContext::hasKey(const std::string& key) -{ - // Check if the key exists in the jexl_context dictionary - PyObject* keyObject = PyUnicode_FromString(key.c_str()); - int result = PyDict_Contains(jexl_context, keyObject); - Py_DECREF(keyObject); - - return result == 1; // Return true if key exists, false otherwise -} - - - -/* **************************************************************************** -* -* JexlContext::release - -*/ -void JexlContext::release(void) -{ - // FIXME PR: this is not correct. Recursively release of the dict object - //Py_XDECREF(jexl_context); -} - - -/* **************************************************************************** -* -* JexlContextList::JexlContextList - -*/ -JexlContextList::JexlContextList() -{ - jexl_context = PyList_New(0); - - if (jexl_context == NULL) - { - // FIXME PR: error control - // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) - } -} - - - -/* **************************************************************************** -* -* JexlContextList::get - -*/ -PyObject* JexlContextList::get(void) -{ - return jexl_context; -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(const std::string& _value) -{ - LM_T(LmtJexl, ("adding to JEXL context list (string): %s=", _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - PyList_Append(jexl_context, value); - Py_DECREF(value); -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(double _value) -{ - LM_T(LmtJexl, ("adding to JEXL context list (double): %f", _value)); - PyObject* value = Py_BuildValue("d", _value); - PyList_Append(jexl_context, value); - Py_DECREF(value); -} - - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(bool _value) -{ - LM_T(LmtJexl, ("adding to JEXL context list (bool): %s", _value? "true" : "false")); - if (_value) - { - PyList_Append(jexl_context, Py_True); - } - else - { - PyList_Append(jexl_context, Py_False); - } -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(void) -{ - LM_T(LmtJexl, ("adding to JEXL context list (none)")); - PyList_Append(jexl_context, Py_None); -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(JexlContext _jexlContextP) -{ - // FIXME PR: implement a toString() method in JexlContext to be used here - //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyList_Append(jexl_context, _jexlContextP.get()); -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(JexlContextList _jexlContextList) -{ - // FIXME PR: implement a toString() method in JexlContextList to be used here - //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyList_Append(jexl_context, _jexlContextList.get()); -} - - - -/* **************************************************************************** -* -* JexlContextList::relesase - -*/ -void JexlContextList::release(void) -{ - // FIXME PR: this is not correct. Recursively release of the list object - Py_XDECREF(jexl_context); -} \ No newline at end of file diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index 8a5ca6a412..84a40bc6f8 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -131,7 +131,7 @@ typedef enum TraceLevels LmtNotImplemented, LmtCurlContext, LmtThreadpool, - LmtJexl, + LmtExpr, LmtOldInfo = 240, // old INFO traces moved to DEBUG in Orion 2.5.0 diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 725e159ed9..4bf143ba76 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1052,7 +1052,7 @@ void ContextAttribute::filterAndOrderMetadata * renderNgsiField true is used in custom notification payloads, which have some small differences * with regards to conventional rendering */ -std::string ContextAttribute::toJson(const std::vector& metadataFilter, bool renderNgsiField, JexlContext* jexlContextP) +std::string ContextAttribute::toJson(const std::vector& metadataFilter, bool renderNgsiField, ExprContextObject* exprContextObjectP) { JsonObjectHelper jh; @@ -1085,7 +1085,7 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi // of DB entities) may lead to NULL, so the check is needed if (childToRenderP != NULL) { - jh.addRaw("value", childToRenderP->toJson(jexlContextP)); + jh.addRaw("value", childToRenderP->toJson(exprContextObjectP)); } } else if (valueType == orion::ValueTypeNumber) @@ -1101,7 +1101,7 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi } else if (valueType == orion::ValueTypeString) { - jh.addRaw("value", smartStringValue(stringValue, jexlContextP, "null")); + jh.addRaw("value", smartStringValue(stringValue, exprContextObjectP, "null")); } else if (valueType == orion::ValueTypeBoolean) { @@ -1296,41 +1296,41 @@ std::string ContextAttribute::toJsonAsValue * * Pretty similar in structure to toJsonValue */ -void ContextAttribute::addToContext(JexlContext* jexlContextP) +void ContextAttribute::addToContext(ExprContextObject* exprContextObjectP) { if (compoundValueP != NULL) { if (valueType == orion::ValueTypeObject) { - jexlContextP->add(name, compoundValueP->toJexlContext()); + exprContextObjectP->add(name, compoundValueP->toExprContextObject()); } else // valueType == orion::ValueTypeVector { - jexlContextP->add(name, compoundValueP->toJexlContextList()); + exprContextObjectP->add(name, compoundValueP->toExprContextList()); } } else if (valueType == orion::ValueTypeNumber) { if ((type == DATE_TYPE) || (type == DATE_TYPE_ALT)) { - jexlContextP->add(name, toJsonString(isodate2str(numberValue))); + exprContextObjectP->add(name, toJsonString(isodate2str(numberValue))); } else // regular number { - jexlContextP->add(name, numberValue); + exprContextObjectP->add(name, numberValue); } } else if (valueType == orion::ValueTypeString) { - jexlContextP->add(name, toJsonString(stringValue)); + exprContextObjectP->add(name, toJsonString(stringValue)); } else if (valueType == orion::ValueTypeBoolean) { - jexlContextP->add(name, boolValue); + exprContextObjectP->add(name, boolValue); } else if (valueType == orion::ValueTypeNull) { - jexlContextP->add(name); + exprContextObjectP->add(name); } else if (valueType == orion::ValueTypeNotGiven) { diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index 6b65827d29..23b74c06f1 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -37,7 +37,7 @@ #include "parse/CompoundValueNode.h" #include "rest/HttpStatusCode.h" #include "mongoDriver/BSONObjBuilder.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" @@ -108,7 +108,7 @@ typedef struct ContextAttribute std::string toJsonV1AsNameString(bool comma); - std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, JexlContext* jexlContextP = NULL); + std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, ExprContextObject* exprContextObjectP = NULL); std::string toJsonValue(void); @@ -119,7 +119,7 @@ typedef struct ContextAttribute MimeType* outMimeTypeP, HttpStatusCode* scP); - void addToContext(JexlContext* jexlContextP); + void addToContext(ExprContextObject* exprContextObjectP); void release(void); std::string getName(void); diff --git a/src/lib/ngsi/ContextElementResponse.cpp b/src/lib/ngsi/ContextElementResponse.cpp index 262d0f2a4c..8f5dace5ca 100644 --- a/src/lib/ngsi/ContextElementResponse.cpp +++ b/src/lib/ngsi/ContextElementResponse.cpp @@ -215,12 +215,12 @@ std::string ContextElementResponse::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { std::string out; - out = entity.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, false, jexlContextP); + out = entity.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, false, exprContextObjectP); return out; } diff --git a/src/lib/ngsi/ContextElementResponse.h b/src/lib/ngsi/ContextElementResponse.h index 471a8e66e6..2394183081 100644 --- a/src/lib/ngsi/ContextElementResponse.h +++ b/src/lib/ngsi/ContextElementResponse.h @@ -33,7 +33,7 @@ #include "ngsi/StringList.h" #include "ngsi/ContextAttribute.h" #include "apiTypesV2/Entity.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" #include "mongoDriver/BSONObj.h" @@ -83,7 +83,7 @@ typedef struct ContextElementResponse const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContext); + ExprContextObject* exprContextObjectP); void applyUpdateOperators(void); diff --git a/src/lib/ngsi/ContextElementResponseVector.cpp b/src/lib/ngsi/ContextElementResponseVector.cpp index 03fd35741a..24f40121f4 100644 --- a/src/lib/ngsi/ContextElementResponseVector.cpp +++ b/src/lib/ngsi/ContextElementResponseVector.cpp @@ -119,14 +119,14 @@ std::string ContextElementResponseVector::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { JsonVectorHelper jvh; for (unsigned int ix = 0; ix < vec.size(); ++ix) { - jvh.addRaw(vec[ix]->toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP)); + jvh.addRaw(vec[ix]->toJson(renderFormat, attrsFilter, blacklist, metadataFilter, exprContextObjectP)); } return jvh.str(); diff --git a/src/lib/ngsi/ContextElementResponseVector.h b/src/lib/ngsi/ContextElementResponseVector.h index 16cb7afb7c..9a6102b990 100644 --- a/src/lib/ngsi/ContextElementResponseVector.h +++ b/src/lib/ngsi/ContextElementResponseVector.h @@ -31,7 +31,7 @@ #include "ngsi/ContextElementResponse.h" #include "apiTypesV2/EntityVector.h" #include "common/RenderFormat.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" @@ -61,7 +61,7 @@ typedef struct ContextElementResponseVector const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP); + ExprContextObject* exprContextObjectP); void push_back(ContextElementResponse* item); unsigned int size(void) const; ContextElementResponse* lookup(Entity* eP, HttpStatusCode code = SccNone); diff --git a/src/lib/ngsi10/NotifyContextRequest.cpp b/src/lib/ngsi10/NotifyContextRequest.cpp index 30122bfcb0..f1ae7167d9 100644 --- a/src/lib/ngsi10/NotifyContextRequest.cpp +++ b/src/lib/ngsi10/NotifyContextRequest.cpp @@ -80,7 +80,7 @@ std::string NotifyContextRequest::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { if ((renderFormat != NGSI_V2_NORMALIZED) && (renderFormat != NGSI_V2_KEYVALUES) && (renderFormat != NGSI_V2_VALUES) && (renderFormat != NGSI_V2_SIMPLIFIEDKEYVALUES) && (renderFormat != NGSI_V2_SIMPLIFIEDNORMALIZED)) @@ -100,7 +100,7 @@ std::string NotifyContextRequest::toJson else { std::string out; - out += contextElementResponseVector[0]->toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, jexlContextP); + out += contextElementResponseVector[0]->toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, exprContextObjectP); return out; } } @@ -114,7 +114,7 @@ std::string NotifyContextRequest::toJson else { std::string out; - out += contextElementResponseVector[0]->toJson(NGSI_V2_KEYVALUES, attrsFilter, blacklist, metadataFilter, jexlContextP); + out += contextElementResponseVector[0]->toJson(NGSI_V2_KEYVALUES, attrsFilter, blacklist, metadataFilter, exprContextObjectP); return out; } } @@ -123,7 +123,7 @@ std::string NotifyContextRequest::toJson JsonObjectHelper jh; jh.addString("subscriptionId", subscriptionId.get()); - jh.addRaw("data", contextElementResponseVector.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP)); + jh.addRaw("data", contextElementResponseVector.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, exprContextObjectP)); return jh.str(); } } diff --git a/src/lib/ngsi10/NotifyContextRequest.h b/src/lib/ngsi10/NotifyContextRequest.h index 80e4a19c8d..c0d179a1f3 100644 --- a/src/lib/ngsi10/NotifyContextRequest.h +++ b/src/lib/ngsi10/NotifyContextRequest.h @@ -53,7 +53,7 @@ typedef struct NotifyContextRequest const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP = NULL); + ExprContextObject* exprContextObjectP = NULL); std::string check(ApiVersion apiVersion, const std::string& predetectedError); void release(void); NotifyContextRequest* clone(void); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 4ab2b5262f..aedcaa1b21 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -121,7 +121,7 @@ static bool setPayload const std::string& notifPayload, const SubscriptionId& subscriptionId, Entity& en, - JexlContext* jexlContextP, + ExprContextObject* exprContextObjectP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -170,7 +170,7 @@ static bool setPayload } else { - if (!macroSubstitute(payloadP, notifPayload, jexlContextP, "", true)) + if (!macroSubstitute(payloadP, notifPayload, exprContextObjectP, "", true)) { return false; } @@ -194,39 +194,17 @@ static bool setPayload static bool setJsonPayload ( orion::CompoundValueNode* json, - JexlContext* jexlContextP, + ExprContextObject* exprContextObjectP, std::string* payloadP, std::string* mimeTypeP ) { - *payloadP = json->toJson(jexlContextP); + *payloadP = json->toJson(exprContextObjectP); *mimeTypeP = "application/json"; // this can be overriden by headers field return true; } -/* **************************************************************************** -* -* removeQuotes - -* -* Entity id and type are special. Different from a attribute, they are always -* strings and cannot take a number, boolean, etc. as value. -* -* FIXME PR: duplicated logic! (both " and ' in different points of the code) -*/ -inline std::string removeQuotes(std::string s) -{ - if (s[0] == '"') - { - return s.substr(1, s.size()-2); - } - else - { - return s; - } -} - - /* **************************************************************************** * @@ -239,7 +217,7 @@ static bool setNgsiPayload const Entity& ngsi, const SubscriptionId& subscriptionId, Entity& en, - JexlContext* jexlContextP, + ExprContextObject* exprContextObjectP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -258,7 +236,7 @@ static bool setNgsiPayload else { // If id is not found in the replacements macro, we use en.id. - effectiveId = removeQuotes(smartStringValue(ngsi.id, jexlContextP, '"' + en.id + '"')); + effectiveId = removeQuotes(smartStringValue(ngsi.id, exprContextObjectP, '"' + en.id + '"')); } std::string effectiveType; @@ -269,7 +247,7 @@ static bool setNgsiPayload else { // If type is not found in the replacements macro, we use en.type. - effectiveType = removeQuotes(smartStringValue(ngsi.type, jexlContextP, '"' + en.type + '"')); + effectiveType = removeQuotes(smartStringValue(ngsi.type, exprContextObjectP, '"' + en.type + '"')); } cer.entity.fill(effectiveId, effectiveType, en.isPattern, en.servicePath); @@ -295,11 +273,11 @@ static bool setNgsiPayload if ((renderFormat == NGSI_V2_SIMPLIFIEDNORMALIZED) || (renderFormat == NGSI_V2_SIMPLIFIEDKEYVALUES)) { - *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP); + *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, exprContextObjectP); } else { - *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, jexlContextP); + *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, exprContextObjectP); } return true; @@ -338,15 +316,15 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - JexlContext jexlContext; - jexlContext.add("id", en.id); - jexlContext.add("type", en.type); - jexlContext.add("service", tenant); - jexlContext.add("servicePath", en.servicePath); - jexlContext.add("authToken", xauthToken); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add("service", tenant); + exprContext.add("servicePath", en.servicePath); + exprContext.add("authToken", xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - en.attributeVector[ix]->addToContext(&jexlContext); + en.attributeVector[ix]->addToContext(&exprContext); } // @@ -373,10 +351,10 @@ static SenderThreadParams* buildSenderParamsCustom // 2. URL // std::string notifUrl = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.url : notification.mqttInfo.url); - if (macroSubstitute(&url, notifUrl, &jexlContext, "", true) == false) + if (macroSubstitute(&url, notifUrl, &exprContext, "", true) == false) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } @@ -390,27 +368,27 @@ static SenderThreadParams* buildSenderParamsCustom { bool includePayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.includePayload : notification.mqttInfo.includePayload); std::string notifPayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.payload : notification.mqttInfo.payload); - if (!setPayload(includePayload, notifPayload, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) + if (!setPayload(includePayload, notifPayload, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } } else if (customPayloadType == ngsiv2::CustomPayloadType::Json) { orion::CompoundValueNode* json = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.json : notification.mqttInfo.json); - setJsonPayload(json, &jexlContext, &payload, &mimeType); + setJsonPayload(json, &exprContext, &payload, &mimeType); renderFormat = NGSI_V2_CUSTOM; } else // customPayloadType == ngsiv2::CustomPayloadType::Ngsi { // Important to use const& for Entity here. Otherwise problems may occur in the object release logic const Entity& ngsi = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.ngsi : notification.mqttInfo.ngsi); - if (!setNgsiPayload(ngsi, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) + if (!setNgsiPayload(ngsi, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } mimeType = "application/json"; @@ -427,10 +405,10 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) + if ((macroSubstitute(&key, it->first, &exprContext, "", true) == false) || (macroSubstitute(&value, it->second, &exprContext, "", true) == false)) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } @@ -454,10 +432,10 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) + if ((macroSubstitute(&key, it->first, &exprContext, "", true) == false) || (macroSubstitute(&value, it->second, &exprContext, "", true) == false)) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } @@ -489,7 +467,7 @@ static SenderThreadParams* buildSenderParamsCustom if (!parseUrl(url, host, port, uriPath, protocol)) { LM_E(("Runtime Error (not sending notification: malformed URL: '%s')", url.c_str())); - jexlContext.release(); + exprContext.release(); return NULL; } @@ -521,10 +499,10 @@ static SenderThreadParams* buildSenderParamsCustom // 8. Topic (only in the case of MQTT notifications) if (notification.type == ngsiv2::MqttNotification) { - if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "", true) == false) + if (macroSubstitute(&topic, notification.mqttInfo.topic, &exprContext, "", true) == false) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } } @@ -560,7 +538,7 @@ static SenderThreadParams* buildSenderParamsCustom snprintf(suffix, sizeof(suffix), "%u", correlatorCounter); paramsP->fiwareCorrelator = fiwareCorrelator + "; cbnotif=" + suffix; - jexlContext.release(); + exprContext.release(); return paramsP; } diff --git a/src/lib/parse/CompoundValueNode.cpp b/src/lib/parse/CompoundValueNode.cpp index f9e45c1555..fdf0fa2008 100644 --- a/src/lib/parse/CompoundValueNode.cpp +++ b/src/lib/parse/CompoundValueNode.cpp @@ -612,7 +612,7 @@ bool CompoundValueNode::equal(const orion::BSONElement& be) * CompoundValueNode:toJson * */ -std::string CompoundValueNode::toJson(JexlContext* jexlContextP) +std::string CompoundValueNode::toJson(ExprContextObject* exprContextObjectP) { std::string out; JsonVectorHelper jvh; @@ -621,7 +621,7 @@ std::string CompoundValueNode::toJson(JexlContext* jexlContextP) switch (valueType) { case orion::ValueTypeString: - return smartStringValue(stringValue, jexlContextP, "null"); + return smartStringValue(stringValue, exprContextObjectP, "null"); case orion::ValueTypeNumber: return double2string(numberValue); @@ -635,14 +635,14 @@ std::string CompoundValueNode::toJson(JexlContext* jexlContextP) case orion::ValueTypeVector: for (unsigned int ix = 0; ix < childV.size(); ix++) { - jvh.addRaw(childV[ix]->toJson(jexlContextP)); + jvh.addRaw(childV[ix]->toJson(exprContextObjectP)); } return jvh.str(); case orion::ValueTypeObject: for (unsigned int ix = 0; ix < childV.size(); ix++) { - joh.addRaw(childV[ix]->name, childV[ix]->toJson(jexlContextP)); + joh.addRaw(childV[ix]->name, childV[ix]->toJson(exprContextObjectP)); } return joh.str(); @@ -660,39 +660,39 @@ std::string CompoundValueNode::toJson(JexlContext* jexlContextP) /* **************************************************************************** * -* CompoundValueNode:toJexlContext +* CompoundValueNode:toExprContextObject * */ -JexlContext CompoundValueNode::toJexlContext(void) +ExprContextObject CompoundValueNode::toExprContextObject(void) { - JexlContext jc; + ExprContextObject co; for (uint64_t ix = 0; ix < childV.size(); ++ix) { CompoundValueNode* child = childV[ix]; switch (child->valueType) { case orion::ValueTypeString: - jc.add(child->name, child->stringValue); + co.add(child->name, child->stringValue); break; case orion::ValueTypeNumber: - jc.add(child->name, child->numberValue); + co.add(child->name, child->numberValue); break; case orion::ValueTypeBoolean: - jc.add(child->name, child->boolValue); + co.add(child->name, child->boolValue); break; case orion::ValueTypeNull: - jc.add(child->name); + co.add(child->name); break; case orion::ValueTypeVector: - jc.add(child->name, child->toJexlContextList()); + co.add(child->name, child->toExprContextList()); break; case orion::ValueTypeObject: - jc.add(child->name, child->toJexlContext()); + co.add(child->name, child->toExprContextObject()); break; case orion::ValueTypeNotGiven: @@ -703,46 +703,46 @@ JexlContext CompoundValueNode::toJexlContext(void) LM_E(("Runtime Error (value type unknown (%s))", name.c_str())); } } - return jc; + return co; } /* **************************************************************************** * -* CompoundValueNode:toJexlContextList +* CompoundValueNode:toExprContextList * */ -JexlContextList CompoundValueNode::toJexlContextList(void) +ExprContextList CompoundValueNode::toExprContextList(void) { - JexlContextList jcl; + ExprContextList cl; for (uint64_t ix = 0; ix < childV.size(); ++ix) { CompoundValueNode* child = childV[ix]; switch (child->valueType) { case orion::ValueTypeString: - jcl.add(child->stringValue); + cl.add(child->stringValue); break; case orion::ValueTypeNumber: - jcl.add(child->numberValue); + cl.add(child->numberValue); break; case orion::ValueTypeBoolean: - jcl.add(child->boolValue); + cl.add(child->boolValue); break; case orion::ValueTypeNull: - jcl.add(); + cl.add(); break; case orion::ValueTypeVector: - jcl.add(child->toJexlContextList()); + cl.add(child->toExprContextList()); break; case orion::ValueTypeObject: - jcl.add(child->toJexlContext()); + cl.add(child->toExprContextObject()); break; case orion::ValueTypeNotGiven: @@ -753,7 +753,7 @@ JexlContextList CompoundValueNode::toJexlContextList(void) LM_E(("Runtime Error (value type unknown)")); } } - return jcl; + return cl; } diff --git a/src/lib/parse/CompoundValueNode.h b/src/lib/parse/CompoundValueNode.h index a77eb1a393..bd15aefe64 100644 --- a/src/lib/parse/CompoundValueNode.h +++ b/src/lib/parse/CompoundValueNode.h @@ -35,7 +35,7 @@ #include "mongoDriver/BSONElement.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" namespace orion @@ -119,10 +119,10 @@ class CompoundValueNode bool equal(const orion::BSONElement& be); std::string finish(void); - std::string toJson(JexlContext* jexlContextP = NULL); + std::string toJson(ExprContextObject* exprContextObjectP = NULL); - JexlContext toJexlContext(void); - JexlContextList toJexlContextList(void); + ExprContextObject toExprContextObject(void); + ExprContextList toExprContextList(void); void shortShow(const std::string& indent); void show(const std::string& indent); From 49a3694ff2639006c8d3a93c42d15ac8f9230d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 16:34:58 +0100 Subject: [PATCH 027/104] FIX refactor common code and fix utest --- src/lib/expressions/CMakeLists.txt | 2 + src/lib/expressions/ExprContext.h | 6 +- src/lib/expressions/ExprManager.cpp | 52 +--------------- src/lib/expressions/ExprManager.h | 7 +-- src/lib/expressions/ExprResult.cpp | 39 +----------- src/lib/expressions/ExprResult.h | 6 +- src/lib/expressions/exprCommon.cpp | 60 +++++++++++++++++++ src/lib/expressions/exprCommon.h | 41 +++++++++++++ src/lib/expressions/exprMgr.h | 6 +- test/unittests/CMakeLists.txt | 1 + .../common/commonMacroSubstitute_test.cpp | 36 +++++++---- test/unittests/main_UnitTest.cpp | 2 + 12 files changed, 144 insertions(+), 114 deletions(-) create mode 100644 src/lib/expressions/exprCommon.cpp create mode 100644 src/lib/expressions/exprCommon.h diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index 4f938e3634..bfdd9da83c 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -21,6 +21,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES + exprCommon.cpp ExprManager.cpp ExprContext.cpp ExprResult.cpp @@ -28,6 +29,7 @@ SET (SOURCES ) SET (HEADERS + exprCommon.h ExprManager.h ExprContext.h ExprResult.h diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 4db34f562c..365eb65905 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_EXPRCONTEXT_H_ -#define SRC_LIB_JEXL_EXPRCONTEXT_H_ +#ifndef SRC_LIB_EXPRESSIONS_EXPRCONTEXT_H_ +#define SRC_LIB_EXPRESSIONS_EXPRCONTEXT_H_ /* * @@ -73,4 +73,4 @@ class ExprContextList }; -#endif // #define SRC_LIB_JEXL_EXPRCONTEXT_H_ +#endif // SRC_LIB_EXPRESSIONS_EXPRCONTEXT_H_ diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 10739d6a1d..985f1a802a 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -27,48 +27,13 @@ #include "expressions/ExprManager.h" #include "expressions/ExprResult.h" +#include "expressions/exprCommon.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" -/* **************************************************************************** -* -* capturePythonError - -*/ -static const char* capturePythonError() -{ - if (PyErr_Occurred()) - { - PyObject* ptype; - PyObject* pvalue; - PyObject* ptraceback; - - // Fetch the exception type, value, and traceback - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - - if (pvalue != NULL) - { - PyObject* str_obj = PyObject_Str(pvalue); - const char* error_message = PyUnicode_AsUTF8(str_obj); - - // Release the Python objects - Py_XDECREF(str_obj); - Py_XDECREF(ptype); - Py_XDECREF(pvalue); - Py_XDECREF(ptraceback); - - return error_message; - } - } - - PyErr_Clear(); - return ""; -} - - - /* **************************************************************************** * * ExprManager::init - @@ -76,7 +41,6 @@ static const char* capturePythonError() void ExprManager::init(void) { pyjexlModule = NULL; - jsonModule = NULL; jexlEngine = NULL; if (sem_init(&sem, 0, 1) == -1) @@ -95,14 +59,6 @@ void ExprManager::init(void) } LM_T(LmtExpr, ("pyjexl module has been loaded")); - jsonModule = PyImport_ImportModule("json"); - if (jsonModule == NULL) - { - const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error importing json module: %s)", error)); - } - LM_T(LmtExpr, ("json module has been loaded")); - jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); if (jexlEngine == NULL) { @@ -170,12 +126,6 @@ void ExprManager::release(void) LM_T(LmtExpr, ("pyjexl module has been freed")); } - if (jsonModule != NULL) - { - Py_XDECREF(jsonModule); - LM_T(LmtExpr, ("json module has been freed")); - } - Py_Finalize(); LM_T(LmtExpr, ("Python interpreter has been finalized")); } \ No newline at end of file diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 7466372b29..92525900e0 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_EXPRMANAGER_H_ -#define SRC_LIB_JEXL_EXPRMANAGER_H_ +#ifndef SRC_LIB_EXPRESSIONS_EXPRMANAGER_H_ +#define SRC_LIB_EXPRESSIONS_EXPRMANAGER_H_ /* * @@ -42,7 +42,6 @@ class ExprManager PyObject* pyjexlModule; PyObject* jexlEngine; PyObject* jsonModule; - //PyObject* customJsonSerializer; sem_t sem; public: @@ -51,4 +50,4 @@ class ExprManager void release(void); }; -#endif // SRC_LIB_JEXL_EXPRMANAGER_H_ \ No newline at end of file +#endif // SRC_LIB_EXPRESSIONS_EXPRMANAGER_H_ \ No newline at end of file diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 036651fdc4..71dcd6c336 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -24,6 +24,7 @@ */ #include "expressions/ExprResult.h" +#include "expressions/exprCommon.h" #include "common/string.h" #include "common/JsonHelper.h" @@ -31,44 +32,6 @@ -/* **************************************************************************** -* -* capturePythonError - -* -* FIXME PR: duplicate code. Unify -*/ -static const char* capturePythonError() -{ - if (PyErr_Occurred()) - { - PyObject* ptype; - PyObject* pvalue; - PyObject* ptraceback; - - // Fetch the exception type, value, and traceback - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - - if (pvalue != NULL) - { - PyObject* str_obj = PyObject_Str(pvalue); - const char* error_message = PyUnicode_AsUTF8(str_obj); - - // Release the Python objects - Py_XDECREF(str_obj); - Py_XDECREF(ptype); - Py_XDECREF(pvalue); - Py_XDECREF(ptraceback); - - return error_message; - } - } - - PyErr_Clear(); - return ""; -} - - - /* **************************************************************************** * * getPyObjectType - diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index bfa4bcbbb7..c29ffd5c49 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_EXPRRESULT_H_ -#define SRC_LIB_JEXL_EXPRRESULT_H_ +#ifndef SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ +#define SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ /* * @@ -61,4 +61,4 @@ class ExprResult -#endif // #define SRC_LIB_JEXL_EXPRRESULT_H_ +#endif // SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ diff --git a/src/lib/expressions/exprCommon.cpp b/src/lib/expressions/exprCommon.cpp new file mode 100644 index 0000000000..1372d73eaa --- /dev/null +++ b/src/lib/expressions/exprCommon.cpp @@ -0,0 +1,60 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "expressions/exprCommon.h" + +/* **************************************************************************** +* +* capturePythonError - +*/ +const char* capturePythonError() +{ + if (PyErr_Occurred()) + { + PyObject* ptype; + PyObject* pvalue; + PyObject* ptraceback; + + // Fetch the exception type, value, and traceback + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + + if (pvalue != NULL) + { + PyObject* strObj = PyObject_Str(pvalue); + const char* errorMessage = PyUnicode_AsUTF8(strObj); + + // Release the Python objects + Py_XDECREF(strObj); + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + + return errorMessage; + } + } + + PyErr_Clear(); + return ""; +} \ No newline at end of file diff --git a/src/lib/expressions/exprCommon.h b/src/lib/expressions/exprCommon.h new file mode 100644 index 0000000000..7b9667d1b4 --- /dev/null +++ b/src/lib/expressions/exprCommon.h @@ -0,0 +1,41 @@ +#ifndef SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ +#define SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include + + + +/* **************************************************************************** +* +* capturePythonError - +*/ +extern const char* capturePythonError(); + + + +#endif // SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ diff --git a/src/lib/expressions/exprMgr.h b/src/lib/expressions/exprMgr.h index 852e396e24..5b57b3feb8 100644 --- a/src/lib/expressions/exprMgr.h +++ b/src/lib/expressions/exprMgr.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_EXPRMGR_H_ -#define SRC_LIB_JEXL_EXPRMGR_H_ +#ifndef SRC_LIB_EXPRESSIONS_EXPRMGR_H_ +#define SRC_LIB_EXPRESSIONS_EXPRMGR_H_ /* * @@ -35,4 +35,4 @@ */ extern ExprManager exprMgr; -#endif // SRC_LIB_JEXL_EXPRMGR_H_ +#endif // SRC_LIB_EXPRESSIONS_EXPRMGR_H_ diff --git a/test/unittests/CMakeLists.txt b/test/unittests/CMakeLists.txt index af8fbaf709..a929c69b39 100644 --- a/test/unittests/CMakeLists.txt +++ b/test/unittests/CMakeLists.txt @@ -217,6 +217,7 @@ include_directories("${PROJECT_SOURCE_DIR}/test") # Needed for the new C driver # FIXME: why do we need this declared here if they are also declared # in the main CMakeFiles.txt? +include_directories("/usr/include/python3.11/") include_directories("/usr/local/include/libmongoc-1.0") include_directories("/usr/local/include/libbson-1.0") diff --git a/test/unittests/common/commonMacroSubstitute_test.cpp b/test/unittests/common/commonMacroSubstitute_test.cpp index 40812fb6fb..be51a1e8af 100644 --- a/test/unittests/common/commonMacroSubstitute_test.cpp +++ b/test/unittests/common/commonMacroSubstitute_test.cpp @@ -49,9 +49,12 @@ TEST(commonMacroSubstitute, simple) const char* correct = "Entity E1/T1, attribute 'attr1'"; std::string result; - std::map replacements; - buildReplacementsMap(en, "", "", &replacements); - b = macroSubstitute(&result, s1, &replacements, ""); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add(caP->name, caP->stringValue); + + b = macroSubstitute(&result, s1, &exprContext, "", true); EXPECT_TRUE(b); EXPECT_STREQ(correct, result.c_str()); } @@ -88,9 +91,12 @@ TEST(commonMacroSubstitute, withRealloc) std::string correct = std::string(base) + "Now, finally something to substitute: Entity E1/T1, attribute 'attr1'"; std::string result; - std::map replacements; - buildReplacementsMap(en, "", "", &replacements); - b = macroSubstitute(&result, s1, &replacements, ""); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add(caP->name, caP->stringValue); + + b = macroSubstitute(&result, s1, &exprContext, "", true); EXPECT_TRUE(b); EXPECT_STREQ(correct.c_str(), result.c_str()); } @@ -120,9 +126,12 @@ TEST(commonMacroSubstitute, bufferTooBigInitially) // correct = std::string(base) + "EntityId000001/EntityType000001"; std::string result; - std::map replacements; - buildReplacementsMap(en, "", "", &replacements); - b = macroSubstitute(&result, s1, &replacements, ""); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add(caP->name, caP->stringValue); + + b = macroSubstitute(&result, s1, &exprContext, "", true); EXPECT_FALSE(b); EXPECT_STREQ("", result.c_str()); @@ -158,9 +167,12 @@ TEST(commonMacroSubstitute, bufferTooBigAfterSubstitution) // correct = std::string(base) + "EntityId000001/EntityType000001"; // > 8MB after substitutions std::string result; - std::map replacements; - buildReplacementsMap(en, "", "", &replacements); - b = macroSubstitute(&result, s1, &replacements, ""); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add(caP->name, caP->stringValue); + + b = macroSubstitute(&result, s1, &exprContext, "", true); EXPECT_FALSE(b); EXPECT_STREQ("", result.c_str()); diff --git a/test/unittests/main_UnitTest.cpp b/test/unittests/main_UnitTest.cpp index 7d7b9bc991..8fa3ad41b7 100644 --- a/test/unittests/main_UnitTest.cpp +++ b/test/unittests/main_UnitTest.cpp @@ -42,6 +42,7 @@ #include "mongoBackend/MongoGlobal.h" #include "ngsiNotify/Notifier.h" #include "alarmMgr/alarmMgr.h" +#include "expressions/exprMgr.h" #include "logSummary/logSummary.h" #include "unittests/unittest.h" @@ -162,6 +163,7 @@ int main(int argC, char** argV) // Note that disableRetryTries, multitenancy and mutex time stats are disabled for unit test mongo init mongoInit(dbURI, dbHost, rplSet, dbName, user, pwd, authMech, authDb, dbSSL, false, false, dbTimeout, writeConcern, dbPoolSize, false); alarmMgr.init(false); + exprMgr.init(); logSummaryInit(&lsPeriod); // setupDatabase(); FIXME #3775: pending on mongo unit test re-enabling From 8417f19b558c763f1dceecbc427142b6c93cb3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 17:01:23 +0100 Subject: [PATCH 028/104] FIX enable venv in ci image utest --- ci/deb/makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/deb/makefile b/ci/deb/makefile index 4cbc001c5d..3e8217c3d3 100644 --- a/ci/deb/makefile +++ b/ci/deb/makefile @@ -69,6 +69,9 @@ install_unit: install unit: @echo '------------------------------------- make unit ----------------------------------------' + # ft_env needed for pyjexl module. Very important to "chain" the two commands with '; \' + # otherwise it seems unitTest doesn't get the venv and it uses default Python in the system + . /opt/ft_env/bin/activate ; \ BUILD_UNIT/test/unittests/unitTest -t 0-255 -dbhost ${MONGO_HOST} --gtest_output=xml:/tmp/builder/logs/unit.xml build_functional: prepare From 8cebfddeba47103a502bf6f3704294d427dc21d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 17:06:24 +0100 Subject: [PATCH 029/104] FIX improvement --- src/lib/expressions/ExprContext.cpp | 147 +++++++++++++++++++++++----- src/lib/expressions/ExprContext.h | 7 +- src/lib/expressions/ExprManager.cpp | 6 +- src/lib/expressions/ExprResult.cpp | 12 +-- 4 files changed, 134 insertions(+), 38 deletions(-) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 8c05fd5764..c5070d486f 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -23,12 +23,11 @@ * Author: Fermin Galan */ -// FIXME PR: add methods should check error in Py_BuildValue() for NULL - #include #include "logMsg/logMsg.h" #include "expressions/ExprContext.h" +#include "expressions/ExprCommon.h" @@ -36,15 +35,15 @@ * * ExprContextObject::ExprContextObject - */ -ExprContextObject::ExprContextObject -() +ExprContextObject::ExprContextObject() { jexlContext = PyDict_New(); if (jexlContext == NULL) { - // FIXME PR: error control - // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + // Note that this ruins the object. We log this situation just one time here, but we include a NULL check + // in every other method to be safer + LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); } } @@ -67,8 +66,17 @@ PyObject* ExprContextObject::get(void) */ void ExprContextObject::add(const std::string& key, const std::string& _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); + return; + } PyDict_SetItemString(jexlContext, key.c_str(), value); Py_DECREF(value); } @@ -81,8 +89,17 @@ void ExprContextObject::add(const std::string& key, const std::string& _value) */ void ExprContextObject::add(const std::string& key, double _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); PyObject* value = Py_BuildValue("d", _value); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); + return; + } PyDict_SetItemString(jexlContext, key.c_str(), value); Py_DECREF(value); } @@ -96,6 +113,10 @@ void ExprContextObject::add(const std::string& key, double _value) */ void ExprContextObject::add(const std::string& key, bool _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value? "true" : "false")); if (_value) { @@ -115,6 +136,10 @@ void ExprContextObject::add(const std::string& key, bool _value) */ void ExprContextObject::add(const std::string& key) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); PyDict_SetItemString(jexlContext, key.c_str(), Py_None); } @@ -127,8 +152,11 @@ void ExprContextObject::add(const std::string& key) */ void ExprContextObject::add(const std::string& key, ExprContextObject exprContextObject) { - // FIXME PR: implement a toString() method in ExprContextObject to be used here - //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (object): %s=%s", key.c_str(), exprContextObject.toString().c_str())); PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.get()); } @@ -140,8 +168,11 @@ void ExprContextObject::add(const std::string& key, ExprContextObject exprContex */ void ExprContextObject::add(const std::string& key, ExprContextList exprContextList) { - // FIXME PR: implement a toString() method in ExprContextList to be used here - //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (list): %s=%s", key.c_str(), exprContextList.toString().c_str())); PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); } @@ -149,16 +180,20 @@ void ExprContextObject::add(const std::string& key, ExprContextList exprContextL /* **************************************************************************** * -* ExprContextObject::hasKey - +* ExprContextObject::toString - */ -bool ExprContextObject::hasKey(const std::string& key) +std::string ExprContextObject::toString(void) { - // Check if the key exists in the jexlContext dictionary - PyObject* keyObject = PyUnicode_FromString(key.c_str()); - int result = PyDict_Contains(jexlContext, keyObject); - Py_DECREF(keyObject); - - return result == 1; // Return true if key exists, false otherwise + const char* str = PyUnicode_AsUTF8(jexlContext); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + return ""; + } + else + { + return std::string(str); + } } @@ -169,11 +204,16 @@ bool ExprContextObject::hasKey(const std::string& key) */ void ExprContextObject::release(void) { + if (jexlContext == NULL) + { + return; + } // FIXME PR: this is not correct. Recursively release of the dict object //Py_XDECREF(jexlContext); } + /* **************************************************************************** * * ExprContextList::ExprContextList - @@ -184,8 +224,9 @@ ExprContextList::ExprContextList() if (jexlContext == NULL) { - // FIXME PR: error control - // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + // Note that this ruins the object. We log this situation just one time here, but we include a NULL check + // in every other method to be safer + LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); } } @@ -208,8 +249,17 @@ PyObject* ExprContextList::get(void) */ void ExprContextList::add(const std::string& _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context list (string): %s=", _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); + return; + } PyList_Append(jexlContext, value); Py_DECREF(value); } @@ -222,8 +272,17 @@ void ExprContextList::add(const std::string& _value) */ void ExprContextList::add(double _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context list (double): %f", _value)); PyObject* value = Py_BuildValue("d", _value); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); + return; + } PyList_Append(jexlContext, value); Py_DECREF(value); } @@ -237,6 +296,10 @@ void ExprContextList::add(double _value) */ void ExprContextList::add(bool _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value? "true" : "false")); if (_value) { @@ -256,6 +319,10 @@ void ExprContextList::add(bool _value) */ void ExprContextList::add(void) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context list (none)")); PyList_Append(jexlContext, Py_None); } @@ -268,8 +335,11 @@ void ExprContextList::add(void) */ void ExprContextList::add(ExprContextObject exprContextObject) { - // FIXME PR: implement a toString() method in ExprContext to be used here - //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context list (object): %s", exprContextObject.toString().c_str())); PyList_Append(jexlContext, exprContextObject.get()); } @@ -281,19 +351,46 @@ void ExprContextList::add(ExprContextObject exprContextObject) */ void ExprContextList::add(ExprContextList exprContextList) { - // FIXME PR: implement a toString() method in ExprContextList to be used here - //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context list (list): %s", exprContextList.toString().c_str())); PyList_Append(jexlContext, exprContextList.get()); } +/* **************************************************************************** +* +* ExprContextList::toString - +*/ +std::string ExprContextList::toString(void) +{ + const char* str = PyUnicode_AsUTF8(jexlContext); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + return ""; + } + else + { + return std::string(str); + } +} + + + /* **************************************************************************** * * ExprContextList::relesase - */ void ExprContextList::release(void) { + if (jexlContext == NULL) + { + return; + } // FIXME PR: this is not correct. Recursively release of the list object - Py_XDECREF(jexlContext); + //Py_XDECREF(jexlContext); } \ No newline at end of file diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 365eb65905..6b41dc7bca 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -50,7 +50,9 @@ class ExprContextObject void add(const std::string& key); void add(const std::string& key, ExprContextObject exprContextObject); void add(const std::string& key, ExprContextList exprContextList); - bool hasKey(const std::string& key); + + std::string toString(void); + void release(void); }; @@ -69,6 +71,9 @@ class ExprContextList void add(void); void add(ExprContextObject exprContextObject); void add(ExprContextList exprContextList); + + std::string toString(void); + void release(void); }; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 985f1a802a..d6aa98d5bd 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -84,8 +84,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error building expression: %s", capturePythonError())); + LM_W(("error building JEXL expression: %s", capturePythonError())); return r; } @@ -93,8 +92,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st Py_XDECREF(expression); if (result == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error evaluating expression: %s", capturePythonError())); + LM_W(("error evaluating JEXL expression: %s", capturePythonError())); return r; } diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 71dcd6c336..f2b8d944c6 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -131,8 +131,7 @@ void ExprResult::fill(PyObject* result) const char* str = PyUnicode_AsUTF8(result); if (str == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); valueType = orion::ValueTypeNull; } else @@ -167,8 +166,7 @@ void ExprResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va str = PyUnicode_AsUTF8(value); if (str == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); } else { @@ -243,8 +241,7 @@ void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke const char * keyStr = PyUnicode_AsUTF8(key); if (keyStr == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); return; } @@ -261,8 +258,7 @@ void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke str = PyUnicode_AsUTF8(value); if (str == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); } else { From 0beb957c2c1ebb8f56189bedc25e57a98c273607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 17:34:44 +0100 Subject: [PATCH 030/104] FIX refactor --- src/lib/expressions/ExprContext.cpp | 2 +- src/lib/expressions/ExprResult.cpp | 148 ++++++++++++++-------------- src/lib/expressions/ExprResult.h | 3 - 3 files changed, 75 insertions(+), 78 deletions(-) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index c5070d486f..41ec5b3ddb 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -27,7 +27,7 @@ #include "logMsg/logMsg.h" #include "expressions/ExprContext.h" -#include "expressions/ExprCommon.h" +#include "expressions/exprCommon.h" diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index f2b8d944c6..986ca52e53 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -72,85 +72,15 @@ static orion::ValueType getPyObjectType(PyObject* obj) } - -/* **************************************************************************** -* -* fill - -* -*/ -void ExprResult::fill(PyObject* result) -{ - // If nothing changes, the returned value would be null (failsafe) - valueType = orion::ValueTypeNull; - - // Special case: expresion evalutes to None - if (result == Py_None) - { - LM_T(LmtExpr, ("ExprResult is null")); - valueType = orion::ValueTypeNull; - return; - } - - // Other not null types - valueType = getPyObjectType(result); - if (valueType == orion::ValueTypeNumber) - { - numberValue = PyFloat_AsDouble(result); - LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); - } - else if (valueType == orion::ValueTypeBoolean) - { - boolValue = PyObject_IsTrue(result); - LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); - } - else if (valueType == orion::ValueTypeObject) - { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(result, &pos, &key, &value)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(compoundValueP, key, value); - } - } - else if (valueType == orion::ValueTypeVector) - { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); - Py_ssize_t size = PyList_Size(result); - for (Py_ssize_t ix = 0; ix < size; ++ix) - { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(compoundValueP, PyList_GetItem(result, ix)); - } - } - else if (valueType == orion::ValueTypeString) - { - const char* str = PyUnicode_AsUTF8(result); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtExpr, ("ExprResult (string): %s", str)); - stringValue = std::string(str); - } - } -} - +static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); // forward declaration /* **************************************************************************** * * processListItem - * -* FIXME PR: maybe this should be static function out of the class? */ -void ExprResult::processListItem(orion::CompoundValueNode* parentP, PyObject* value) +void processListItem(orion::CompoundValueNode* parentP, PyObject* value) { orion::CompoundValueNode* nodeP; @@ -234,9 +164,8 @@ void ExprResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va * * processDictItem - * -* FIXME PR: maybe this should be static function out of the class? */ -void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) { const char * keyStr = PyUnicode_AsUTF8(key); if (keyStr == NULL) @@ -322,6 +251,77 @@ void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke +/* **************************************************************************** +* +* fill - +* +*/ +void ExprResult::fill(PyObject* result) +{ + // If nothing changes, the returned value would be null (failsafe) + valueType = orion::ValueTypeNull; + + // Special case: expresion evalutes to None + if (result == Py_None) + { + LM_T(LmtExpr, ("ExprResult is null")); + valueType = orion::ValueTypeNull; + return; + } + + // Other not null types + valueType = getPyObjectType(result); + if (valueType == orion::ValueTypeNumber) + { + numberValue = PyFloat_AsDouble(result); + LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); + } + else if (valueType == orion::ValueTypeBoolean) + { + boolValue = PyObject_IsTrue(result); + LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); + } + else if (valueType == orion::ValueTypeObject) + { + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(result, &pos, &key, &value)) + { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() + processDictItem(compoundValueP, key, value); + } + } + else if (valueType == orion::ValueTypeVector) + { + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); + Py_ssize_t size = PyList_Size(result); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(compoundValueP, PyList_GetItem(result, ix)); + } + } + else if (valueType == orion::ValueTypeString) + { + const char* str = PyUnicode_AsUTF8(result); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtExpr, ("ExprResult (string): %s", str)); + stringValue = std::string(str); + } + } +} + + + /* **************************************************************************** * * toString - diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index c29ffd5c49..df1ec0f597 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -52,9 +52,6 @@ class ExprResult void fill(PyObject* result); - void processListItem(orion::CompoundValueNode* parentP, PyObject* item); - void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); - std::string toString(void); void release(void); }; From 6f77ee597953e6f0c1e69de5b417bbd99cc542c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 17:55:40 +0100 Subject: [PATCH 031/104] FIX indent --- src/lib/apiTypesV2/Entity.cpp | 4 ++-- src/lib/apiTypesV2/Entity.h | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/apiTypesV2/Entity.cpp b/src/lib/apiTypesV2/Entity.cpp index f864e0bcb2..f51cbd6b35 100644 --- a/src/lib/apiTypesV2/Entity.cpp +++ b/src/lib/apiTypesV2/Entity.cpp @@ -308,7 +308,7 @@ std::string Entity::toJson bool blacklist, const std::vector& metadataFilter, bool renderNgsiField, - ExprContextObject* exprContextObjectP + ExprContextObject* exprContextObjectP ) { std::vector orderedAttrs; @@ -444,7 +444,7 @@ std::string Entity::toJsonNormalized const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField, - ExprContextObject* exprContextObjectP + ExprContextObject* exprContextObjectP ) { JsonObjectHelper jh; diff --git a/src/lib/apiTypesV2/Entity.h b/src/lib/apiTypesV2/Entity.h index f8acc3af97..9d534d52ed 100644 --- a/src/lib/apiTypesV2/Entity.h +++ b/src/lib/apiTypesV2/Entity.h @@ -91,8 +91,8 @@ class Entity const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - bool renderNgsiField = false, - ExprContextObject* exprContext = NULL); + bool renderNgsiField = false, + ExprContextObject* exprContextObjectP = NULL); std::string toJson(RenderFormat renderFormat, bool renderNgsiField = false); @@ -145,8 +145,8 @@ class Entity std::string toJsonKeyvalues(const std::vector& orderedAttrs); std::string toJsonNormalized(const std::vector& orderedAttrs, const std::vector& metadataFilter, - bool renderNgsiField = false, - ExprContextObject* exprContext = NULL); + bool renderNgsiField = false, + ExprContextObject* exprContextObject = NULL); }; #endif // SRC_LIB_APITYPESV2_ENTITY_H_ From b56cd7e9062214c35a8ddcc519fcd5eb8bd31ee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 21 Feb 2024 13:25:21 +0100 Subject: [PATCH 032/104] FIX add libpython to version operation --- .../cases/0000_version_operation/version_via_rest.test | 1 + test/functionalTest/cases/0501_cors/version_request.test | 3 +++ .../cases/1916_fiware_correlator/fiware_correlator.test | 2 ++ test/unittests/serviceRoutines/versionTreat_test.cpp | 2 ++ 4 files changed, 8 insertions(+) diff --git a/test/functionalTest/cases/0000_version_operation/version_via_rest.test b/test/functionalTest/cases/0000_version_operation/version_via_rest.test index ddd58071be..e4fd1517f7 100644 --- a/test/functionalTest/cases/0000_version_operation/version_via_rest.test +++ b/test/functionalTest/cases/0000_version_operation/version_via_rest.test @@ -64,6 +64,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/functionalTest/cases/0501_cors/version_request.test b/test/functionalTest/cases/0501_cors/version_request.test index 2cdcafe8a1..57b963b40a 100644 --- a/test/functionalTest/cases/0501_cors/version_request.test +++ b/test/functionalTest/cases/0501_cors/version_request.test @@ -94,6 +94,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -130,6 +131,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -164,6 +166,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test index 5e52e7eea5..01cd198b53 100644 --- a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test +++ b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test @@ -73,6 +73,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -107,6 +108,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/unittests/serviceRoutines/versionTreat_test.cpp b/test/unittests/serviceRoutines/versionTreat_test.cpp index 9950e5d6f0..fd8a4e2e4b 100644 --- a/test/unittests/serviceRoutines/versionTreat_test.cpp +++ b/test/unittests/serviceRoutines/versionTreat_test.cpp @@ -94,6 +94,8 @@ TEST(versionTreat, ok) EXPECT_TRUE(strstr(out.c_str(), "boost") != NULL); EXPECT_TRUE(strstr(out.c_str(), "libcurl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "libmicrohttpd") != NULL); + EXPECT_TRUE(strstr(out.c_str(), "libmosquitto") != NULL); + EXPECT_TRUE(strstr(out.c_str(), "libpython") != NULL); EXPECT_TRUE(strstr(out.c_str(), "openssl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "rapidjson") != NULL); EXPECT_TRUE(strstr(out.c_str(), "mongoc") != NULL); From df3e8ddc3bed53fe17d9c7fc2801e4d819949cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 21 Feb 2024 13:51:30 +0100 Subject: [PATCH 033/104] FIX version operation --- src/lib/serviceRoutines/versionTreat.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index 981c81f2b6..5ea06218d9 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -46,6 +46,7 @@ #include #include #include +#include /* **************************************************************************** * @@ -70,6 +71,7 @@ std::string libVersions(void) std::string mhd = " \"libmicrohttpd\": "; std::string ssl = " \"openssl\": "; std::string rjson = " \"rapidjson\": "; + std::string python = " \"libpython\": "; std::string mongo = " \"mongoc\": "; std::string bson = " \"bson\": "; @@ -84,10 +86,17 @@ std::string libVersions(void) char mosqVersion[16]; snprintf(mosqVersion, sizeof(mosqVersion), "%d.%d.%d", mosqMayor, mosqMinor, mosqRevision); + char pyVersion[16]; + snprintf(pyVersion, sizeof(pyVersion), "%d.%d.%d", + (PY_VERSION_HEX >> 24) & 0xFF, + (PY_VERSION_HEX >> 16) & 0xFF, + (PY_VERSION_HEX >> 8) & 0xFF); + total += boost + "\"" + BOOST_LIB_VERSION "\"" + ",\n"; total += curl + "\"" + curlVersion + "\"" + ",\n"; total += mosq + "\"" + mosqVersion + "\"" + ",\n"; total += mhd + "\"" + MHD_get_version() + "\"" + ",\n"; + total += python + "\"" + pyVersion + "\"" + ",\n"; #ifdef OLD_SSL_VERSION_FORMAT // Needed by openssl 1.1.1n in Debian 11 and before total += ssl + "\"" + SHLIB_VERSION_NUMBER "\"" + ",\n"; From a8411ab4b7baf20c074843db8bc46bd3895e0c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 21 Feb 2024 16:06:33 +0100 Subject: [PATCH 034/104] FIX new ftest --- ...c.test => jexl_basic_attrs_in_update.test} | 12 +- .../jexl_basic_attrs_not_in_update.test | 173 ++++++++++++++ .../jexl_expr_attrs_weird_syntax.test | 213 ++++++++++++++++++ .../jexl_json_navigation.test | 199 ++++++++++++++++ 4 files changed, 591 insertions(+), 6 deletions(-) rename test/functionalTest/cases/4004_jexl_expressions_in_subs/{jexl_basic.test => jexl_basic_attrs_in_update.test} (94%) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test similarity index 94% rename from test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test rename to test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test index 209b61474c..5e031e80e7 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification +JEXL expression in custom notification (source attributes in udpate) --SHELL-INIT-- dbInit CB @@ -35,7 +35,7 @@ accumulatorStart --pretty-print # 02. Create entity E1 with A=1 and B=2 # 03. Update entity E1 with A=foo and B=bar # 04. Update entity E1 with A=2.1 and B=-3.8 -# 05. Dump accumulator and see two notifications (sum: 3 and sum: foobar) +# 05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7) # @@ -120,8 +120,8 @@ echo echo -echo "05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1,7)" -echo "=================================================================================" +echo "05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7)" +echo "===============================================================================" accumulatorDump echo echo @@ -164,8 +164,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1,7) -================================================================================= +05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7) +=============================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 221 diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test new file mode 100644 index 0000000000..eaef9821eb --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test @@ -0,0 +1,173 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (source attributes not in udpate) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression sum: A+B +# 02. Create entity E1 with A=1 and B=2 +# 03. Update entity E1 with C=foo to trigger notification +# 04. Dump accumulator and see notifications (sum: 3) +# + + +echo "01. Create custom sub with custom expression sum: A+B" +echo "=====================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ], + "condition": { + "attrs": [ "C" ] + } + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "sum": { + "value": "${A+B}", + "type": "Calculated" + } + } + }, + "attrs": [ "sum" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1 and B=2" +echo "=====================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with C=foo to trigger notification" +echo "=======================================================" +payload='{ + "C": { + "value": "foo", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (sum: 3)" +echo "===================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: A+B +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 and B=2 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with C=foo to trigger notification +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (sum: 3) +=================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 129 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test new file mode 100644 index 0000000000..3390b28e82 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -0,0 +1,213 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Legacy expression using attributes with weird syntax + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression: A-B and A:B +# 02. Create entity E1 with A-B 1 and A:B 2 +# 03. Update entity E1 with A-B 3 and A:B 4 +# 04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null) +# + + +echo "01. Create custom sub with custom expression: A-B and A:B" +echo "=========================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "R1": { + "value": "${A-B}", + "type": "Calculated" + }, + "R2": { + "value": "${A:B}", + "type": "Calculated" + } + } + }, + "attrs": [ "R1", "R2" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A-B 1 and A:B 2" +echo "=========================================" +payload='{ + "id": "E1", + "type": "T", + "A-B": { + "value": 1, + "type": "Number" + }, + "A:B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A-B 3 and A:B 4" +echo "=========================================" +payload='{ + "A-B": { + "value": 3, + "type": "Number" + }, + "A:B": { + "value": 4, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null)" +echo "==================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression: A-B and A:B +========================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A-B 1 and A:B 2 +========================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A-B 3 and A:B 4 +========================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null) +================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 185 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "R1": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "R2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 185 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "R1": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "R2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test new file mode 100644 index 0000000000..236b872365 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test @@ -0,0 +1,199 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (source attributes not in udpate) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression cal: A[0] + A[1].A1 + B.B1[2].C +# 02. Create entity E1 with A and B so cal result in 60 +# 03. Update entity E1 with A and B so cal result in 6 +# 04. Dump accumulator and see notifications (cal 60 and cal 6) +# + + +echo "01. Create custom sub with custom expression cal: A[0] + A[1].A1 + B.B1[2].C" +echo "============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "cal": { + "value": "${A[0] + A[1].A1 + B.B1[2].C}", + "type": "Calculated" + } + } + }, + "attrs": [ "cal" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A and B so cal result in 60" +echo "=====================================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": [ 10, { "A1": 20 } ], + "type": "StructuredValue" + }, + "B": { + "value": { "B1": [ 11, 22, { "C": 30 }]}, + "type": "StructuredValue" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A and B so cal result in 6" +echo "====================================================" +payload='{ + "A": { + "value": [ 1, { "A1": 2 } ], + "type": "StructuredValue" + }, + "B": { + "value": { "B1": [ 11, 22, { "C": 3 }]}, + "type": "StructuredValue" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (cal 60 and cal 6)" +echo "=============================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression cal: A[0] + A[1].A1 + B.B1[2].C +============================================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A and B so cal result in 60 +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A and B so cal result in 6 +==================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (cal 60 and cal 6) +============================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 130 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": 60 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 129 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": 6 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 92e92896358aaa61e150581758862f5562e6713f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 26 Feb 2024 17:22:28 +0100 Subject: [PATCH 035/104] ADD exprLang --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/admin/database_model.md | 1 + doc/manuals/orion-api.md | 2 + src/lib/apiTypesV2/HttpInfo.cpp | 8 +- src/lib/apiTypesV2/HttpInfo.h | 2 + src/lib/apiTypesV2/MqttInfo.cpp | 8 +- src/lib/apiTypesV2/MqttInfo.h | 2 + src/lib/common/macroSubstitute.cpp | 10 +- src/lib/expressions/ExprContext.cpp | 187 +++-- src/lib/expressions/ExprContext.h | 13 +- src/lib/expressions/ExprManager.cpp | 58 +- src/lib/jsonParseV2/parseSubscription.cpp | 40 + .../mongoBackend/MongoCommonSubscription.cpp | 7 + src/lib/mongoBackend/dbConstants.h | 1 + src/lib/ngsiNotify/Notifier.cpp | 4 +- .../notification_templates_cache_refresh.test | 3 +- ...s_with_decoded_chars_in_notifications.test | 3 +- .../http_info_for_sub_update.test | 6 +- .../custom_url_validity_check.test | 3 +- ...tom_notification_http_json_basic_crud.test | 25 +- ...stom_notification_http_json_constants.test | 4 +- ...m_notification_http_json_replacements.test | 4 +- ...ication_http_json_replacements_inside.test | 4 +- ...n_http_json_replacements_inside_multi.test | 4 +- ...n_http_json_replacements_intermediate.test | 4 +- ...tom_notification_mqtt_json_basic_crud.test | 25 +- ...stom_notification_mqtt_json_constants.test | 6 +- ...m_notification_mqtt_json_replacements.test | 4 +- ...ication_mqtt_json_replacements_inside.test | 4 +- ...n_mqtt_json_replacements_inside_multi.test | 4 +- ...n_mqtt_json_replacements_intermediate.test | 4 +- .../self_notification_multi_hop.test | 9 +- .../self_notification_one_hop.test | 3 +- .../mqtt_custom_subscription_basic_crud.test | 8 +- .../mqtt_custom_subscription_update.test | 12 +- .../3001_mqtt/mqtt_notification_custom.test | 3 +- ...tification_problem_with_custom_header.test | 4 +- .../null_payload_get.test | 6 +- .../per_sub_httpTimeout_crud_custom.test | 15 +- ...r_sub_httpTimeout_notification_custom.test | 6 +- .../mqtt_custom_notifications_auth.test | 12 +- ...http_custom_exprlang_field_basic_crud.test | 664 +++++++++++++++++ .../jexl_expr_attrs_weird_syntax.test | 12 +- .../legacy_expr_attrs_weird_syntax.test | 216 ++++++ ...mqtt_custom_exprlang_field_basic_crud.test | 699 ++++++++++++++++++ ...m_notification_http_ngsi_attr_no_type.test | 3 +- ...tom_notification_http_ngsi_basic_crud.test | 25 +- ...cation_http_ngsi_basic_crud_compounds.test | 21 +- ...fication_http_ngsi_basic_crud_partial.test | 21 +- ...m_notification_mqtt_ngsi_attr_no_type.test | 3 +- ...tom_notification_mqtt_ngsi_basic_crud.test | 25 +- ...cation_mqtt_ngsi_basic_crud_compounds.test | 21 +- ...fication_mqtt_ngsi_basic_crud_partial.test | 21 +- .../mqttCustom_retain_basic_crud.test | 30 +- .../ngsi_patching_special_attr_types.test | 6 +- 55 files changed, 2091 insertions(+), 205 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index fdb575ad63..daa5db26e3 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ +- Add: exprLang field in custom notifications (#4004) - Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) - Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) diff --git a/doc/manuals/admin/database_model.md b/doc/manuals/admin/database_model.md index ae438dbef6..1c3ad517c6 100644 --- a/doc/manuals/admin/database_model.md +++ b/doc/manuals/admin/database_model.md @@ -329,6 +329,7 @@ notifications. It is a number between 0 and 1800000. If defined to 0 or omitted, in the Orion API. More detail of this functionality [here](../orion-api.md#ngsi-payload-patching). The value of this field is an object with a `attrs` key which value is a simplified version of `attrs` in [the entities collection](#entities-collection). +- **exprLang**: optional field to store the expression language. Only for custom subscriptions. If omitted `legacy` is assumed. - **lastFailure**: the time (as integer number, meaning seconds) when last notification failure occurred. Not present if the subscription has never failed. - **lastFailureReason**: text describing the cause of the last failure. diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 7b66992569..4fbc8d5ab0 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -3946,6 +3946,7 @@ A `httpCustom` object contains the following subfields. | `payload` | ✓ | string | Text-based payload to be used in notifications. In case of empty string or omitted, the default payload (see [Notification Messages](#notification-messages) sections) is used. If `null`, notification will not include any payload. | | `json` | ✓ | object | JSON-based payload to be used in notifications. See [JSON Payloads](#json-payloads) section for more details. | | `ngsi` | ✓ | object | NGSI patching for payload to be used in notifications. See [NGSI payload patching](#ngsi-payload-patching) section for more details. | +| `exprLang` | ✓ | string | Expression language to use, either `legacy` or `jexl` (default is `jexl`) | | `timeout` | ✓ | number | Maximum time (in milliseconds) the subscription waits for the response. The maximum value allowed for this parameter is 1800000 (30 minutes). If `timeout` is defined to 0 or omitted, then the value passed as `-httpTimeout` CLI parameter is used. See section in the [Command line options](admin/cli.md#command-line-options) for more details. | `payload`, `json` or `ngsi` cannot be used at the same time, they are mutually exclusive. @@ -3967,6 +3968,7 @@ A `mqttCustom` object contains the following subfields. | `payload` | ✓ | string | Text-based payload to be used in notifications. In case of empty string or omitted, the default payload (see [Notification Messages](#notification-messages) sections) is used. If `null`, notification will not include any payload. | | `json` | ✓ | object | JSON-based payload to be used in notifications. See [JSON Payloads](#json-payloads) section for more details. | | `ngsi` | ✓ | object | NGSI patching for payload to be used in notifications. See [NGSI payload patching](#ngsi-payload-patching) section for more details. | +| `exprLang` | ✓ | string | Expression language to use, either `legacy` or `jexl` (default is `jexl`) | `payload`, `json` or `ngsi` cannot be used at the same time, they are mutually exclusive. diff --git a/src/lib/apiTypesV2/HttpInfo.cpp b/src/lib/apiTypesV2/HttpInfo.cpp index 4f5e1fd80e..cc4c000283 100644 --- a/src/lib/apiTypesV2/HttpInfo.cpp +++ b/src/lib/apiTypesV2/HttpInfo.cpp @@ -106,6 +106,8 @@ std::string HttpInfo::toJson() { jh.addRaw("headers", objectToJson(headers)); } + + jh.addString("exprLang", this->exprLang); } return jh.str(); @@ -131,7 +133,7 @@ void HttpInfo::fill(const orion::BSONObj& bo) if (bo.hasField(CSUB_NGSI)) n++; if (n > 1) { - LM_E(("custom notification must not have more than one payload related field")); + LM_E(("Runtime Error (custom notification must not have more than one payload related field)")); return; } @@ -232,6 +234,9 @@ void HttpInfo::fill(const orion::BSONObj& bo) orion::BSONObj headers = getObjectFieldF(bo, CSUB_HEADERS); headers.toStringMap(&this->headers); } + + // expression language used in custom notifications + this->exprLang = bo.hasField(CSUB_EXPRLANG) ? getStringFieldF(bo, CSUB_EXPRLANG) : "legacy"; } } @@ -252,6 +257,7 @@ void HttpInfo::fill(const HttpInfo& _httpInfo) this->custom = _httpInfo.custom; this->includePayload = _httpInfo.includePayload; this->timeout = _httpInfo.timeout; + this->exprLang = _httpInfo.exprLang; this->json = _httpInfo.json == NULL? NULL : _httpInfo.json->clone(); diff --git a/src/lib/apiTypesV2/HttpInfo.h b/src/lib/apiTypesV2/HttpInfo.h index 0af3d273ef..8d32651374 100644 --- a/src/lib/apiTypesV2/HttpInfo.h +++ b/src/lib/apiTypesV2/HttpInfo.h @@ -57,6 +57,8 @@ struct HttpInfo bool includePayload; long long timeout; + std::string exprLang; + HttpInfo(); std::string toJson(); diff --git a/src/lib/apiTypesV2/MqttInfo.cpp b/src/lib/apiTypesV2/MqttInfo.cpp index d46b9e4507..c8829b0bb2 100644 --- a/src/lib/apiTypesV2/MqttInfo.cpp +++ b/src/lib/apiTypesV2/MqttInfo.cpp @@ -102,6 +102,8 @@ std::string MqttInfo::toJson() jh.addRaw("ngsi", this->ngsi.toJson(NGSI_V2_NORMALIZED, true)); break; } + + jh.addString("exprLang", this->exprLang); } return jh.str(); @@ -143,7 +145,7 @@ void MqttInfo::fill(const orion::BSONObj& bo) if (bo.hasField(CSUB_NGSI)) n++; if (n > 1) { - LM_E(("custom notification must not have more than one payload related field")); + LM_E(("Runtime Error (custom notification must not have more than one payload related field)")); return; } @@ -216,6 +218,9 @@ void MqttInfo::fill(const orion::BSONObj& bo) this->ngsi.attributeVector.fill(getObjectFieldF(ngsiObj, ENT_ATTRS)); } } + + // expression language used in custom notifications + this->exprLang = bo.hasField(CSUB_EXPRLANG) ? getStringFieldF(bo, CSUB_EXPRLANG) : "legacy"; } } @@ -238,6 +243,7 @@ void MqttInfo::fill(const MqttInfo& _mqttInfo) this->providedAuth = _mqttInfo.providedAuth; this->user = _mqttInfo.user; this->passwd = _mqttInfo.passwd; + this->exprLang = _mqttInfo.exprLang; this->json = _mqttInfo.json == NULL ? NULL : _mqttInfo.json->clone(); diff --git a/src/lib/apiTypesV2/MqttInfo.h b/src/lib/apiTypesV2/MqttInfo.h index 738e73c6a9..bbc3b9260d 100644 --- a/src/lib/apiTypesV2/MqttInfo.h +++ b/src/lib/apiTypesV2/MqttInfo.h @@ -53,6 +53,8 @@ struct MqttInfo CustomPayloadType payloadType; bool includePayload; + std::string exprLang; + bool providedAuth; std::string user; std::string passwd; diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 3c0fc7e105..e1619ca77c 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -60,7 +60,15 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e } else { - return r.toString(); + // in legacy mode an extra remove quotes step is needed + if (exprContextObjectP->isLegacy()) + { + return removeQuotes(r.toString()); + } + else + { + return r.toString(); + } } } else if (exprContextObjectP != NULL) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 41ec5b3ddb..87f1de9f9d 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -25,6 +25,7 @@ #include +#include "common/string.h" #include "logMsg/logMsg.h" #include "expressions/ExprContext.h" #include "expressions/exprCommon.h" @@ -35,15 +36,20 @@ * * ExprContextObject::ExprContextObject - */ -ExprContextObject::ExprContextObject() +ExprContextObject::ExprContextObject(bool _legacy) { - jexlContext = PyDict_New(); + legacy = _legacy; - if (jexlContext == NULL) + if (!legacy) { - // Note that this ruins the object. We log this situation just one time here, but we include a NULL check - // in every other method to be safer - LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); + jexlContext = PyDict_New(); + + if (jexlContext == NULL) + { + // Note that this ruins the object. We log this situation just one time here, but we include a NULL check + // in every other method to be safer + LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); + } } } @@ -51,34 +57,52 @@ ExprContextObject::ExprContextObject() /* **************************************************************************** * -* ExprContextObject::get - +* ExprContextObject::getJexlContext - */ -PyObject* ExprContextObject::get(void) +PyObject* ExprContextObject::getJexlContext(void) { return jexlContext; } +/* **************************************************************************** +* +* ExprContextObject::getJexlContext - +*/ +std::map* ExprContextObject::getMap(void) +{ + return &repl; +} + + + /* **************************************************************************** * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, const std::string& _value) +void ExprContextObject::add(const std::string &key, const std::string &_value) { - if (jexlContext == NULL) + if (legacy) { - return; + repl.insert(std::pair(key, _value)); } - LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - if (value == NULL) + else { - LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); - return; + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); + PyObject* value = Py_BuildValue("s", _value.c_str()); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); + return; + } + PyDict_SetItemString(jexlContext, key.c_str(), value); + Py_DECREF(value); } - PyDict_SetItemString(jexlContext, key.c_str(), value); - Py_DECREF(value); } @@ -87,44 +111,57 @@ void ExprContextObject::add(const std::string& key, const std::string& _value) * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, double _value) +void ExprContextObject::add(const std::string &key, double _value) { - if (jexlContext == NULL) + if (legacy) { - return; + repl.insert(std::pair(key, double2string(_value))); } - LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); - PyObject* value = Py_BuildValue("d", _value); - if (value == NULL) + else { - LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); - return; + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); + PyObject* value = Py_BuildValue("d", _value); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); + return; + } + PyDict_SetItemString(jexlContext, key.c_str(), value); + Py_DECREF(value); } - PyDict_SetItemString(jexlContext, key.c_str(), value); - Py_DECREF(value); } - /* **************************************************************************** * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, bool _value) +void ExprContextObject::add(const std::string &key, bool _value) { - if (jexlContext == NULL) + if (legacy) { - return; - } - LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value? "true" : "false")); - if (_value) - { - PyDict_SetItemString(jexlContext, key.c_str(), Py_True); + repl.insert(std::pair(key, _value? "true": "false")); } else { - PyDict_SetItemString(jexlContext, key.c_str(), Py_False); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); + if (_value) + { + PyDict_SetItemString(jexlContext, key.c_str(), Py_True); + } + else + { + PyDict_SetItemString(jexlContext, key.c_str(), Py_False); + } } } @@ -134,14 +171,21 @@ void ExprContextObject::add(const std::string& key, bool _value) * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key) +void ExprContextObject::add(const std::string &key) { - if (jexlContext == NULL) + if (legacy) { - return; + repl.insert(std::pair(key, "null")); + } + else + { + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), Py_None); } - LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), Py_None); } @@ -150,14 +194,22 @@ void ExprContextObject::add(const std::string& key) * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, ExprContextObject exprContextObject) +void ExprContextObject::add(const std::string &key, ExprContextObject exprContextObject) { - if (jexlContext == NULL) + if (legacy) { - return; + // FIXME PR: not sure if this is going to work... + repl.insert(std::pair(key, exprContextObject.toString())); + } + else + { + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (object): %s=%s", key.c_str(), exprContextObject.toString().c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.getJexlContext()); } - LM_T(LmtExpr, ("adding to expression context object (object): %s=%s", key.c_str(), exprContextObject.toString().c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.get()); } @@ -166,18 +218,36 @@ void ExprContextObject::add(const std::string& key, ExprContextObject exprContex * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, ExprContextList exprContextList) +void ExprContextObject::add(const std::string &key, ExprContextList exprContextList) { - if (jexlContext == NULL) + if (legacy) { - return; + // FIXME PR: not sure if this is going to work... + repl.insert(std::pair(key, exprContextList.toString())); + } + else + { + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (list): %s=%s", key.c_str(), exprContextList.toString().c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); } - LM_T(LmtExpr, ("adding to expression context object (list): %s=%s", key.c_str(), exprContextList.toString().c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); } +/* **************************************************************************** +* +* ExprContextObject::isLegacy - +*/ +bool ExprContextObject::isLegacy(void) +{ + return legacy; +} + + /* **************************************************************************** * * ExprContextObject::toString - @@ -247,7 +317,7 @@ PyObject* ExprContextList::get(void) * * ExprContextList::add - */ -void ExprContextList::add(const std::string& _value) +void ExprContextList::add(const std::string &_value) { if (jexlContext == NULL) { @@ -255,7 +325,7 @@ void ExprContextList::add(const std::string& _value) } LM_T(LmtExpr, ("adding to expression context list (string): %s=", _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); - if (value == NULL) + if (value == NULL) { LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); return; @@ -278,7 +348,7 @@ void ExprContextList::add(double _value) } LM_T(LmtExpr, ("adding to expression context list (double): %f", _value)); PyObject* value = Py_BuildValue("d", _value); - if (value == NULL) + if (value == NULL) { LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); return; @@ -289,7 +359,6 @@ void ExprContextList::add(double _value) - /* **************************************************************************** * * ExprContextList::add - @@ -300,7 +369,7 @@ void ExprContextList::add(bool _value) { return; } - LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value? "true" : "false")); + LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value ? "true" : "false")); if (_value) { PyList_Append(jexlContext, Py_True); @@ -340,7 +409,7 @@ void ExprContextList::add(ExprContextObject exprContextObject) return; } LM_T(LmtExpr, ("adding to expression context list (object): %s", exprContextObject.toString().c_str())); - PyList_Append(jexlContext, exprContextObject.get()); + PyList_Append(jexlContext, exprContextObject.getJexlContext()); } diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 6b41dc7bca..19c165f276 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -28,6 +28,7 @@ #include #include +#include class ExprContextList; // forward declaration @@ -38,12 +39,16 @@ class ExprContextList; // forward declaration class ExprContextObject { private: - PyObject* jexlContext; + bool legacy; + PyObject* jexlContext; // used in regular (i.e. not legacy) mode + std::map repl; // used in legacy mode public: - ExprContextObject(); + ExprContextObject(bool legacy = false); + + PyObject* getJexlContext(void); + std::map* getMap(void); - PyObject* get(void); void add(const std::string& key, const std::string& value); void add(const std::string& key, double value); void add(const std::string& key, bool value); @@ -51,6 +56,8 @@ class ExprContextObject void add(const std::string& key, ExprContextObject exprContextObject); void add(const std::string& key, ExprContextList exprContextList); + bool isLegacy(void); + std::string toString(void); void release(void); diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index d6aa98d5bd..bfe6d14b81 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -54,16 +54,14 @@ void ExprManager::init(void) pyjexlModule = PyImport_ImportModule("pyjexl"); if (pyjexlModule == NULL) { - const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", error)); + LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", capturePythonError())); } LM_T(LmtExpr, ("pyjexl module has been loaded")); jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); if (jexlEngine == NULL) { - const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", error)); + LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", capturePythonError())); } LM_T(LmtExpr, ("jexl engine has been created")); } @@ -79,28 +77,46 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st ExprResult r; r.valueType = orion::ValueTypeNull; - LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - - PyObject* expression = Py_BuildValue("s", _expression.c_str()); - if (expression == NULL) + if (exprContextObjectP->isLegacy()) { - LM_W(("error building JEXL expression: %s", capturePythonError())); - return r; - } + // std::map based evaluation. Only pure replacement is supported + LM_T(LmtExpr, ("evaluating legacy expresion: <%s>", _expression.c_str())); + + std::map* replacementsP = exprContextObjectP->getMap(); - PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, exprContextObjectP->get()); - Py_XDECREF(expression); - if (result == NULL) + std::map::iterator iter = replacementsP->find(_expression); + if (iter != replacementsP->end()) + { + r.valueType = orion::ValueTypeString; + r.stringValue = iter->second; + } + } + else { - LM_W(("error evaluating JEXL expression: %s", capturePythonError())); - return r; + // JEXL based evaluation + LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); + + PyObject* expression = Py_BuildValue("s", _expression.c_str()); + if (expression == NULL) + { + LM_W(("error building JEXL expression: %s", capturePythonError())); + return r; + } + + PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, exprContextObjectP->getJexlContext()); + Py_XDECREF(expression); + if (result == NULL) + { + LM_W(("error evaluating JEXL expression: %s", capturePythonError())); + return r; + } + + r.fill(result); + + // FIXME PR: does this Py_XDECREF() recursively in the case of dicts or lists? + Py_XDECREF(result); } - r.fill(result); - - // FIXME PR: does this Py_XDECREF() recursively in the case of dicts or lists? - Py_XDECREF(result); - return r; } diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index 4207dfa023..cea25d5b52 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -1031,6 +1031,26 @@ static std::string parseNotification(ConnectionInfo* ciP, SubscriptionUpdate* su return r; } + // exprLang + Opt exprLangOpt = getStringOpt(httpCustom, "exprLang", "exprLang httpCustom notification"); + std::string exprLang = "jexl"; + + if (!exprLangOpt.ok()) + { + return badInput(ciP, exprLangOpt.error); + } + + if (exprLangOpt.given) + { + exprLang = exprLangOpt.value; + if ((exprLang != "jexl") && (exprLang != "legacy")) + { + return badInput(ciP, "not valid exprLang, valid ones are jexl or legacy"); + } + } + + subsP->notification.httpInfo.exprLang = exprLang; + subsP->notification.httpInfo.custom = true; } else if (notification.HasMember("mqtt")) @@ -1141,6 +1161,26 @@ static std::string parseNotification(ConnectionInfo* ciP, SubscriptionUpdate* su return r; } + // exprLang + Opt exprLangOpt = getStringOpt(mqttCustom, "exprLang", "exprLang mqttCustom notification"); + std::string exprLang = "jexl"; + + if (!exprLangOpt.ok()) + { + return badInput(ciP, exprLangOpt.error); + } + + if (exprLangOpt.given) + { + exprLang = exprLangOpt.value; + if ((exprLang != "jexl") && (exprLang != "legacy")) + { + return badInput(ciP, "not valid exprLang, valid ones are jexl or legacy"); + } + } + + subsP->notification.mqttInfo.exprLang = exprLang; + subsP->notification.mqttInfo.custom = true; } diff --git a/src/lib/mongoBackend/MongoCommonSubscription.cpp b/src/lib/mongoBackend/MongoCommonSubscription.cpp index dd6bcf3101..a58ea5f55b 100644 --- a/src/lib/mongoBackend/MongoCommonSubscription.cpp +++ b/src/lib/mongoBackend/MongoCommonSubscription.cpp @@ -183,6 +183,9 @@ static void setCustomHttpInfo(const HttpInfo& httpInfo, orion::BSONObjBuilder* b b->append(CSUB_NGSI, bob.obj()); } + + b->append(CSUB_EXPRLANG, httpInfo.exprLang); + LM_T(LmtMongo, ("Subscription exprLang: %s", httpInfo.exprLang.c_str())); } @@ -255,6 +258,9 @@ static void setCustomMqttInfo(const ngsiv2::MqttInfo& mqttInfo, orion::BSONObjBu b->append(CSUB_NGSI, bob.obj()); } + + b->append(CSUB_EXPRLANG, mqttInfo.exprLang); + LM_T(LmtMongo, ("Subscription exprLang: %s", mqttInfo.exprLang.c_str())); } @@ -626,6 +632,7 @@ void setOnlyChanged(const Subscription& sub, orion::BSONObjBuilder* b) } + /* **************************************************************************** * * setCovered - diff --git a/src/lib/mongoBackend/dbConstants.h b/src/lib/mongoBackend/dbConstants.h index b55cf29925..241507efd9 100644 --- a/src/lib/mongoBackend/dbConstants.h +++ b/src/lib/mongoBackend/dbConstants.h @@ -119,6 +119,7 @@ #define CSUB_NGSI "ngsi" #define CSUB_BLACKLIST "blacklist" #define CSUB_ONLYCHANGED "onlyChanged" +#define CSUB_EXPRLANG "exprLang" #define CSUB_COVERED "covered" #define CSUB_NOTIFYONMETADATACHANGE "notifyOnMetadataChange" #define CSUB_LASTFAILURE "lastFailure" diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index aedcaa1b21..3f8516798f 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -315,8 +315,10 @@ static SenderThreadParams* buildSenderParamsCustom std::map headers; Entity& en = notifyCerP->entity; + const std::string exprLang = notification.type == ngsiv2::HttpNotification ? notification.httpInfo.exprLang : notification.mqttInfo.exprLang; + // Used by several macroSubstitute() calls along this function - ExprContextObject exprContext; + ExprContextObject exprContext(exprLang == "legacy"); exprContext.add("id", en.id); exprContext.add("type", en.type); exprContext.add("service", tenant); diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test index 853d3e6abd..f1b0c00cbb 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test @@ -157,7 +157,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 695 +Content-Length: 713 [ { @@ -167,6 +167,7 @@ Content-Length: 695 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "A1": "${A1}", "A2": "${A2}", diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test index 9a548cec40..a31092de4a 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test @@ -137,7 +137,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 550 +Content-Length: 568 { "id": "REGEX([0-9a-f]{24})", @@ -146,6 +146,7 @@ Content-Length: 550 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "hs1": "1", "hs2": "${A1}" diff --git a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test index 7a4956a172..2bdcc9c72c 100644 --- a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test +++ b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test @@ -173,7 +173,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 495 +Content-Length: 513 [ { @@ -183,6 +183,7 @@ Content-Length: 495 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-Type": "application/json", "H1": "${A1}" @@ -231,7 +232,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 497 +Content-Length: 515 [ { @@ -241,6 +242,7 @@ Content-Length: 497 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-Type": "application/json", "H2": "${A2}" diff --git a/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test b/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test index 57a7a22c99..5969637f5d 100644 --- a/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test +++ b/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test @@ -142,7 +142,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 286 +Content-Length: 304 [ { @@ -152,6 +152,7 @@ Content-Length: 286 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "method": "PUT", "url": "${abc}" }, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test index c50d612856..89828afff7 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test @@ -254,7 +254,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 325 +Content-Length: 343 { "id": "REGEX([0-9a-f]{24})", @@ -263,6 +263,7 @@ Content-Length: 325 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "x1": 1, "x2": 2 @@ -301,7 +302,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -310,6 +311,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -345,7 +347,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 321 +Content-Length: 339 { "id": "REGEX([0-9a-f]{24})", @@ -354,6 +356,7 @@ Content-Length: 321 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "yy", "zz" @@ -392,7 +395,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 317 +Content-Length: 335 { "id": "REGEX([0-9a-f]{24})", @@ -401,6 +404,7 @@ Content-Length: 317 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ 1, 2, @@ -440,7 +444,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -449,6 +453,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -486,7 +491,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -495,6 +500,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -530,7 +536,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 334 +Content-Length: 352 { "id": "REGEX([0-9a-f]{24})", @@ -539,6 +545,7 @@ Content-Length: 334 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "z4": true, "z5": [ @@ -572,7 +579,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 655 +Content-Length: 691 [ { @@ -582,6 +589,7 @@ Content-Length: 655 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -608,6 +616,7 @@ Content-Length: 655 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "z4": true, "z5": [ diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test index 2bb34d3d32..d49ac8a108 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test @@ -180,7 +180,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 880 +Content-Length: 916 [ { @@ -190,6 +190,7 @@ Content-Length: 880 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "array": [ "22", @@ -245,6 +246,7 @@ Content-Length: 880 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "foo", 10, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test index 08f26a3682..e0a2d267ec 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 802 +Content-Length: 838 [ { @@ -197,6 +197,7 @@ Content-Length: 802 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "array": "${array}", "bool": "${bool}", @@ -230,6 +231,7 @@ Content-Length: 802 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "${text}", "${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test index 758775b4e3..e1b86b3f9f 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 815 +Content-Length: 851 [ { @@ -197,6 +197,7 @@ Content-Length: 815 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "A": "ns:${text}", "B": "ns:${number}", @@ -230,6 +231,7 @@ Content-Length: 815 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "ns:${text}", "ns:${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test index 95f7928e31..191994521c 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 921 +Content-Length: 957 [ { @@ -197,6 +197,7 @@ Content-Length: 921 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "A": "ns:${text}:${object}", "B": "ns:${number}:${array}", @@ -230,6 +231,7 @@ Content-Length: 921 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "ns:${text}:${object}", "ns:${number}:${array}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test index 7a19421ab8..568d36fc19 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test @@ -157,7 +157,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 715 +Content-Length: 751 [ { @@ -167,6 +167,7 @@ Content-Length: 715 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "x": { "x1": "${A}", @@ -202,6 +203,7 @@ Content-Length: 715 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "22", { diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test index 0c24073d89..dccebf8230 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test @@ -261,7 +261,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 358 +Content-Length: 376 { "id": "REGEX([0-9a-f]{24})", @@ -270,6 +270,7 @@ Content-Length: 358 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "x1": 1, "x2": 2 @@ -311,7 +312,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -320,6 +321,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -358,7 +360,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 354 +Content-Length: 372 { "id": "REGEX([0-9a-f]{24})", @@ -367,6 +369,7 @@ Content-Length: 354 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "yy", "zz" @@ -408,7 +411,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 350 +Content-Length: 368 { "id": "REGEX([0-9a-f]{24})", @@ -417,6 +420,7 @@ Content-Length: 350 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ 1, 2, @@ -459,7 +463,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -468,6 +472,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -508,7 +513,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -517,6 +522,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -555,7 +561,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 367 +Content-Length: 385 { "id": "REGEX([0-9a-f]{24})", @@ -564,6 +570,7 @@ Content-Length: 367 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "z4": true, "z5": [ @@ -600,7 +607,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 721 +Content-Length: 757 [ { @@ -610,6 +617,7 @@ Content-Length: 721 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -639,6 +647,7 @@ Content-Length: 721 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "z4": true, "z5": [ diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test index d8fe2d436c..e84b665130 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test @@ -186,7 +186,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 946 +Content-Length: 982 [ { @@ -196,6 +196,7 @@ Content-Length: 946 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "array": [ "22", @@ -223,7 +224,7 @@ Content-Length: 946 "y1", "y2" ] - }, + }, "text": "foo" }, "qos": 0, @@ -254,6 +255,7 @@ Content-Length: 946 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "foo", 10, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test index 954ed3bdba..993180a8ce 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test @@ -193,7 +193,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 868 +Content-Length: 904 [ { @@ -203,6 +203,7 @@ Content-Length: 868 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "array": "${array}", "bool": "${bool}", @@ -239,6 +240,7 @@ Content-Length: 868 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "${text}", "${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test index 3a6f4a15c5..7133b2e675 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test @@ -192,7 +192,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 881 +Content-Length: 917 [ { @@ -202,6 +202,7 @@ Content-Length: 881 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "A": "ns:${text}", "B": "ns:${number}", @@ -238,6 +239,7 @@ Content-Length: 881 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "ns:${text}", "ns:${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test index 940492dc04..58c287e5d5 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test @@ -193,7 +193,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 987 +Content-Length: 1023 [ { @@ -203,6 +203,7 @@ Content-Length: 987 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "A": "ns:${text}:${object}", "B": "ns:${number}:${array}", @@ -239,6 +240,7 @@ Content-Length: 987 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "ns:${text}:${object}", "ns:${number}:${array}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test index 020245e27b..2fc44f8b6b 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test @@ -163,7 +163,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 781 +Content-Length: 817 [ { @@ -173,6 +173,7 @@ Content-Length: 781 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "x": { "x1": "${A}", @@ -211,6 +212,7 @@ Content-Length: 781 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "22", { diff --git a/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test b/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test index 2fe079ec78..6fa972f30b 100644 --- a/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test +++ b/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test @@ -373,7 +373,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 564 +Content-Length: 582 { "id": "REGEX([0-9a-f]{24})", @@ -382,6 +382,7 @@ Content-Length: 564 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-type": "application/json" }, @@ -415,7 +416,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 564 +Content-Length: 582 { "id": "REGEX([0-9a-f]{24})", @@ -424,6 +425,7 @@ Content-Length: 564 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-type": "application/json" }, @@ -457,7 +459,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 564 +Content-Length: 582 { "id": "REGEX([0-9a-f]{24})", @@ -466,6 +468,7 @@ Content-Length: 564 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-type": "application/json" }, diff --git a/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test b/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test index ffaffdb044..12727d8b09 100644 --- a/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test +++ b/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test @@ -128,7 +128,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 566 +Content-Length: 584 [ { @@ -138,6 +138,7 @@ Content-Length: 566 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-type": "application/json" }, diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test index d482101617..36ca87149f 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test @@ -282,7 +282,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 2428 +Content-Length: 2536 [ { @@ -295,6 +295,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "temperature:${temperature}", "qos": 0, "retain": false, @@ -327,6 +328,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "temperature:${temperature}", "qos": 2, "retain": false, @@ -359,6 +361,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "/orion/${dest}", @@ -390,6 +393,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 2, "retain": false, "topic": "/orion/${dest}", @@ -421,6 +425,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": null, "qos": 0, "retain": false, @@ -453,6 +458,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": null, "qos": 2, "retain": false, diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test index 531e251861..1d65654747 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test @@ -204,7 +204,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 389 +Content-Length: 407 { "description": "Original sub", @@ -214,6 +214,7 @@ Content-Length: 389 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "t:${t}", "qos": 2, "retain": false, @@ -252,7 +253,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 389 +Content-Length: 407 { "description": "Modified sub", @@ -262,6 +263,7 @@ Content-Length: 389 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "t:${t}", "qos": 2, "retain": false, @@ -300,7 +302,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 382 +Content-Length: 400 { "description": "Modified sub", @@ -310,6 +312,7 @@ Content-Length: 382 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": null, "qos": 0, "retain": false, @@ -392,7 +395,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 372 +Content-Length: 390 { "description": "Modified sub", @@ -402,6 +405,7 @@ Content-Length: 372 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 1, "retain": false, "topic": "/orionbk/${d}", diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test b/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test index ecc29badba..597cfa5457 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test @@ -245,7 +245,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 509 +Content-Length: 527 [ { @@ -258,6 +258,7 @@ Content-Length: 509 "lastNotification": "REGEX(.*)", "lastSuccess": "REGEX(.*)", "mqttCustom": { + "exprLang": "jexl", "payload": "{ %22A%22: %22${A}%22 }", "qos": 0, "retain": false, diff --git a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test b/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test index b4e9cc5695..9e63211726 100644 --- a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test +++ b/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test @@ -159,7 +159,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1012 +Content-Length: 1048 [ { @@ -171,6 +171,7 @@ Content-Length: 1012 "attrsFormat": "legacy", "covered": false, "httpCustom": { + "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "lastNotification": "REGEX(.*)", @@ -201,6 +202,7 @@ Content-Length: 1012 "attrsFormat": "legacy", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "{ %22A1%22: %22Value of A1: ${A1}%22 }", "url": "http://localhost:9997/notify" }, diff --git a/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test b/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test index 0955bd6c86..13e3163639 100644 --- a/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test +++ b/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test @@ -189,7 +189,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1200 +Content-Length: 1272 [ { @@ -199,6 +199,7 @@ Content-Length: 1200 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "abc", "url": "http://localhost:9997/notify" }, @@ -224,6 +225,7 @@ Content-Length: 1200 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "onlyChangedAttrs": false @@ -248,6 +250,7 @@ Content-Length: 1200 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": null, "url": "http://localhost:9997/notify" }, @@ -273,6 +276,7 @@ Content-Length: 1200 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "onlyChangedAttrs": false diff --git a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test index d874dbefdd..badbe98993 100644 --- a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test +++ b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test @@ -327,7 +327,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 379 { "description": "HTTP sub", @@ -337,6 +337,7 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 1000, "url": "http://localhost:1234" @@ -373,7 +374,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 379 { "description": "HTTP sub", @@ -383,6 +384,7 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 2000, "url": "http://localhost:1234" @@ -419,7 +421,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 346 +Content-Length: 364 { "description": "HTTP sub", @@ -429,6 +431,7 @@ Content-Length: 346 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "url": "http://localhost:1234" }, @@ -464,7 +467,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 379 { "description": "HTTP sub", @@ -474,6 +477,7 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 1000, "url": "http://localhost:1234" @@ -520,7 +524,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 346 +Content-Length: 364 { "description": "HTTP sub", @@ -530,6 +534,7 @@ Content-Length: 346 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "url": "http://localhost:1234" }, diff --git a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test index 0937cb5cec..bbdcdaebb0 100644 --- a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test +++ b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test @@ -175,7 +175,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 496 +Content-Length: 514 { "description": "HTTP sub", @@ -185,6 +185,7 @@ Content-Length: 496 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 12000, "url": "http://localhost:9997/noresponse" @@ -217,7 +218,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 532 +Content-Length: 550 { "description": "HTTP sub", @@ -228,6 +229,7 @@ Content-Length: 532 "covered": false, "failsCounter": 1, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 8000, "url": "http://localhost:9997/noresponse" diff --git a/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test b/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test index 59a65b45f0..c50332dc8f 100644 --- a/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test +++ b/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test @@ -317,7 +317,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 366 +Content-Length: 384 { "id": "REGEX([0-9a-f\-]{24})", @@ -326,6 +326,7 @@ Content-Length: 366 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, @@ -371,7 +372,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 367 +Content-Length: 385 { "id": "REGEX([0-9a-f\-]{24})", @@ -380,6 +381,7 @@ Content-Length: 367 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, @@ -425,7 +427,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 334 +Content-Length: 352 { "id": "REGEX([0-9a-f\-]{24})", @@ -434,6 +436,7 @@ Content-Length: 334 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "/orion", @@ -477,7 +480,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 366 +Content-Length: 384 { "id": "REGEX([0-9a-f\-]{24})", @@ -486,6 +489,7 @@ Content-Length: 366 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test new file mode 100644 index 0000000000..656e55da9b --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test @@ -0,0 +1,664 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Basic CRUD for exprLang field (http variant) + +--SHELL-INIT-- +dbInit CB +brokerStart CB 0-255 + +--SHELL-- + +# +# 01. Create subscription with invalid exprLang, see error +# 02. Create subscription with exprLang jexl +# 03a. Get subscription and see exprLang jexl +# 03b. Get subscription (as a list) and see exprLang jexl +# 04. Update subscription with exprLang legacy +# 05a. Get subscription and see exprLang legacy +# 05b. Get subscription (as a list) and see exprLang legacy +# 06. Update subscription without exprLang +# 07a. Get subscription and see exprLang jexl +# 07b. Get subscription (as a list) and see exprLang jexl +# 08. Update subscription with exprLang legacy +# 09a. Get subscription and see exprLang legacy +# 09b. Get subscription (as a list) and see exprLang legacy +# 10. Update subscription with invalid exprLang, see error +# 11a. Get subscription and see exprLang legacy +# 11b. Get subscription (as a list) and see exprLang legacy +# + +echo "01. Create subscription with invalid exprLang, see error" +echo "========================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": "foo" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create subscription with exprLang jexl" +echo "==========================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": "jexl" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "03a. Get subscription and see exprLang jexl" +echo "===========================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "03b. Get subscription (as a list) and see exprLang jexl" +echo "=======================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "04. Update subscription with exprLang legacy" +echo "============================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "05a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "05b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "06. Update subscription without exprLang" +echo "========================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "topic": "orion" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "07a. Get subscription and see exprLang jexl" +echo "===========================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "07b. Get subscription (as a list) and see exprLang jexl" +echo "=======================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "08. Update subscription with exprLang legacy" +echo "============================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "09a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "09b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "10. Update subscription with invalid exprLang, see error" +echo "========================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": 42 + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "11a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "11b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + +--REGEXPECT-- +01. Create subscription with invalid exprLang, see error +======================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 88 + +{ + "description": "not valid exprLang, valid ones are jexl or legacy", + "error": "BadRequest" +} + + +02. Create subscription with exprLang jexl +========================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03a. Get subscription and see exprLang jexl +=========================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 320 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "jexl", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +03b. Get subscription (as a list) and see exprLang jexl +======================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "jexl", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +04. Update subscription with exprLang legacy +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +05b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 324 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +06. Update subscription without exprLang +======================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +07a. Get subscription and see exprLang jexl +=========================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 320 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "jexl", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +07b. Get subscription (as a list) and see exprLang jexl +======================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "jexl", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +08. Update subscription with exprLang legacy +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +09a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +09b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 324 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +10. Update subscription with invalid exprLang, see error +======================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 87 + +{ + "description": "exprLang httpCustom notification is not a string", + "error": "BadRequest" +} + + +11a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +11b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 324 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +--TEARDOWN-- +brokerStop CB +dbDrop CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test index 3390b28e82..cf42ab7ebe 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -31,15 +31,17 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with custom expression: A-B and A:B +# Same as legacy_expr_attrs_weird_syntax.test, but using jexl expression language +# +# 01. 01. Create custom sub with custom expression: R1=A-B and R2=A:B # 02. Create entity E1 with A-B 1 and A:B 2 # 03. Update entity E1 with A-B 3 and A:B 4 # 04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null) # -echo "01. Create custom sub with custom expression: A-B and A:B" -echo "=========================================================" +echo "01. Create custom sub with custom expression: R1=A-B and R2=A:B" +echo "===============================================================" payload='{ "subject": { "entities": [ @@ -115,8 +117,8 @@ echo --REGEXPECT-- -01. Create custom sub with custom expression: A-B and A:B -========================================================= +01. Create custom sub with custom expression: R1=A-B and R2=A:B +=============================================================== HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test new file mode 100644 index 0000000000..15da6fc5bb --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test @@ -0,0 +1,216 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Legacy expression using attributes with weird syntax + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# Same as jexl_expr_attrs_weird_syntax.test, but using legacy expression language +# +# 01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B +# 02. Create entity E1 with A-B 1 and A:B 2 +# 03. Update entity E1 with A-B 3 and A:B 4 +# 04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4) +# + + +echo "01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B" +echo "======================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "exprLang": "legacy", + "ngsi": { + "R1": { + "value": "${A-B}", + "type": "Calculated" + }, + "R2": { + "value": "${A:B}", + "type": "Calculated" + } + } + }, + "attrs": [ "R1", "R2" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A-B 1 and A:B 2" +echo "=========================================" +payload='{ + "id": "E1", + "type": "T", + "A-B": { + "value": 1, + "type": "Number" + }, + "A:B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A-B 3 and A:B 4" +echo "=========================================" +payload='{ + "A-B": { + "value": 3, + "type": "Number" + }, + "A:B": { + "value": 4, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4)" +echo "======================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B +====================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A-B 1 and A:B 2 +========================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A-B 3 and A:B 4 +========================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4) +====================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 179 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "R1": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "R2": { + "metadata": {}, + "type": "Calculated", + "value": 2 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 179 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "R1": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "R2": { + "metadata": {}, + "type": "Calculated", + "value": 4 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test new file mode 100644 index 0000000000..b155766fea --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test @@ -0,0 +1,699 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Basic CRUD for exprLang field (mqtt variant) + +--SHELL-INIT-- +dbInit CB +brokerStart CB 0-255 + +--SHELL-- + +# +# 01. Create subscription with invalid exprLang, see error +# 02. Create subscription with exprLang jexl +# 03a. Get subscription and see exprLang jexl +# 03b. Get subscription (as a list) and see exprLang jexl +# 04. Update subscription with exprLang legacy +# 05a. Get subscription and see exprLang legacy +# 05b. Get subscription (as a list) and see exprLang legacy +# 06. Update subscription without exprLang +# 07a. Get subscription and see exprLang jexl +# 07b. Get subscription (as a list) and see exprLang jexl +# 08. Update subscription with exprLang legacy +# 09a. Get subscription and see exprLang legacy +# 09b. Get subscription (as a list) and see exprLang legacy +# 10. Update subscription with invalid exprLang, see error +# 11a. Get subscription and see exprLang legacy +# 11b. Get subscription (as a list) and see exprLang legacy +# + +echo "01. Create subscription with invalid exprLang, see error" +echo "========================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": "foo" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create subscription with exprLang jexl" +echo "==========================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": "jexl" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "03a. Get subscription and see exprLang jexl" +echo "===========================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "03b. Get subscription (as a list) and see exprLang jexl" +echo "=======================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "04. Update subscription with exprLang legacy" +echo "============================================" +payload='{ + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "05a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "05b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "06. Update subscription without exprLang" +echo "========================================" +payload='{ + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "07a. Get subscription and see exprLang jexl" +echo "===========================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "07b. Get subscription (as a list) and see exprLang jexl" +echo "=======================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "08. Update subscription with exprLang legacy" +echo "============================================" +payload='{ + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "09a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "09b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "10. Update subscription with invalid exprLang, see error" +echo "========================================================" +payload='{ + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": 42 + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "11a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "11b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + +--REGEXPECT-- +01. Create subscription with invalid exprLang, see error +======================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 88 + +{ + "description": "not valid exprLang, valid ones are jexl or legacy", + "error": "BadRequest" +} + + +02. Create subscription with exprLang jexl +========================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03a. Get subscription and see exprLang jexl +=========================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 359 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "jexl", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +03b. Get subscription (as a list) and see exprLang jexl +======================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "jexl", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +04. Update subscription with exprLang legacy +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +05b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 363 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +06. Update subscription without exprLang +======================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +07a. Get subscription and see exprLang jexl +=========================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 359 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "jexl", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +07b. Get subscription (as a list) and see exprLang jexl +======================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "jexl", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +08. Update subscription with exprLang legacy +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +09a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +09b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 363 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +10. Update subscription with invalid exprLang, see error +======================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 87 + +{ + "description": "exprLang mqttCustom notification is not a string", + "error": "BadRequest" +} + + +11a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +11b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 363 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +--TEARDOWN-- +brokerStop CB +dbDrop CB diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test index 4f2b9778b6..ed8fee4246 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test @@ -110,7 +110,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 567 +Content-Length: 585 { "id": "REGEX([0-9a-f]{24})", @@ -119,6 +119,7 @@ Content-Length: 567 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "SomeType", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test index 9d671877f0..71d19596e5 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test @@ -275,7 +275,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 365 +Content-Length: 383 { "id": "REGEX([0-9a-f]{24})", @@ -284,6 +284,7 @@ Content-Length: 365 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -326,7 +327,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -335,6 +336,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -370,7 +372,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 365 +Content-Length: 383 { "id": "REGEX([0-9a-f]{24})", @@ -379,6 +381,7 @@ Content-Length: 365 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -421,7 +424,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 365 +Content-Length: 383 { "id": "REGEX([0-9a-f]{24})", @@ -430,6 +433,7 @@ Content-Length: 365 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "C": { "type": "Number", @@ -472,7 +476,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -481,6 +485,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -518,7 +523,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -527,6 +532,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -562,7 +568,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 365 +Content-Length: 383 { "id": "REGEX([0-9a-f]{24})", @@ -571,6 +577,7 @@ Content-Length: 365 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", @@ -605,7 +612,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 686 +Content-Length: 722 [ { @@ -615,6 +622,7 @@ Content-Length: 686 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -641,6 +649,7 @@ Content-Length: 686 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test index a34bfa6d96..3ee0ec39fb 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test @@ -293,7 +293,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 414 +Content-Length: 432 { "id": "REGEX([0-9a-f]{24})", @@ -302,6 +302,7 @@ Content-Length: 414 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "StructuredValue", @@ -353,7 +354,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -362,6 +363,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -397,7 +399,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 415 +Content-Length: 433 { "id": "REGEX([0-9a-f]{24})", @@ -406,6 +408,7 @@ Content-Length: 415 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "StructuredValue", @@ -461,7 +464,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 411 +Content-Length: 429 { "id": "REGEX([0-9a-f]{24})", @@ -470,6 +473,7 @@ Content-Length: 411 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "C": { "type": "StructuredValue", @@ -521,7 +525,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -530,6 +534,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -567,7 +572,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -576,6 +581,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -611,7 +617,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 417 +Content-Length: 435 { "id": "REGEX([0-9a-f]{24})", @@ -620,6 +626,7 @@ Content-Length: 417 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "StructuredValue", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test index 2e60112a6e..d6330c8c1f 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test @@ -259,7 +259,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 353 +Content-Length: 371 { "id": "REGEX([0-9a-f]{24})", @@ -268,6 +268,7 @@ Content-Length: 353 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -309,7 +310,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -318,6 +319,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -353,7 +355,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 343 +Content-Length: 361 { "id": "REGEX([0-9a-f]{24})", @@ -362,6 +364,7 @@ Content-Length: 343 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -402,7 +405,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 333 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -411,6 +414,7 @@ Content-Length: 333 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "id": "E3", "type": "T3" @@ -449,7 +453,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -458,6 +462,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -495,7 +500,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -504,6 +509,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -539,7 +545,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 355 +Content-Length: 373 { "id": "REGEX([0-9a-f]{24})", @@ -548,6 +554,7 @@ Content-Length: 355 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test index 8043fd7c4f..0f62f2759f 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test @@ -111,7 +111,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 600 +Content-Length: 618 { "id": "REGEX([0-9a-f]{24})", @@ -120,6 +120,7 @@ Content-Length: 600 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "SomeType", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test index c08b1ab6ca..d8421cc232 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test @@ -282,7 +282,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 398 +Content-Length: 416 { "id": "REGEX([0-9a-f]{24})", @@ -291,6 +291,7 @@ Content-Length: 398 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -336,7 +337,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -345,6 +346,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -383,7 +385,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 398 +Content-Length: 416 { "id": "REGEX([0-9a-f]{24})", @@ -392,6 +394,7 @@ Content-Length: 398 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -437,7 +440,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 398 +Content-Length: 416 { "id": "REGEX([0-9a-f]{24})", @@ -446,6 +449,7 @@ Content-Length: 398 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "C": { "type": "Number", @@ -491,7 +495,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -500,6 +504,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -540,7 +545,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -549,6 +554,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -587,7 +593,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 398 +Content-Length: 416 { "id": "REGEX([0-9a-f]{24})", @@ -596,6 +602,7 @@ Content-Length: 398 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", @@ -633,7 +640,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 752 +Content-Length: 788 [ { @@ -643,6 +650,7 @@ Content-Length: 752 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -672,6 +680,7 @@ Content-Length: 752 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test index fcbbb8f196..2eaf4e7c1e 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test @@ -300,7 +300,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 447 +Content-Length: 465 { "id": "REGEX([0-9a-f]{24})", @@ -309,6 +309,7 @@ Content-Length: 447 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "StructuredValue", @@ -363,7 +364,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -372,6 +373,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -410,7 +412,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 448 +Content-Length: 466 { "id": "REGEX([0-9a-f]{24})", @@ -419,6 +421,7 @@ Content-Length: 448 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "StructuredValue", @@ -477,7 +480,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 444 +Content-Length: 462 { "id": "REGEX([0-9a-f]{24})", @@ -486,6 +489,7 @@ Content-Length: 444 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "C": { "type": "StructuredValue", @@ -540,7 +544,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -549,6 +553,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -589,7 +594,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -598,6 +603,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -636,7 +642,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 450 +Content-Length: 468 { "id": "REGEX([0-9a-f]{24})", @@ -645,6 +651,7 @@ Content-Length: 450 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "StructuredValue", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test index 4ed1f9ce81..0ea36db1d8 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test @@ -266,7 +266,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 386 +Content-Length: 404 { "id": "REGEX([0-9a-f]{24})", @@ -275,6 +275,7 @@ Content-Length: 386 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -319,7 +320,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -328,6 +329,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -366,7 +368,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 376 +Content-Length: 394 { "id": "REGEX([0-9a-f]{24})", @@ -375,6 +377,7 @@ Content-Length: 376 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -418,7 +421,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 366 +Content-Length: 384 { "id": "REGEX([0-9a-f]{24})", @@ -427,6 +430,7 @@ Content-Length: 366 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "id": "E3", "type": "T3" @@ -468,7 +472,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -477,6 +481,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -517,7 +522,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -526,6 +531,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -564,7 +570,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 388 +Content-Length: 406 { "id": "REGEX([0-9a-f]{24})", @@ -573,6 +579,7 @@ Content-Length: 388 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test b/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test index 9aff1e0a20..4054d27904 100644 --- a/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test +++ b/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test @@ -262,7 +262,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 340 +Content-Length: 358 { "id": "REGEX([0-9a-f]{24})", @@ -271,6 +271,7 @@ Content-Length: 340 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -300,7 +301,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 342 +Content-Length: 360 [ { @@ -310,6 +311,7 @@ Content-Length: 342 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -348,7 +350,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 341 +Content-Length: 359 { "id": "REGEX([0-9a-f]{24})", @@ -357,6 +359,7 @@ Content-Length: 341 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -386,7 +389,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 343 +Content-Length: 361 [ { @@ -396,6 +399,7 @@ Content-Length: 343 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -434,7 +438,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 341 +Content-Length: 359 { "id": "REGEX([0-9a-f]{24})", @@ -443,6 +447,7 @@ Content-Length: 341 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -472,7 +477,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 343 +Content-Length: 361 [ { @@ -482,6 +487,7 @@ Content-Length: 343 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -520,7 +526,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 340 +Content-Length: 358 { "id": "REGEX([0-9a-f]{24})", @@ -529,6 +535,7 @@ Content-Length: 340 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -558,7 +565,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 342 +Content-Length: 360 [ { @@ -568,6 +575,7 @@ Content-Length: 342 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -612,7 +620,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 340 +Content-Length: 358 { "id": "REGEX([0-9a-f]{24})", @@ -621,6 +629,7 @@ Content-Length: 340 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -650,7 +659,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 342 +Content-Length: 360 [ { @@ -660,6 +669,7 @@ Content-Length: 342 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", diff --git a/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test b/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test index 8352a77b7b..191958e178 100644 --- a/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test +++ b/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test @@ -146,7 +146,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 524 +Content-Length: 542 { "description": "DateTime test", @@ -159,6 +159,7 @@ Content-Length: 524 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "dateobservedto": { "type": "DateTime", @@ -205,7 +206,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 528 +Content-Length: 546 { "description": "DateTime test", @@ -218,6 +219,7 @@ Content-Length: 528 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "dateobservedto2": { "type": "DateTime", From f2a601afd91ff749119ddbb09953252700e0b711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 26 Feb 2024 18:30:57 +0100 Subject: [PATCH 036/104] FIX temporarly force legacy mode in ExprContext to check all ftest in legacy mode --- src/lib/ngsiNotify/Notifier.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 3f8516798f..01fbcc4ec0 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -318,7 +318,9 @@ static SenderThreadParams* buildSenderParamsCustom const std::string exprLang = notification.type == ngsiv2::HttpNotification ? notification.httpInfo.exprLang : notification.mqttInfo.exprLang; // Used by several macroSubstitute() calls along this function - ExprContextObject exprContext(exprLang == "legacy"); + // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode + //ExprContextObject exprContext(exprLang == "legacy"); + ExprContextObject exprContext(true); exprContext.add("id", en.id); exprContext.add("type", en.type); exprContext.add("service", tenant); From 450374abc86b23906a55a1e5f61fb342bd90b9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 28 Feb 2024 19:00:10 +0100 Subject: [PATCH 037/104] wip --- src/lib/common/macroSubstitute.cpp | 10 +++- src/lib/expressions/ExprContext.cpp | 87 ++++++++++------------------- src/lib/expressions/ExprContext.h | 6 +- src/lib/expressions/ExprResult.cpp | 4 +- src/lib/ngsi/ContextAttribute.cpp | 21 ++++++- src/lib/ngsi/ContextAttribute.h | 2 +- src/lib/ngsiNotify/Notifier.cpp | 4 +- 7 files changed, 65 insertions(+), 69 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index e1619ca77c..23b631a710 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -60,7 +60,7 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e } else { - // in legacy mode an extra remove quotes step is needed + // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { return removeQuotes(r.toString()); @@ -80,6 +80,7 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // error already logged in macroSubstitute, using stringValue itself as failsafe effectiveValue = stringValue; } + // toJsonString will stringfly JSON values in macros return '"' + toJsonString(effectiveValue) + '"'; } @@ -108,6 +109,13 @@ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, c else { std::string s = r.toString(); + + // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. + if (exprContextObjectP->isLegacy()) + { + s = removeQuotes(s); + } + if (raw) { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 87f1de9f9d..43737af844 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -81,11 +81,18 @@ std::map* ExprContextObject::getMap(void) * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string &key, const std::string &_value) +void ExprContextObject::add(const std::string &key, const std::string &_value, bool raw) { if (legacy) { - repl.insert(std::pair(key, _value)); + std::string value = _value; + if (!raw) + { + // This is the case of regular string. The raw case is for JSON generated from compound values + value = '"' + _value + '"'; + } + LM_T(LmtExpr, ("adding to legacy expression context object (string): %s=%s", key.c_str(), value.c_str())); + repl.insert(std::pair(key, value)); } else { @@ -93,7 +100,7 @@ void ExprContextObject::add(const std::string &key, const std::string &_value) { return; } - LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); + LM_T(LmtExpr, ("adding to JEXL expression context object (string): %s=%s", key.c_str(), _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); if (value == NULL) { @@ -115,6 +122,7 @@ void ExprContextObject::add(const std::string &key, double _value) { if (legacy) { + LM_T(LmtExpr, ("adding to legacy expression context object (double): %s=%f", key.c_str(), _value)); repl.insert(std::pair(key, double2string(_value))); } else @@ -123,7 +131,7 @@ void ExprContextObject::add(const std::string &key, double _value) { return; } - LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); + LM_T(LmtExpr, ("adding to JEXL expression context object (double): %s=%f", key.c_str(), _value)); PyObject* value = Py_BuildValue("d", _value); if (value == NULL) { @@ -145,6 +153,7 @@ void ExprContextObject::add(const std::string &key, bool _value) { if (legacy) { + LM_T(LmtExpr, ("adding to legacy expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); repl.insert(std::pair(key, _value? "true": "false")); } else @@ -153,7 +162,7 @@ void ExprContextObject::add(const std::string &key, bool _value) { return; } - LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); + LM_T(LmtExpr, ("adding to JEXL expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); if (_value) { PyDict_SetItemString(jexlContext, key.c_str(), Py_True); @@ -175,6 +184,7 @@ void ExprContextObject::add(const std::string &key) { if (legacy) { + LM_T(LmtExpr, ("adding to legacy expression context object (none): %s", key.c_str())); repl.insert(std::pair(key, "null")); } else @@ -183,7 +193,7 @@ void ExprContextObject::add(const std::string &key) { return; } - LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); + LM_T(LmtExpr, ("adding to JEXL expression context object (none): %s", key.c_str())); PyDict_SetItemString(jexlContext, key.c_str(), Py_None); } } @@ -198,8 +208,7 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex { if (legacy) { - // FIXME PR: not sure if this is going to work... - repl.insert(std::pair(key, exprContextObject.toString())); + LM_E(("Runtime Error (this method must not be invoked in legacy mode)")); } else { @@ -207,7 +216,8 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex { return; } - LM_T(LmtExpr, ("adding to expression context object (object): %s=%s", key.c_str(), exprContextObject.toString().c_str())); + // FIXME P3: improve this implemeting a toString() method for ExprContextObject + LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=[object]", key.c_str()); PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.getJexlContext()); } } @@ -222,8 +232,7 @@ void ExprContextObject::add(const std::string &key, ExprContextList exprContextL { if (legacy) { - // FIXME PR: not sure if this is going to work... - repl.insert(std::pair(key, exprContextList.toString())); + LM_E(("Runtime Error (this method must not be invoked in legacy mode)")); } else { @@ -231,7 +240,8 @@ void ExprContextObject::add(const std::string &key, ExprContextList exprContextL { return; } - LM_T(LmtExpr, ("adding to expression context object (list): %s=%s", key.c_str(), exprContextList.toString().c_str())); + // FIXME P3: improve this implemeting a toString() method for ExprContextList + LM_T(LmtExpr, ("adding to JEXL expression context object (list): %s=[list]", key.c_str())); PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); } } @@ -248,25 +258,6 @@ bool ExprContextObject::isLegacy(void) } -/* **************************************************************************** -* -* ExprContextObject::toString - -*/ -std::string ExprContextObject::toString(void) -{ - const char* str = PyUnicode_AsUTF8(jexlContext); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - return ""; - } - else - { - return std::string(str); - } -} - - /* **************************************************************************** * @@ -323,7 +314,7 @@ void ExprContextList::add(const std::string &_value) { return; } - LM_T(LmtExpr, ("adding to expression context list (string): %s=", _value.c_str())); + LM_T(LmtExpr, ("adding to JEXL expression context list (string): %s=", _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); if (value == NULL) { @@ -346,7 +337,7 @@ void ExprContextList::add(double _value) { return; } - LM_T(LmtExpr, ("adding to expression context list (double): %f", _value)); + LM_T(LmtExpr, ("adding to JEXL expression context list (double): %f", _value)); PyObject* value = Py_BuildValue("d", _value); if (value == NULL) { @@ -369,7 +360,7 @@ void ExprContextList::add(bool _value) { return; } - LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value ? "true" : "false")); + LM_T(LmtExpr, ("adding to JEXL expression context list (bool): %s", _value ? "true" : "false")); if (_value) { PyList_Append(jexlContext, Py_True); @@ -392,7 +383,7 @@ void ExprContextList::add(void) { return; } - LM_T(LmtExpr, ("adding to expression context list (none)")); + LM_T(LmtExpr, ("adding to JEXL expression context list (none)")); PyList_Append(jexlContext, Py_None); } @@ -408,7 +399,8 @@ void ExprContextList::add(ExprContextObject exprContextObject) { return; } - LM_T(LmtExpr, ("adding to expression context list (object): %s", exprContextObject.toString().c_str())); + // FIXME P3: improve this implemeting a toString() method for ExprContextObject + LM_T(LmtExpr, ("adding to JEXL expression context list (object): [object]"))); PyList_Append(jexlContext, exprContextObject.getJexlContext()); } @@ -424,32 +416,13 @@ void ExprContextList::add(ExprContextList exprContextList) { return; } - LM_T(LmtExpr, ("adding to expression context list (list): %s", exprContextList.toString().c_str())); + // FIXME P3: improve this implemeting a toString() method for ExprContextList + LM_T(LmtExpr, ("adding to JEXL expression context list (list): [list]")); PyList_Append(jexlContext, exprContextList.get()); } -/* **************************************************************************** -* -* ExprContextList::toString - -*/ -std::string ExprContextList::toString(void) -{ - const char* str = PyUnicode_AsUTF8(jexlContext); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - return ""; - } - else - { - return std::string(str); - } -} - - - /* **************************************************************************** * * ExprContextList::relesase - diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 19c165f276..a3068b3138 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -49,7 +49,7 @@ class ExprContextObject PyObject* getJexlContext(void); std::map* getMap(void); - void add(const std::string& key, const std::string& value); + void add(const std::string& key, const std::string& value, bool raw = false); void add(const std::string& key, double value); void add(const std::string& key, bool value); void add(const std::string& key); @@ -58,8 +58,6 @@ class ExprContextObject bool isLegacy(void); - std::string toString(void); - void release(void); }; @@ -79,8 +77,6 @@ class ExprContextList void add(ExprContextObject exprContextObject); void add(ExprContextList exprContextList); - std::string toString(void); - void release(void); }; diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 986ca52e53..a57cbc9ab4 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -341,7 +341,9 @@ std::string ExprResult::toString(void) } else if (valueType == orion::ValueTypeString) { - return "\"" + toJsonString(stringValue) + "\""; + // FIXME PR: does this break the no legacy + //return "\"" + toJsonString(stringValue) + "\""; + return "\"" + stringValue + "\""; } else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) { diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 4bf143ba76..77e86480a9 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1296,17 +1296,32 @@ std::string ContextAttribute::toJsonAsValue * * Pretty similar in structure to toJsonValue */ -void ContextAttribute::addToContext(ExprContextObject* exprContextObjectP) +void ContextAttribute::addToContext(ExprContextObject* exprContextObjectP, bool legacy) { if (compoundValueP != NULL) { + // In legacy expression, objects are vector are strings to be stored in a std::map if (valueType == orion::ValueTypeObject) { - exprContextObjectP->add(name, compoundValueP->toExprContextObject()); + if (legacy) + { + exprContextObjectP->add(name, compoundValueP->toJson(), true); + } + else + { + exprContextObjectP->add(name, compoundValueP->toExprContextObject()); + } } else // valueType == orion::ValueTypeVector { - exprContextObjectP->add(name, compoundValueP->toExprContextList()); + if (legacy) + { + exprContextObjectP->add(name, compoundValueP->toJson(), true); + } + else + { + exprContextObjectP->add(name, compoundValueP->toExprContextList()); + } } } else if (valueType == orion::ValueTypeNumber) diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index 23b74c06f1..caa36cbfe5 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -119,7 +119,7 @@ typedef struct ContextAttribute MimeType* outMimeTypeP, HttpStatusCode* scP); - void addToContext(ExprContextObject* exprContextObjectP); + void addToContext(ExprContextObject* exprContextObjectP, bool legacy); void release(void); std::string getName(void); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 01fbcc4ec0..8c09270602 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -328,7 +328,9 @@ static SenderThreadParams* buildSenderParamsCustom exprContext.add("authToken", xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - en.attributeVector[ix]->addToContext(&exprContext); + // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode + //en.attributeVector[ix]->addToContext(&exprContext, exprLang == "legacy"); + en.attributeVector[ix]->addToContext(&exprContext, true); } // From 238667780e62055d18643c4f9b18f3de9c5c11f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 5 Mar 2024 18:56:14 +0100 Subject: [PATCH 038/104] FIX ftest for legacy forced test --- src/lib/expressions/ExprContext.cpp | 4 +- src/lib/ngsiNotify/Notifier.cpp | 29 +- ...n_templates_many_notifications_legacy.test | 449 ++++++++++++++++++ .../covered_custom_notification_legacy.test | 217 +++++++++ 4 files changed, 689 insertions(+), 10 deletions(-) create mode 100644 test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test create mode 100644 test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 43737af844..46e85c5e1e 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -217,7 +217,7 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex return; } // FIXME P3: improve this implemeting a toString() method for ExprContextObject - LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=[object]", key.c_str()); + LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=[object]", key.c_str())); PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.getJexlContext()); } } @@ -400,7 +400,7 @@ void ExprContextList::add(ExprContextObject exprContextObject) return; } // FIXME P3: improve this implemeting a toString() method for ExprContextObject - LM_T(LmtExpr, ("adding to JEXL expression context list (object): [object]"))); + LM_T(LmtExpr, ("adding to JEXL expression context list (object): [object]")); PyList_Append(jexlContext, exprContextObject.getJexlContext()); } diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 8c09270602..a2f0c14071 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -319,18 +319,31 @@ static SenderThreadParams* buildSenderParamsCustom // Used by several macroSubstitute() calls along this function // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode - //ExprContextObject exprContext(exprLang == "legacy"); - ExprContextObject exprContext(true); + //bool legacy = (exprLang == "legacy"); + bool legacy = true; + ExprContextObject exprContext(legacy); + + // It seems that add() semantics are different in legacy and jexl mode. In jexl mode, if the key already exists, it is + // updated. In legacy model, if the key already exists, the operation is ignored (so previous value is preserved). Taking + // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence + // over macros comming from macros of the same name we conditionally add them depending the case exprContext.add("id", en.id); exprContext.add("type", en.type); - exprContext.add("service", tenant); - exprContext.add("servicePath", en.servicePath); - exprContext.add("authToken", xauthToken); + if (!legacy) + { + exprContext.add("service", tenant); + exprContext.add("servicePath", en.servicePath); + exprContext.add("authToken", xauthToken); + } for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode - //en.attributeVector[ix]->addToContext(&exprContext, exprLang == "legacy"); - en.attributeVector[ix]->addToContext(&exprContext, true); + en.attributeVector[ix]->addToContext(&exprContext, legacy); + } + if (legacy) + { + exprContext.add("service", tenant); + exprContext.add("servicePath", en.servicePath); + exprContext.add("authToken", xauthToken); } // diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test new file mode 100644 index 0000000000..d624115962 --- /dev/null +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test @@ -0,0 +1,449 @@ +# Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Notification Templates - many notifications (legacy expressions + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create regular subscription for E1 +# 02. Create same subscription but with 'httpCustom' instead of 'http' +# 03. Create custom subscription for E1 +# 04. Create E1/T1 +# 05. Create E1/T2 +# 06. Create E1/T3 +# 07. Dump accumulator and see 9 notifications +# + +echo "01. Create regular subscription for E1" +echo "======================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1" + } + ], + "condition": { + "attrs": [] + } + }, + "notification": { + "http": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + }, + "throttling": 0 +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create same subscription but with 'httpCustom' instead of 'http'" +echo "====================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1" + } + ], + "condition": { + "attrs": [] + } + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + }, + "exprLang": "legacy" + }, + "throttling": 0 +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create custom subscription for E1" +echo "=====================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1" + } + ], + "condition": { + "attrs": [] + } + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "method": "PUT", + "payload": "{ %22A1%22: %22${A1}%22, %22A2%22: %22${A2}%22, %22A3%22: %22${A3}%22 }", + "qs": { "id": "${id}", "type": "${type}", "a1": "${A1}", "a2": "${A2}", "a3": "${A3}" }, + "headers": { "entity-id": "${id}", "entity-type": "${type}", "A1": "${A1}", "A2": "${A2}", "A3": "${A3}" }, + "exprLang": "legacy" + } + }, + "throttling": 0 +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "04. Create E1/T1" +echo "================" +payload='{ + "id": "E1", + "type": "T1", + "A1": true +}' +orionCurl --url /v2/entities?options=keyValues --payload "$payload" +echo +echo + + +echo "05. Create E1/T2" +echo "================" +payload='{ + "id": "E1", + "type": "T2", + "A2": null +}' +orionCurl --url /v2/entities?options=keyValues --payload "$payload" +echo +echo + + +echo "06. Create E1/T3" +echo "================" +payload='{ + "id": "E1", + "type": "T3", + "A3": 3 +}' +orionCurl --url /v2/entities?options=keyValues --payload "$payload" +echo +echo + + +echo "07. Dump accumulator and see 9 notifications" +echo "============================================" +sleep 0.5 # Without this sleep, sometimes the last notification is missing +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create regular subscription for E1 +====================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create same subscription but with 'httpCustom' instead of 'http' +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom subscription for E1 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create E1/T1 +================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T1 +Content-Length: 0 + + + +05. Create E1/T2 +================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T2 +Content-Length: 0 + + + +06. Create E1/T3 +================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T3 +Content-Length: 0 + + + +07. Dump accumulator and see 9 notifications +#SORT_START +============================================ +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 129 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A1": { + "metadata": {}, + "type": "Boolean", + "value": true + }, + "id": "E1", + "type": "T1" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 129 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A1": { + "metadata": {}, + "type": "Boolean", + "value": true + }, + "id": "E1", + "type": "T1" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 +Fiware-Servicepath: / +Entity-Id: E1 +Content-Length: 36 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +A1: true +Entity-Type: T1 +Content-Type: text/plain; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "A1": "true", + "A2": "", + "A3": "" +} +#SORT_END +#SORT_START +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 126 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A2": { + "metadata": {}, + "type": "None", + "value": null + }, + "id": "E1", + "type": "T2" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 126 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A2": { + "metadata": {}, + "type": "None", + "value": null + }, + "id": "E1", + "type": "T2" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +PUT http://127.0.0.1:REGEX(\d+)/notify?a2=null&id=E1&type=T2 +Fiware-Servicepath: / +Entity-Id: E1 +Content-Length: 36 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +A2: null +Entity-Type: T2 +Content-Type: text/plain; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "A1": "", + "A2": "null", + "A3": "" +} +#SORT_END +#SORT_START +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 125 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A3": { + "metadata": {}, + "type": "Number", + "value": 3 + }, + "id": "E1", + "type": "T3" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 125 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A3": { + "metadata": {}, + "type": "Number", + "value": 3 + }, + "id": "E1", + "type": "T3" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 +Fiware-Servicepath: / +A3: 3 +Entity-Id: E1 +Content-Length: 33 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Entity-Type: T3 +Content-Type: text/plain; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "A1": "", + "A2": "", + "A3": "3" +} +======================================= +#SORT_END + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test new file mode 100644 index 0000000000..8321f78322 --- /dev/null +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test @@ -0,0 +1,217 @@ +# Copyright 2022 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Covered custom notifictions (legacy expressions) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create covered custom subscriptions for E1 covering attributes A1 and A2 +# 02. Create not covered custom subscriptions for E2 +# 03. Create E1 with attribute A1=1 +# 04. Dump & reset, see notifications A1=1 and A2=null in custom payload +# 05. Create E2 with attribute A1=1 +# 06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) +# + +echo "01. Create covered custom subscriptions for E1 covering attributes A1 and A2" +echo "============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "covered": true, + "attrs": [ "A1", "A2" ], + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "payload": "{ %22A1-value%22: ${A1}, %22A2-value%22: ${A2} }", + "headers": { + "content-type": "application/json%3B charset%3Dutf-8" + }, + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create not covered custom subscriptions for E2" +echo "==================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E2", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "payload": "{ %22A1-value%22: ${A1}, %22A2-value%22: ${A2} }", + "headers": { + "content-type": "application/json%3B charset%3Dutf-8" + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create E1 with attribute A1=1" +echo "=================================" +payload='{ + "id": "E1", + "type": "T", + "A1": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "04. Dump & reset, see notifications A1=1 and A2=null in custom payload" +echo "======================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +echo "05. Create E2 with attribute A1=1" +echo "=================================" +payload='{ + "id": "E2", + "type": "T", + "A1": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON)" +echo "========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create covered custom subscriptions for E1 covering attributes A1 and A2 +============================================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create not covered custom subscriptions for E2 +================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create E1 with attribute A1=1 +================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +04. Dump & reset, see notifications A1=1 and A2=null in custom payload +====================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 35 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ "A1-value": 1, "A2-value": null }======================================= + + +05. Create E2 with attribute A1=1 +================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) +======================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 31 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ "A1-value": 1, "A2-value": }======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 742936f9b54c2ee7a014041928e2fe659d5ef160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Mar 2024 09:42:17 +0100 Subject: [PATCH 039/104] FIX rollback hack to use always legacy expressions --- src/lib/ngsiNotify/Notifier.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index a2f0c14071..6ee599bd5f 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -318,9 +318,7 @@ static SenderThreadParams* buildSenderParamsCustom const std::string exprLang = notification.type == ngsiv2::HttpNotification ? notification.httpInfo.exprLang : notification.mqttInfo.exprLang; // Used by several macroSubstitute() calls along this function - // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode - //bool legacy = (exprLang == "legacy"); - bool legacy = true; + bool legacy = (exprLang == "legacy"); ExprContextObject exprContext(legacy); // It seems that add() semantics are different in legacy and jexl mode. In jexl mode, if the key already exists, it is From ea94c572e53c673810011fd72c64f594004dbe5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Mar 2024 16:15:40 +0100 Subject: [PATCH 040/104] FIX change pyjexl by tcjexl --- ci/deb/build-dep.sh | 4 +- src/lib/expressions/ExprManager.cpp | 18 +-- src/lib/expressions/ExprManager.h | 2 +- .../jexl_transformation.test | 144 ++++++++++++++++++ 4 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test diff --git a/ci/deb/build-dep.sh b/ci/deb/build-dep.sh index 4d35fe450e..28540d6fe6 100755 --- a/ci/deb/build-dep.sh +++ b/ci/deb/build-dep.sh @@ -55,7 +55,7 @@ echo "INSTALL: MongoDB shell" \ && apt-get -y update \ && apt-get -y install mongodb-mongosh -# pyjexl is needed by Context Broker functionality, the others are for ftest suite +# tcjexl is needed by Context Broker functionality, the others are for ftest suite echo "INSTALL: python special dependencies" \ && cd /opt \ && python3 -m venv /opt/ft_env \ @@ -64,7 +64,7 @@ echo "INSTALL: python special dependencies" \ && pip install Werkzeug==2.0.2 \ && pip install paho-mqtt==1.6.1 \ && pip install amqtt==0.11.0b1 \ -&& pip install pyjexl==0.3.0 \ +&& pip install tcjexl==0.1.0 \ && deactivate # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0 diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index bfe6d14b81..d87f37f536 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -40,7 +40,7 @@ */ void ExprManager::init(void) { - pyjexlModule = NULL; + tcjexlModule = NULL; jexlEngine = NULL; if (sem_init(&sem, 0, 1) == -1) @@ -51,14 +51,14 @@ void ExprManager::init(void) Py_Initialize(); LM_T(LmtExpr, ("Python interpreter has been initialized")); - pyjexlModule = PyImport_ImportModule("pyjexl"); - if (pyjexlModule == NULL) + tcjexlModule = PyImport_ImportModule("tcjexl"); + if (tcjexlModule == NULL) { - LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", capturePythonError())); + LM_X(1, ("Fatal Error (error importing tcjexl module: %s)", capturePythonError())); } - LM_T(LmtExpr, ("pyjexl module has been loaded")); + LM_T(LmtExpr, ("tcjexl module has been loaded")); - jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); + jexlEngine = PyObject_CallMethod(tcjexlModule, "JEXL", NULL); if (jexlEngine == NULL) { LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", capturePythonError())); @@ -134,10 +134,10 @@ void ExprManager::release(void) LM_T(LmtExpr, ("jexl engine has been freed")); } - if (pyjexlModule != NULL) + if (tcjexlModule != NULL) { - Py_XDECREF(pyjexlModule); - LM_T(LmtExpr, ("pyjexl module has been freed")); + Py_XDECREF(tcjexlModule); + LM_T(LmtExpr, ("tcjexl module has been freed")); } Py_Finalize(); diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 92525900e0..55c8e90f96 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -39,7 +39,7 @@ class ExprManager { private: - PyObject* pyjexlModule; + PyObject* tcjexlModule; PyObject* jexlEngine; PyObject* jsonModule; sem_t sem; diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test new file mode 100644 index 0000000000..693818d970 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test @@ -0,0 +1,144 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using transformation) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression low: A|lowercase +# 02. Create entity E1 with A=IwantAllInLOWERCase +# 03. Dump accumulator and see notifications (low: iwantallinlowercase) +# + + +echo "01. Create custom sub with custom expression low: A|lowercase" +echo "=============================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "low": { + "value": "${A|lowercase}", + "type": "Calculated" + } + } + }, + "attrs": [ "low" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=IwantAllInLOWERCase" +echo "===============================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": "IwantAllInLOWERCase", + "type": "Text" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Dump accumulator and see notifications (low: iwantallinlowercase)" +echo "=====================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression low: A|lowercase +============================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=IwantAllInLOWERCase +=============================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Dump accumulator and see notifications (low: iwantallinlowercase) +===================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 149 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "low": { + "metadata": {}, + "type": "Calculated", + "value": "iwantallinlowercase" + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From a71c173a3a9092b42d5df85e9bc61c9ce0311a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 7 Mar 2024 12:20:42 +0100 Subject: [PATCH 041/104] ADD ftest for null values and not defined attrs cases --- .../jexl_basic_attrs_in_update.test | 2 +- .../jexl_basic_attrs_not_in_update.test | 2 +- .../jexl_expr_attrs_weird_syntax.test | 2 +- .../jexl_json_navigation.test | 2 +- .../jexl_not_defined_attrs.test | 191 ++++++++++++++++++ .../jexl_null_values.test | 191 ++++++++++++++++++ 6 files changed, 386 insertions(+), 4 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test index 5e031e80e7..601f69d25a 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification (source attributes in udpate) +JEXL basic expression in custom notification (source attributes in udpate) --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test index eaef9821eb..f6c634a077 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification (source attributes not in udpate) +JEXL basic expression in custom notification (source attributes not in udpate) --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test index cf42ab7ebe..ac53119941 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -Legacy expression using attributes with weird syntax +JEXL expression using attributes with weird syntax --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test index 236b872365..05e8a0b061 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification (source attributes not in udpate) +JEXL expression in custom notification (JSON navigation in objects and arrays) --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test new file mode 100644 index 0000000000..4fe7de3997 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test @@ -0,0 +1,191 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (not defined attributes) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression cal: A+1 +# 02. Create entity E1 with B=1 +# 03. Update entity E1 with B=2 +# 04. Dump accumulator and see notifications (cal: null, cal: null) +# + + +echo "01. Create custom sub with custom expression cal: A+1" +echo "=====================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "cal": { + "value": "${A+1}", + "type": "Calculated" + } + } + }, + "attrs": [ "cal" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with B=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with B=2" +echo "=============================" +payload='{ + "B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (cal: null, cal: null)" +echo "=================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression cal: A+1 +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with B=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with B=2 +============================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (cal: null, cal: null) +================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test new file mode 100644 index 0000000000..35bfbe4567 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test @@ -0,0 +1,191 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (null values) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression cal: A+1 +# 02. Create entity E1 with A=null +# 03. Update entity E1 with A=null again (with forcedUpdate) +# 04. Dump accumulator and see notifications (cal: null, cal: null) +# + + +echo "01. Create custom sub with custom expression cal: A+1" +echo "=====================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "cal": { + "value": "${A+1}", + "type": "Calculated" + } + } + }, + "attrs": [ "cal" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=null" +echo "================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A=null again (with forcedUpdate)" +echo "==========================================================" +payload='{ + "A": { + "value": "null", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs?options=forcedUpdate -X POST --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (cal: null, cal: null)" +echo "=================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression cal: A+1 +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A=null again (with forcedUpdate) +========================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (cal: null, cal: null) +================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From e6f0e6bdcf328fd623d0feed38bd78d71728af77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 7 Mar 2024 12:46:27 +0100 Subject: [PATCH 042/104] ADD ftest for array filtering --- .../jexl_array_filtering.test | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test new file mode 100644 index 0000000000..c20beb102f --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test @@ -0,0 +1,297 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (array filtering) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression sum: a[.b<5] +# 02. Create entity E1 with a=[{b:1},{b:34},{b:4}] +# 03. Update entity E1 with a=[{c:1},{b:-1},{b:2}] +# 04. Update entity E1 with a=[{b:-1},{b:foo}] +# 05. Update entity E1 with a=[{b:-1, c:1},{b:20, d:3}] +# 06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}]) +# + + +echo "01. Create custom sub with custom expression sum: a[.b<5]" +echo "=========================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "cal": { + "value": "${a[.b<5]}", + "type": "TextUnrestricted" + } + } + }, + "attrs": [ "cal" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with a=[{b:1},{b:34},{b:4}]" +echo "================================================" +payload='{ + "id": "E1", + "type": "T", + "a": { + "value": [{"b": 1}, {"b": 34}, {"b": 4}], + "type": "Array" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with a=[{c:1},{b:-1},{b:2}]" +echo "================================================" +payload='{ + "a": { + "value": [{"c": 1}, {"b": -1}, {"b": 2}], + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "04. Update entity E1 with a=[{b:-1},{b:foo}]" +echo "============================================" +payload='{ + "a": { + "value": [{"b": -1}, {"b": "foo"}], + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "05. Update entity E1 with a=[{b:-1, c:1},{b:20, d:3}]" +echo "=====================================================" +payload='{ + "a": { + "value": [{"b": -1, "c": 1}, {"b": 20, "d": 3}], + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}])" +echo "=====================================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: a[.b<5] +========================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with a=[{b:1},{b:34},{b:4}] +================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with a=[{c:1},{b:-1},{b:2}] +================================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Update entity E1 with a=[{b:-1},{b:foo}] +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Update entity E1 with a=[{b:-1, c:1},{b:20, d:3}] +===================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}]) +===================================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 151 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "TextUnrestricted", + "value": [ + { + "b": 1 + }, + { + "b": 4 + } + ] + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 138 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "TextUnrestricted", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 138 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "TextUnrestricted", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 150 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "TextUnrestricted", + "value": [ + { + "b": -1, + "c": 1 + } + ] + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 3a1474ab03096e92ea6c74afc67ea144f38664bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 11 Mar 2024 11:37:44 +0100 Subject: [PATCH 043/104] FIX upgrade tcjexl to 0.2.0 in CI Dockerfile --- ci/deb/build-dep.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deb/build-dep.sh b/ci/deb/build-dep.sh index 28540d6fe6..aac74f5988 100755 --- a/ci/deb/build-dep.sh +++ b/ci/deb/build-dep.sh @@ -64,7 +64,7 @@ echo "INSTALL: python special dependencies" \ && pip install Werkzeug==2.0.2 \ && pip install paho-mqtt==1.6.1 \ && pip install amqtt==0.11.0b1 \ -&& pip install tcjexl==0.1.0 \ +&& pip install tcjexl==0.2.0 \ && deactivate # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0 From 0c6fcd4d10c171c270c8346ee23626772ce69b29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 11 Mar 2024 12:31:58 +0100 Subject: [PATCH 044/104] ADD ftest jexl_transformation_with_arguments --- .../jexl_transformation_with_arguments.test | 250 ++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test new file mode 100644 index 0000000000..85075dc93a --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test @@ -0,0 +1,250 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using transformation with arguments) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression included: S|includes(R) +# 02. Create entity E1 with S=a given string, R=given +# 03. Update entity E1 with S=anotherstring, R=strong +# 04. Update entity E1 with S=morestringarecoming, R=string +# 05. Dump accumulator and see notifications (included: true, included: false, include: true) +# + + +echo "01. Create custom sub with custom expression included: S|includes(R)" +echo "====================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "includes": { + "value": "${S|includes(R)}", + "type": "TextUnrestricted" + } + } + }, + "attrs": [ "includes" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with S=a given string, R=given" +echo "===================================================" +payload='{ + "id": "E1", + "type": "T", + "S": { + "value": "a given string", + "type": "Text" + }, + "R": { + "value": "given", + "type": "Text" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with S=anotherstring, R=strong" +echo "===================================================" +payload='{ + "S": { + "value": "anotherstring", + "type": "Text" + }, + "R": { + "value": "strong", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" +echo +echo + + +echo "04. Update entity E1 with S=morestringarecoming, R=string" +echo "=========================================================" +payload='{ + "S": { + "value": "morestringarecoming", + "type": "Text" + }, + "R": { + "value": "string", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see notifications (included: true, included: false, include: true)" +echo "===========================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression included: S|includes(R) +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with S=a given string, R=given +=================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with S=anotherstring, R=strong +=================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Update entity E1 with S=morestringarecoming, R=string +========================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see notifications (included: true, included: false, include: true) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 143 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "includes": { + "metadata": {}, + "type": "TextUnrestricted", + "value": true + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 144 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "includes": { + "metadata": {}, + "type": "TextUnrestricted", + "value": false + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 143 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "includes": { + "metadata": {}, + "type": "TextUnrestricted", + "value": true + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 2cc242432e489449dde8a4e1de7e94c2e0c75486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 11 Mar 2024 17:31:40 +0100 Subject: [PATCH 045/104] FIX remove forbidden chars restrictions in ngsi subfields --- doc/manuals/orion-api.md | 3 +- src/lib/jsonParseV2/parseContextAttribute.cpp | 5 +- src/lib/jsonParseV2/parseContextAttribute.h | 3 +- src/lib/jsonParseV2/parseSubscription.cpp | 14 +- src/lib/ngsi/ContextAttribute.cpp | 6 +- src/lib/ngsi/ContextAttribute.h | 2 +- .../jexl_array_filtering.test | 18 +- .../jexl_transformation_in_id_and_type.test | 197 ++++++++++++++++++ .../jexl_transformation_with_arguments.test | 14 +- 9 files changed, 227 insertions(+), 35 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 4fbc8d5ab0..50493f53c1 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -398,11 +398,12 @@ the following will be used (note that "%25" is the encoding for "%"). GET /v2/entities/E%253C01%253E ``` -There are some exception cases in which the above restrictions do not apply. In particular, in the following fields: +There are some exception cases in which the above restrictions do not apply. In particular, in the following cases: * URL parameter `q` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `mq` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `georel` and `coords` allow `;` +* Within `ngsi` (i.e. `id`, `type` and attribute values) in [NGSI Payload patching](#ngsi-payload-patching) (to support characters used in the JEXL expression syntax) * Whichever attribute value which uses `TextUnrestricted` as attribute type (see [Special Attribute Types](#special-attribute-types) section) ## Identifiers syntax restrictions diff --git a/src/lib/jsonParseV2/parseContextAttribute.cpp b/src/lib/jsonParseV2/parseContextAttribute.cpp index 751dfb66d3..caf6ac190a 100644 --- a/src/lib/jsonParseV2/parseContextAttribute.cpp +++ b/src/lib/jsonParseV2/parseContextAttribute.cpp @@ -254,7 +254,8 @@ std::string parseContextAttribute ConnectionInfo* ciP, const rapidjson::Value::ConstMemberIterator& iter, ContextAttribute* caP, - bool checkAttrSpecialTypes + bool checkAttrSpecialTypes, + bool relaxForbiddenCheck ) { std::string name = iter->name.GetString(); @@ -383,7 +384,7 @@ std::string parseContextAttribute caP->type = (compoundVector)? defaultType(orion::ValueTypeVector) : defaultType(caP->valueType); } - std::string r = caP->check(ciP->apiVersion, ciP->requestType); + std::string r = caP->check(ciP->apiVersion, ciP->requestType, relaxForbiddenCheck); if (r != "OK") { alarmMgr.badInput(clientIp, "JSON Parse Error", r); diff --git a/src/lib/jsonParseV2/parseContextAttribute.h b/src/lib/jsonParseV2/parseContextAttribute.h index e3210e9403..2ab0ed39e6 100644 --- a/src/lib/jsonParseV2/parseContextAttribute.h +++ b/src/lib/jsonParseV2/parseContextAttribute.h @@ -42,7 +42,8 @@ extern std::string parseContextAttribute ConnectionInfo* ciP, const rapidjson::Value::ConstMemberIterator& iter, ContextAttribute* caP, - bool checkAttrSpecialTypes + bool checkAttrSpecialTypes, + bool relaxForbiddenCheck = false ); diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index cea25d5b52..5fbe2faa38 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -525,11 +525,6 @@ static std::string parseCustomPayload } ngsi->id = iter->value.GetString(); - - if (forbiddenIdChars(V2, ngsi->id.c_str(), "")) - { - return badInput(ciP, ERROR_DESC_BAD_REQUEST_INVALID_CHAR_ENTID); - } } else if (name == "type") { @@ -539,11 +534,6 @@ static std::string parseCustomPayload } ngsi->type = iter->value.GetString(); - - if (forbiddenIdChars(V2, ngsi->type.c_str(), "")) - { - return badInput(ciP, ERROR_DESC_BAD_REQUEST_INVALID_CHAR_ENTTYPE); - } } else // attribute { @@ -551,7 +541,9 @@ static std::string parseCustomPayload ngsi->attributeVector.push_back(caP); - std::string r = parseContextAttribute(ciP, iter, caP, false); + // Note we are using relaxForbiddenCheck true in this case, as JEXL expressions typically use forbidden + // chars and we don't want to fail in that case + std::string r = parseContextAttribute(ciP, iter, caP, false, true); if (r == "max deep reached") { diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 77e86480a9..79ce3f57fc 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1363,7 +1363,7 @@ void ContextAttribute::addToContext(ExprContextObject* exprContextObjectP, bool * * ContextAttribute::check - */ -std::string ContextAttribute::check(ApiVersion apiVersion, RequestType requestType) +std::string ContextAttribute::check(ApiVersion apiVersion, RequestType requestType, bool relaxForbiddenCheck) { size_t len; char errorMsg[128]; @@ -1414,14 +1414,14 @@ std::string ContextAttribute::check(ApiVersion apiVersion, RequestType requestTy return "Invalid characters in attribute type"; } - if ((compoundValueP != NULL) && (compoundValueP->childV.size() != 0) && (type != TEXT_UNRESTRICTED_TYPE)) + if ((!relaxForbiddenCheck) && (compoundValueP != NULL) && (compoundValueP->childV.size() != 0) && (type != TEXT_UNRESTRICTED_TYPE)) { return compoundValueP->check(""); } if (valueType == orion::ValueTypeString) { - if ((type != TEXT_UNRESTRICTED_TYPE) && (forbiddenChars(stringValue.c_str()))) + if ((!relaxForbiddenCheck) && (type != TEXT_UNRESTRICTED_TYPE) && (forbiddenChars(stringValue.c_str()))) { alarmMgr.badInput(clientIp, "found a forbidden character in the value of an attribute", stringValue); return "Invalid characters in attribute value"; diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index caa36cbfe5..f242bf101d 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -134,7 +134,7 @@ typedef struct ContextAttribute /* Helper method to be use in some places wher '%s' is needed */ std::string getValue(void) const; - std::string check(ApiVersion apiVersion, RequestType requestType); + std::string check(ApiVersion apiVersion, RequestType requestType, bool relaxForbiddenCheck = false); ContextAttribute* clone(); bool compoundItemExists(const std::string& compoundPath, orion::CompoundValueNode** compoundItemPP = NULL); diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test index c20beb102f..0a3bb36e7d 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test @@ -57,7 +57,7 @@ payload='{ "ngsi": { "cal": { "value": "${a[.b<5]}", - "type": "TextUnrestricted" + "type": "Calculated" } } }, @@ -179,7 +179,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) ===================================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 151 +Content-Length: 145 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -192,7 +192,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": [ { "b": 1 @@ -211,7 +211,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 138 +Content-Length: 132 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -224,7 +224,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": null }, "id": "E1", @@ -236,7 +236,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 138 +Content-Length: 132 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -249,7 +249,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": null }, "id": "E1", @@ -261,7 +261,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 150 +Content-Length: 144 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -274,7 +274,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": [ { "b": -1, diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test new file mode 100644 index 0000000000..fc128a4e4d --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test @@ -0,0 +1,197 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using transformation in id and type) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression id=S|substring(0,2), type=S|substring(3,5) +# 02. Create entity E with S=E1:T1, A:1 +# 03. Update entity E with S=E2:T2, A:2 +# 04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2) +# + + +echo "01. Create custom sub with custom expression id=S|substring(0,2), type=S|substring(3,5)" +echo "=======================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "id": "${S|substring(0,2)}", + "type": "${S|substring(3,5)}" + } + }, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E with S=E1:T1, A:1" +echo "=====================================" +payload='{ + "id": "E", + "type": "T", + "S": { + "value": "E1:T1", + "type": "Text" + }, + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E with S=E2:T2, A:2" +echo "=====================================" +payload='{ + "S": { + "value": "E2:T2", + "type": "Text" + }, + "A": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2)" +echo "=================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression id=S|substring(0,2), type=S|substring(3,5) +======================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E with S=E1:T1, A:1 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. Update entity E with S=E2:T2, A:2 +===================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2) +================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 124 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "id": "E1", + "type": "T1" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 124 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "id": "E2", + "type": "T2" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test index 85075dc93a..5616763fde 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test @@ -56,7 +56,7 @@ payload='{ "ngsi": { "includes": { "value": "${S|includes(R)}", - "type": "TextUnrestricted" + "type": "Boolean" } } }, @@ -169,7 +169,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) =========================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 143 +Content-Length: 134 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -183,7 +183,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "id": "E1", "includes": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Boolean", "value": true }, "type": "T" @@ -194,7 +194,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 144 +Content-Length: 135 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -208,7 +208,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "id": "E1", "includes": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Boolean", "value": false }, "type": "T" @@ -219,7 +219,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 143 +Content-Length: 134 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -233,7 +233,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "id": "E1", "includes": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Boolean", "value": true }, "type": "T" From 3968dfa14480082c16386446f545ee14f2d465db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 12 Apr 2024 14:36:05 +0200 Subject: [PATCH 046/104] FIX remove Python jexl implementation --- CMakeLists.txt | 4 +- ci/deb/build-dep.sh | 2 - ci/deb/makefile | 3 - docker/Dockerfile | 1 - docker/Dockerfile.alpine | 1 - src/lib/common/JsonHelper.cpp | 11 ++ src/lib/common/JsonHelper.h | 1 + src/lib/expressions/CMakeLists.txt | 2 - src/lib/expressions/ExprContext.cpp | 181 ++++------------------- src/lib/expressions/ExprContext.h | 27 ++-- src/lib/expressions/ExprManager.cpp | 79 +++------- src/lib/expressions/ExprManager.h | 11 +- src/lib/expressions/ExprResult.cpp | 3 +- src/lib/expressions/ExprResult.h | 3 +- src/lib/expressions/exprCommon.cpp | 60 -------- src/lib/expressions/exprCommon.h | 41 ----- src/lib/ngsiNotify/Notifier.cpp | 1 + src/lib/serviceRoutines/versionTreat.cpp | 12 +- test/unittests/CMakeLists.txt | 1 - 19 files changed, 89 insertions(+), 355 deletions(-) delete mode 100644 src/lib/expressions/exprCommon.cpp delete mode 100644 src/lib/expressions/exprCommon.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 03bfaab443..b871e47ad8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,6 +234,7 @@ find_package (mongoc-1.0 1.24.3 EXACT) SET (COMMON_STATIC_LIBS microhttpd.a mosquitto.a + cjexl.a mongo::mongoc_static ) @@ -246,7 +247,6 @@ SET (DYNAMIC_LIBS uuid crypto sasl2 - python3.11 ) # @@ -265,8 +265,6 @@ endif (UNIT_TEST) # Common include # include_directories("/usr/include") -include_directories("/usr/include/python3.11/") - # Needed for the new C driver include_directories("/usr/local/include/libmongoc-1.0") diff --git a/ci/deb/build-dep.sh b/ci/deb/build-dep.sh index aac74f5988..ea8531172a 100755 --- a/ci/deb/build-dep.sh +++ b/ci/deb/build-dep.sh @@ -55,7 +55,6 @@ echo "INSTALL: MongoDB shell" \ && apt-get -y update \ && apt-get -y install mongodb-mongosh -# tcjexl is needed by Context Broker functionality, the others are for ftest suite echo "INSTALL: python special dependencies" \ && cd /opt \ && python3 -m venv /opt/ft_env \ @@ -64,7 +63,6 @@ echo "INSTALL: python special dependencies" \ && pip install Werkzeug==2.0.2 \ && pip install paho-mqtt==1.6.1 \ && pip install amqtt==0.11.0b1 \ -&& pip install tcjexl==0.2.0 \ && deactivate # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0 diff --git a/ci/deb/makefile b/ci/deb/makefile index 7646a27dbd..be524b592f 100644 --- a/ci/deb/makefile +++ b/ci/deb/makefile @@ -69,9 +69,6 @@ install_unit: install unit: @echo '------------------------------------- make unit ----------------------------------------' - # ft_env needed for pyjexl module. Very important to "chain" the two commands with '; \' - # otherwise it seems unitTest doesn't get the venv and it uses default Python in the system - . /opt/ft_env/bin/activate ; \ BUILD_UNIT/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:/tmp/builder/logs/unit.xml build_functional: prepare diff --git a/docker/Dockerfile b/docker/Dockerfile index a54f4844f4..b5b80157d4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -88,7 +88,6 @@ RUN \ cd cmake-build && \ # Different from ci/deb/build-dep.sh and build from source documentation, we add here also # the MONGOC_TEST_USE_CRYPT_SHARED=FALSE. It needs Python and in this tiny image we don't have it - # FIXME PR: now that Python is going to be include for JEXL functionalty, maybe this should be changed... cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DMONGOC_TEST_USE_CRYPT_SHARED=FALSE .. && \ make && \ make install && \ diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index b871d42a5d..524335f677 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -93,7 +93,6 @@ RUN \ cd cmake-build && \ # Different from ci/deb/build-dep.sh and build from source documentation, we add here also # the MONGOC_TEST_USE_CRYPT_SHARED=FALSE. It needs Python and in this tiny image we don't have it - # FIXME PR: now that Python is going to be include for JEXL functionalty, maybe this should be changed... cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DMONGOC_TEST_USE_CRYPT_SHARED=FALSE .. && \ make && \ make install && \ diff --git a/src/lib/common/JsonHelper.cpp b/src/lib/common/JsonHelper.cpp index 812e9ee1f1..3897f011cf 100644 --- a/src/lib/common/JsonHelper.cpp +++ b/src/lib/common/JsonHelper.cpp @@ -406,6 +406,17 @@ void JsonVectorHelper::addBool(bool b) +/* **************************************************************************** +* +* JsonObjectHelper::addNull - +*/ +void JsonVectorHelper::addNull(void) +{ + addRaw("null"); +} + + + /* **************************************************************************** * * JsonVectorHelper::str - diff --git a/src/lib/common/JsonHelper.h b/src/lib/common/JsonHelper.h index 52e633c14d..47a52c0ff0 100644 --- a/src/lib/common/JsonHelper.h +++ b/src/lib/common/JsonHelper.h @@ -64,6 +64,7 @@ class JsonVectorHelper void addNumber(double value); void addDate(double timestamp); void addBool(bool b); + void addNull(void); std::string str(); diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index bfdd9da83c..4f938e3634 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -21,7 +21,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES - exprCommon.cpp ExprManager.cpp ExprContext.cpp ExprResult.cpp @@ -29,7 +28,6 @@ SET (SOURCES ) SET (HEADERS - exprCommon.h ExprManager.h ExprContext.h ExprResult.h diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 46e85c5e1e..4b59840475 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -28,8 +28,6 @@ #include "common/string.h" #include "logMsg/logMsg.h" #include "expressions/ExprContext.h" -#include "expressions/exprCommon.h" - /* **************************************************************************** @@ -39,18 +37,6 @@ ExprContextObject::ExprContextObject(bool _legacy) { legacy = _legacy; - - if (!legacy) - { - jexlContext = PyDict_New(); - - if (jexlContext == NULL) - { - // Note that this ruins the object. We log this situation just one time here, but we include a NULL check - // in every other method to be safer - LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); - } - } } @@ -59,16 +45,16 @@ ExprContextObject::ExprContextObject(bool _legacy) * * ExprContextObject::getJexlContext - */ -PyObject* ExprContextObject::getJexlContext(void) +std::string ExprContextObject::getJexlContext(void) { - return jexlContext; + return jh.str(); } /* **************************************************************************** * -* ExprContextObject::getJexlContext - +* ExprContextObject::getMap - */ std::map* ExprContextObject::getMap(void) { @@ -96,19 +82,8 @@ void ExprContextObject::add(const std::string &key, const std::string &_value, b } else { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context object (string): %s=%s", key.c_str(), _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - if (value == NULL) - { - LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); - return; - } - PyDict_SetItemString(jexlContext, key.c_str(), value); - Py_DECREF(value); + jh.addString(key, _value); } } @@ -127,19 +102,8 @@ void ExprContextObject::add(const std::string &key, double _value) } else { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context object (double): %s=%f", key.c_str(), _value)); - PyObject* value = Py_BuildValue("d", _value); - if (value == NULL) - { - LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); - return; - } - PyDict_SetItemString(jexlContext, key.c_str(), value); - Py_DECREF(value); + jh.addNumber(key, _value); } } @@ -158,19 +122,8 @@ void ExprContextObject::add(const std::string &key, bool _value) } else { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); - if (_value) - { - PyDict_SetItemString(jexlContext, key.c_str(), Py_True); - } - else - { - PyDict_SetItemString(jexlContext, key.c_str(), Py_False); - } + jh.addBool(key, _value); } } @@ -189,12 +142,8 @@ void ExprContextObject::add(const std::string &key) } else { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context object (none): %s", key.c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), Py_None); + jh.addNull(key); } } @@ -212,13 +161,9 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex } else { - if (jexlContext == NULL) - { - return; - } - // FIXME P3: improve this implemeting a toString() method for ExprContextObject - LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=[object]", key.c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.getJexlContext()); + std::string s = exprContextObject.getJexlContext(); + LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=%s", key.c_str(), s.c_str())); + jh.addRaw(key, s); } } @@ -236,13 +181,9 @@ void ExprContextObject::add(const std::string &key, ExprContextList exprContextL } else { - if (jexlContext == NULL) - { - return; - } - // FIXME P3: improve this implemeting a toString() method for ExprContextList - LM_T(LmtExpr, ("adding to JEXL expression context object (list): %s=[list]", key.c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); + std::string s = exprContextList.get(); + LM_T(LmtExpr, ("adding to JEXL expression context object (list): %s=%s", key.c_str(), s.c_str())); + jh.addRaw(key, s); } } @@ -265,12 +206,7 @@ bool ExprContextObject::isLegacy(void) */ void ExprContextObject::release(void) { - if (jexlContext == NULL) - { - return; - } - // FIXME PR: this is not correct. Recursively release of the dict object - //Py_XDECREF(jexlContext); + // FIXME PR: this method is probably no longer needed } @@ -281,14 +217,7 @@ void ExprContextObject::release(void) */ ExprContextList::ExprContextList() { - jexlContext = PyList_New(0); - - if (jexlContext == NULL) - { - // Note that this ruins the object. We log this situation just one time here, but we include a NULL check - // in every other method to be safer - LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); - } + // FIXME PR: this method is probably no longer needed } @@ -297,9 +226,9 @@ ExprContextList::ExprContextList() * * ExprContextList::get - */ -PyObject* ExprContextList::get(void) +std::string ExprContextList::get(void) { - return jexlContext; + return jh.str(); } @@ -310,19 +239,9 @@ PyObject* ExprContextList::get(void) */ void ExprContextList::add(const std::string &_value) { - if (jexlContext == NULL) - { - return; - } - LM_T(LmtExpr, ("adding to JEXL expression context list (string): %s=", _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - if (value == NULL) - { - LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); - return; - } - PyList_Append(jexlContext, value); - Py_DECREF(value); + + LM_T(LmtExpr, ("adding to JEXL expression context list (string): %s", _value.c_str())); + jh.addString(_value); } @@ -333,19 +252,8 @@ void ExprContextList::add(const std::string &_value) */ void ExprContextList::add(double _value) { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context list (double): %f", _value)); - PyObject* value = Py_BuildValue("d", _value); - if (value == NULL) - { - LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); - return; - } - PyList_Append(jexlContext, value); - Py_DECREF(value); + jh.addNumber(_value); } @@ -356,19 +264,8 @@ void ExprContextList::add(double _value) */ void ExprContextList::add(bool _value) { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context list (bool): %s", _value ? "true" : "false")); - if (_value) - { - PyList_Append(jexlContext, Py_True); - } - else - { - PyList_Append(jexlContext, Py_False); - } + jh.addBool(_value); } @@ -379,12 +276,8 @@ void ExprContextList::add(bool _value) */ void ExprContextList::add(void) { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context list (none)")); - PyList_Append(jexlContext, Py_None); + jh.addNull(); } @@ -395,13 +288,9 @@ void ExprContextList::add(void) */ void ExprContextList::add(ExprContextObject exprContextObject) { - if (jexlContext == NULL) - { - return; - } - // FIXME P3: improve this implemeting a toString() method for ExprContextObject - LM_T(LmtExpr, ("adding to JEXL expression context list (object): [object]")); - PyList_Append(jexlContext, exprContextObject.getJexlContext()); + std::string s = exprContextObject.getJexlContext(); + LM_T(LmtExpr, ("adding to JEXL expression context list (object): %s", s.c_str())); + jh.addRaw(s); } @@ -412,13 +301,10 @@ void ExprContextList::add(ExprContextObject exprContextObject) */ void ExprContextList::add(ExprContextList exprContextList) { - if (jexlContext == NULL) - { - return; - } - // FIXME P3: improve this implemeting a toString() method for ExprContextList - LM_T(LmtExpr, ("adding to JEXL expression context list (list): [list]")); - PyList_Append(jexlContext, exprContextList.get()); + + std::string s = exprContextList.get(); + LM_T(LmtExpr, ("adding to JEXL expression context list (list): %s", s.c_str())); + jh.addRaw(s); } @@ -429,10 +315,5 @@ void ExprContextList::add(ExprContextList exprContextList) */ void ExprContextList::release(void) { - if (jexlContext == NULL) - { - return; - } - // FIXME PR: this is not correct. Recursively release of the list object - //Py_XDECREF(jexlContext); + // FIXME PR: this method is probably no longer needed } \ No newline at end of file diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index a3068b3138..41a9acd980 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -26,10 +26,11 @@ * Author: Fermin Galan */ -#include #include #include +#include "common/JsonHelper.h" + class ExprContextList; // forward declaration /* **************************************************************************** @@ -40,13 +41,13 @@ class ExprContextObject { private: bool legacy; - PyObject* jexlContext; // used in regular (i.e. not legacy) mode - std::map repl; // used in legacy mode + JsonObjectHelper jh; // used in regular (i.e. not legacy) mode + std::map repl; // used in legacy mode public: ExprContextObject(bool legacy = false); - PyObject* getJexlContext(void); + std::string getJexlContext(void); std::map* getMap(void); void add(const std::string& key, const std::string& value, bool raw = false); @@ -64,20 +65,20 @@ class ExprContextObject class ExprContextList { private: - PyObject* jexlContext; + JsonVectorHelper jh; public: ExprContextList(); - PyObject* get(void); - void add(const std::string& value); - void add(double value); - void add(bool value); - void add(void); - void add(ExprContextObject exprContextObject); - void add(ExprContextList exprContextList); + std::string get(void); + void add(const std::string& value); + void add(double value); + void add(bool value); + void add(void); + void add(ExprContextObject exprContextObject); + void add(ExprContextList exprContextList); - void release(void); + void release(void); }; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index d87f37f536..a9e16fa34b 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -23,15 +23,24 @@ * Author: Fermin Galan */ -#include - #include "expressions/ExprManager.h" #include "expressions/ExprResult.h" -#include "expressions/exprCommon.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" +// Interface to use libcjexl.a +extern "C" { + void* new_engine(); +} + +extern "C" { + void free_engine(void* ptr); +} + +extern "C" { + const char* eval(void* ptr, const char* script_ptr, const char* context_ptr); +} /* **************************************************************************** @@ -40,30 +49,13 @@ */ void ExprManager::init(void) { - tcjexlModule = NULL; - jexlEngine = NULL; + // FIXME PR: this is probably not needed + //if (sem_init(&sem, 0, 1) == -1) + //{ + // LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); + //} - if (sem_init(&sem, 0, 1) == -1) - { - LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); - } - - Py_Initialize(); - LM_T(LmtExpr, ("Python interpreter has been initialized")); - - tcjexlModule = PyImport_ImportModule("tcjexl"); - if (tcjexlModule == NULL) - { - LM_X(1, ("Fatal Error (error importing tcjexl module: %s)", capturePythonError())); - } - LM_T(LmtExpr, ("tcjexl module has been loaded")); - - jexlEngine = PyObject_CallMethod(tcjexlModule, "JEXL", NULL); - if (jexlEngine == NULL) - { - LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", capturePythonError())); - } - LM_T(LmtExpr, ("jexl engine has been created")); + jexlEngine = new_engine(); } @@ -95,26 +87,8 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st { // JEXL based evaluation LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - - PyObject* expression = Py_BuildValue("s", _expression.c_str()); - if (expression == NULL) - { - LM_W(("error building JEXL expression: %s", capturePythonError())); - return r; - } - - PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, exprContextObjectP->getJexlContext()); - Py_XDECREF(expression); - if (result == NULL) - { - LM_W(("error evaluating JEXL expression: %s", capturePythonError())); - return r; - } - + const char* result = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); r.fill(result); - - // FIXME PR: does this Py_XDECREF() recursively in the case of dicts or lists? - Py_XDECREF(result); } return r; @@ -128,18 +102,5 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st */ void ExprManager::release(void) { - if (jexlEngine != NULL) - { - Py_XDECREF(jexlEngine); - LM_T(LmtExpr, ("jexl engine has been freed")); - } - - if (tcjexlModule != NULL) - { - Py_XDECREF(tcjexlModule); - LM_T(LmtExpr, ("tcjexl module has been freed")); - } - - Py_Finalize(); - LM_T(LmtExpr, ("Python interpreter has been finalized")); + free_engine(jexlEngine); } \ No newline at end of file diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 55c8e90f96..821f7e9ad9 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -26,8 +26,8 @@ * Author: Fermin Galan */ -#include -#include +// FIXME PR: probably this is no longer needed +//#include #include "expressions/ExprContext.h" #include "expressions/ExprResult.h" @@ -39,10 +39,9 @@ class ExprManager { private: - PyObject* tcjexlModule; - PyObject* jexlEngine; - PyObject* jsonModule; - sem_t sem; + // FIXME PR: probably this is no longer needed + //sem_t sem; + void* jexlEngine; public: void init(void); diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index a57cbc9ab4..8c3dd8d86d 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -24,7 +24,6 @@ */ #include "expressions/ExprResult.h" -#include "expressions/exprCommon.h" #include "common/string.h" #include "common/JsonHelper.h" @@ -256,7 +255,7 @@ void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* * fill - * */ -void ExprResult::fill(PyObject* result) +void ExprResult::fill(std::string result) { // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index df1ec0f597..12dd928497 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -30,7 +30,6 @@ #include "parse/CompoundValueNode.h" -#include #include /* **************************************************************************** @@ -50,7 +49,7 @@ class ExprResult // Use only when valueType is object or vector orion::CompoundValueNode* compoundValueP; - void fill(PyObject* result); + void fill(std::string result); std::string toString(void); void release(void); diff --git a/src/lib/expressions/exprCommon.cpp b/src/lib/expressions/exprCommon.cpp deleted file mode 100644 index 1372d73eaa..0000000000 --- a/src/lib/expressions/exprCommon.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -#include "expressions/exprCommon.h" - -/* **************************************************************************** -* -* capturePythonError - -*/ -const char* capturePythonError() -{ - if (PyErr_Occurred()) - { - PyObject* ptype; - PyObject* pvalue; - PyObject* ptraceback; - - // Fetch the exception type, value, and traceback - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - - if (pvalue != NULL) - { - PyObject* strObj = PyObject_Str(pvalue); - const char* errorMessage = PyUnicode_AsUTF8(strObj); - - // Release the Python objects - Py_XDECREF(strObj); - Py_XDECREF(ptype); - Py_XDECREF(pvalue); - Py_XDECREF(ptraceback); - - return errorMessage; - } - } - - PyErr_Clear(); - return ""; -} \ No newline at end of file diff --git a/src/lib/expressions/exprCommon.h b/src/lib/expressions/exprCommon.h deleted file mode 100644 index 7b9667d1b4..0000000000 --- a/src/lib/expressions/exprCommon.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ -#define SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ - -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -#include - - - -/* **************************************************************************** -* -* capturePythonError - -*/ -extern const char* capturePythonError(); - - - -#endif // SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 6ee599bd5f..0c6fe587ff 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -321,6 +321,7 @@ static SenderThreadParams* buildSenderParamsCustom bool legacy = (exprLang == "legacy"); ExprContextObject exprContext(legacy); + // FIXME PR: no-legacy context is now based in JsonHelper, which may mess with key repetition. Check this // It seems that add() semantics are different in legacy and jexl mode. In jexl mode, if the key already exists, it is // updated. In legacy model, if the key already exists, the operation is ignored (so previous value is preserved). Taking // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index 5ea06218d9..30a42ff25a 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -46,7 +46,6 @@ #include #include #include -#include /* **************************************************************************** * @@ -71,7 +70,7 @@ std::string libVersions(void) std::string mhd = " \"libmicrohttpd\": "; std::string ssl = " \"openssl\": "; std::string rjson = " \"rapidjson\": "; - std::string python = " \"libpython\": "; + std::string cjexl = " \"libcjexl\": "; std::string mongo = " \"mongoc\": "; std::string bson = " \"bson\": "; @@ -86,17 +85,12 @@ std::string libVersions(void) char mosqVersion[16]; snprintf(mosqVersion, sizeof(mosqVersion), "%d.%d.%d", mosqMayor, mosqMinor, mosqRevision); - char pyVersion[16]; - snprintf(pyVersion, sizeof(pyVersion), "%d.%d.%d", - (PY_VERSION_HEX >> 24) & 0xFF, - (PY_VERSION_HEX >> 16) & 0xFF, - (PY_VERSION_HEX >> 8) & 0xFF); - total += boost + "\"" + BOOST_LIB_VERSION "\"" + ",\n"; total += curl + "\"" + curlVersion + "\"" + ",\n"; total += mosq + "\"" + mosqVersion + "\"" + ",\n"; total += mhd + "\"" + MHD_get_version() + "\"" + ",\n"; - total += python + "\"" + pyVersion + "\"" + ",\n"; + // FIXME PR + //total += cjexl + "\"" + cjexlVersion + "\"" + ",\n"; #ifdef OLD_SSL_VERSION_FORMAT // Needed by openssl 1.1.1n in Debian 11 and before total += ssl + "\"" + SHLIB_VERSION_NUMBER "\"" + ",\n"; diff --git a/test/unittests/CMakeLists.txt b/test/unittests/CMakeLists.txt index a929c69b39..af8fbaf709 100644 --- a/test/unittests/CMakeLists.txt +++ b/test/unittests/CMakeLists.txt @@ -217,7 +217,6 @@ include_directories("${PROJECT_SOURCE_DIR}/test") # Needed for the new C driver # FIXME: why do we need this declared here if they are also declared # in the main CMakeFiles.txt? -include_directories("/usr/include/python3.11/") include_directories("/usr/local/include/libmongoc-1.0") include_directories("/usr/local/include/libbson-1.0") From 6504fe66905cb497ae7d8660d6b12af2a125a787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Apr 2024 13:42:13 +0200 Subject: [PATCH 047/104] FIX solution without ExprResult class (have problemas with rendering decimals) --- src/lib/common/macroSubstitute.cpp | 20 +- src/lib/expressions/CMakeLists.txt | 2 - src/lib/expressions/ExprManager.cpp | 13 +- src/lib/expressions/ExprManager.h | 3 +- src/lib/expressions/ExprResult.cpp | 383 ---------------------------- src/lib/expressions/ExprResult.h | 60 ----- 6 files changed, 15 insertions(+), 466 deletions(-) delete mode 100644 src/lib/expressions/ExprResult.cpp delete mode 100644 src/lib/expressions/ExprResult.h diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 23b631a710..d9ba2bc54a 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -53,8 +53,8 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - ExprResult r = exprMgr.evaluate(exprContextObjectP, macroName); - if (r.valueType == orion::ValueTypeNull) + std::string r = exprMgr.evaluate(exprContextObjectP, macroName); + if (r == "null") { return notFoundDefault; } @@ -63,11 +63,11 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - return removeQuotes(r.toString()); + return removeQuotes(r); } else { - return r.toString(); + return r; } } } @@ -100,31 +100,31 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e */ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, const std::string key, const std::string& notFoundDefault, bool raw) { - ExprResult r = exprMgr.evaluate(exprContextObjectP, key); + std::string r = exprMgr.evaluate(exprContextObjectP, key); - if (r.valueType == orion::ValueTypeNull) + if (r == "null") { return notFoundDefault; } else { - std::string s = r.toString(); + std::string r; // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - s = removeQuotes(s); + r = removeQuotes(r); } if (raw) { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), // so double quotes have to be be removed - return removeQuotes(s); + return removeQuotes(r); } else { - return s; + return r; } } } diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index 4f938e3634..f38c3d8d6e 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -23,14 +23,12 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES ExprManager.cpp ExprContext.cpp - ExprResult.cpp exprMgr.cpp ) SET (HEADERS ExprManager.h ExprContext.h - ExprResult.h exprMgr.h ) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index a9e16fa34b..2d5dffa29e 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -24,7 +24,6 @@ */ #include "expressions/ExprManager.h" -#include "expressions/ExprResult.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" @@ -64,11 +63,9 @@ void ExprManager::init(void) * * ExprManager::evaluate - */ -ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) +std::string ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) { - ExprResult r; - r.valueType = orion::ValueTypeNull; - + std::string r; if (exprContextObjectP->isLegacy()) { // std::map based evaluation. Only pure replacement is supported @@ -79,16 +76,14 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st std::map::iterator iter = replacementsP->find(_expression); if (iter != replacementsP->end()) { - r.valueType = orion::ValueTypeString; - r.stringValue = iter->second; + r = iter->second; } } else { // JEXL based evaluation LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - const char* result = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); - r.fill(result); + r = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); } return r; diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 821f7e9ad9..c73a4ebc40 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -30,7 +30,6 @@ //#include #include "expressions/ExprContext.h" -#include "expressions/ExprResult.h" /* **************************************************************************** * @@ -45,7 +44,7 @@ class ExprManager public: void init(void); - ExprResult evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); + std::string evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); void release(void); }; diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp deleted file mode 100644 index 8c3dd8d86d..0000000000 --- a/src/lib/expressions/ExprResult.cpp +++ /dev/null @@ -1,383 +0,0 @@ -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -#include "expressions/ExprResult.h" - -#include "common/string.h" -#include "common/JsonHelper.h" -#include "logMsg/logMsg.h" - - - -/* **************************************************************************** -* -* getPyObjectType - -*/ -static orion::ValueType getPyObjectType(PyObject* obj) -{ - // PyBool_Check() has to be done before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be - // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 - if (PyBool_Check(obj)) - { - return orion::ValueTypeBoolean; - } - else if (PyLong_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyFloat_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyDict_Check(obj)) - { - return orion::ValueTypeObject; - } - else if (PyList_Check(obj)) - { - return orion::ValueTypeVector; - } - else if (obj == Py_None) - { - return orion::ValueTypeNull; - } - else - { - // For other types we use string (this is also a failsafe for types not being strings) - return orion::ValueTypeString; - } -} - - -static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); // forward declaration - - -/* **************************************************************************** -* -* processListItem - -* -*/ -void processListItem(orion::CompoundValueNode* parentP, PyObject* value) -{ - orion::CompoundValueNode* nodeP; - - const char* str; - double d; - bool b; - PyObject *keyAux, *valueAux; - Py_ssize_t pos, size; - - switch (getPyObjectType(value)) - { - case orion::ValueTypeString: - str = PyUnicode_AsUTF8(value); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - } - else - { - LM_T(LmtExpr, ("processListITem (string): %s", str)); - nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); - parentP->add(nodeP); - } - break; - - case orion::ValueTypeNumber: - d = PyFloat_AsDouble(value); - LM_T(LmtExpr, ("processList (double): %f", d)); - nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); - parentP->add(nodeP); - break; - - case orion::ValueTypeBoolean: - b = PyObject_IsTrue(value); - LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); - nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); - parentP->add(nodeP); - break; - - case orion::ValueTypeNull: - nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeNull); - parentP->add(nodeP); - break; - - case orion::ValueTypeVector: - nodeP = new orion::CompoundValueNode(orion::ValueTypeVector); - size = PyList_Size(value); - for (Py_ssize_t ix = 0; ix < size; ++ix) - { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(nodeP, PyList_GetItem(value, ix)); - } - parentP->add(nodeP); - break; - - case orion::ValueTypeObject: - nodeP = new orion::CompoundValueNode(orion::ValueTypeObject); - pos = 0; - while (PyDict_Next(value, &pos, &keyAux, &valueAux)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(nodeP, keyAux, valueAux); - } - parentP->add(nodeP); - break; - - case orion::ValueTypeNotGiven: - LM_E(("Runtime Error (value type not given))")); - break; - - default: - LM_E(("Runtime Error (value type unknown))")); - } -} - - - -/* **************************************************************************** -* -* processDictItem - -* -*/ -void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) -{ - const char * keyStr = PyUnicode_AsUTF8(key); - if (keyStr == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - return; - } - - orion::CompoundValueNode* nodeP; - const char* str; - double d; - bool b; - PyObject *keyAux, *valueAux; - Py_ssize_t pos, size; - - switch (getPyObjectType(value)) - { - case orion::ValueTypeString: - str = PyUnicode_AsUTF8(value); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - } - else - { - LM_T(LmtExpr, ("processListITem (string): %s", str)); - nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); - parentP->add(nodeP); - } - break; - - case orion::ValueTypeNumber: - d = PyFloat_AsDouble(value); - LM_T(LmtExpr, ("processList (double): %f", d)); - nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); - parentP->add(nodeP); - break; - - case orion::ValueTypeBoolean: - b = PyObject_IsTrue(value); - LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); - nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); - parentP->add(nodeP); - break; - - case orion::ValueTypeNull: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeNull); - parentP->add(nodeP); - break; - - case orion::ValueTypeVector: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeVector); - size = PyList_Size(value); - for (Py_ssize_t ix = 0; ix < size; ++ix) - { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(nodeP, PyList_GetItem(value, ix)); - } - parentP->add(nodeP); - break; - - case orion::ValueTypeObject: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeObject); - pos = 0; - while (PyDict_Next(value, &pos, &keyAux, &valueAux)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(nodeP, keyAux, valueAux); - } - parentP->add(nodeP); - break; - - case orion::ValueTypeNotGiven: - LM_E(("Runtime Error (value type not given))")); - break; - - default: - LM_E(("Runtime Error (value type unknown))")); - } -} - - - -/* **************************************************************************** -* -* fill - -* -*/ -void ExprResult::fill(std::string result) -{ - // If nothing changes, the returned value would be null (failsafe) - valueType = orion::ValueTypeNull; - - // Special case: expresion evalutes to None - if (result == Py_None) - { - LM_T(LmtExpr, ("ExprResult is null")); - valueType = orion::ValueTypeNull; - return; - } - - // Other not null types - valueType = getPyObjectType(result); - if (valueType == orion::ValueTypeNumber) - { - numberValue = PyFloat_AsDouble(result); - LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); - } - else if (valueType == orion::ValueTypeBoolean) - { - boolValue = PyObject_IsTrue(result); - LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); - } - else if (valueType == orion::ValueTypeObject) - { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(result, &pos, &key, &value)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(compoundValueP, key, value); - } - } - else if (valueType == orion::ValueTypeVector) - { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); - Py_ssize_t size = PyList_Size(result); - for (Py_ssize_t ix = 0; ix < size; ++ix) - { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(compoundValueP, PyList_GetItem(result, ix)); - } - } - else if (valueType == orion::ValueTypeString) - { - const char* str = PyUnicode_AsUTF8(result); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtExpr, ("ExprResult (string): %s", str)); - stringValue = std::string(str); - } - } -} - - - -/* **************************************************************************** -* -* toString - -* -* Pretty similar to ContextAttribute::toJsonValue() -* -*/ -std::string ExprResult::toString(void) -{ - if (valueType == orion::ValueTypeNumber) - { - return double2string(numberValue); - } - else if (valueType == orion::ValueTypeBoolean) - { - return boolValue ? "true" : "false"; - } - else if (valueType == orion::ValueTypeString) - { - // FIXME PR: does this break the no legacy - //return "\"" + toJsonString(stringValue) + "\""; - return "\"" + stringValue + "\""; - } - else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) - { - if (compoundValueP != NULL) - { - return compoundValueP->toJson(); - } - else - { - LM_E(("Runtime Error (result is vector/object but compountValue is NULL)")); - return ""; - } - } - else if (valueType == orion::ValueTypeNull) - { - return "null"; - } - else - { - LM_E(("Runtime Error (not allowed type in ExprResult: %s)", valueTypeName(valueType))); - return ""; - } -} - - - -/* **************************************************************************** -* -* ExprResult::release - -*/ -void ExprResult::release(void) -{ - if (compoundValueP != NULL) - { - delete compoundValueP; - compoundValueP = NULL; - } -} \ No newline at end of file diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h deleted file mode 100644 index 12dd928497..0000000000 --- a/src/lib/expressions/ExprResult.h +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ -#define SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ - -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -#include "orionTypes/OrionValueType.h" - -#include "parse/CompoundValueNode.h" - -#include - -/* **************************************************************************** -* -* ExprResult - -*/ -class ExprResult -{ -public: - // Similar to the fields used in ContextAttribute.h - - orion::ValueType valueType; // Type of value - std::string stringValue; // "value" as a String - double numberValue; // "value" as a Number - bool boolValue; // "value" as a Boolean - - // Use only when valueType is object or vector - orion::CompoundValueNode* compoundValueP; - - void fill(std::string result); - - std::string toString(void); - void release(void); -}; - - - -#endif // SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ From ed875f52937c1635c163d27129124c85db23e233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Apr 2024 13:42:58 +0200 Subject: [PATCH 048/104] Revert "FIX solution without ExprResult class (have problemas with rendering decimals)" This reverts commit 6ec9f58d85f71d2c96aa01e1bc4d5cceb5e97c37. --- src/lib/common/macroSubstitute.cpp | 20 +- src/lib/expressions/CMakeLists.txt | 2 + src/lib/expressions/ExprManager.cpp | 13 +- src/lib/expressions/ExprManager.h | 3 +- src/lib/expressions/ExprResult.cpp | 383 ++++++++++++++++++++++++++++ src/lib/expressions/ExprResult.h | 60 +++++ 6 files changed, 466 insertions(+), 15 deletions(-) create mode 100644 src/lib/expressions/ExprResult.cpp create mode 100644 src/lib/expressions/ExprResult.h diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index d9ba2bc54a..23b631a710 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -53,8 +53,8 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - std::string r = exprMgr.evaluate(exprContextObjectP, macroName); - if (r == "null") + ExprResult r = exprMgr.evaluate(exprContextObjectP, macroName); + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } @@ -63,11 +63,11 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - return removeQuotes(r); + return removeQuotes(r.toString()); } else { - return r; + return r.toString(); } } } @@ -100,31 +100,31 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e */ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, const std::string key, const std::string& notFoundDefault, bool raw) { - std::string r = exprMgr.evaluate(exprContextObjectP, key); + ExprResult r = exprMgr.evaluate(exprContextObjectP, key); - if (r == "null") + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } else { - std::string r; + std::string s = r.toString(); // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - r = removeQuotes(r); + s = removeQuotes(s); } if (raw) { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), // so double quotes have to be be removed - return removeQuotes(r); + return removeQuotes(s); } else { - return r; + return s; } } } diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index f38c3d8d6e..4f938e3634 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -23,12 +23,14 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES ExprManager.cpp ExprContext.cpp + ExprResult.cpp exprMgr.cpp ) SET (HEADERS ExprManager.h ExprContext.h + ExprResult.h exprMgr.h ) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 2d5dffa29e..a9e16fa34b 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -24,6 +24,7 @@ */ #include "expressions/ExprManager.h" +#include "expressions/ExprResult.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" @@ -63,9 +64,11 @@ void ExprManager::init(void) * * ExprManager::evaluate - */ -std::string ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) +ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) { - std::string r; + ExprResult r; + r.valueType = orion::ValueTypeNull; + if (exprContextObjectP->isLegacy()) { // std::map based evaluation. Only pure replacement is supported @@ -76,14 +79,16 @@ std::string ExprManager::evaluate(ExprContextObject* exprContextObjectP, const s std::map::iterator iter = replacementsP->find(_expression); if (iter != replacementsP->end()) { - r = iter->second; + r.valueType = orion::ValueTypeString; + r.stringValue = iter->second; } } else { // JEXL based evaluation LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - r = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); + const char* result = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); + r.fill(result); } return r; diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index c73a4ebc40..821f7e9ad9 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -30,6 +30,7 @@ //#include #include "expressions/ExprContext.h" +#include "expressions/ExprResult.h" /* **************************************************************************** * @@ -44,7 +45,7 @@ class ExprManager public: void init(void); - std::string evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); + ExprResult evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); void release(void); }; diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp new file mode 100644 index 0000000000..8c3dd8d86d --- /dev/null +++ b/src/lib/expressions/ExprResult.cpp @@ -0,0 +1,383 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "expressions/ExprResult.h" + +#include "common/string.h" +#include "common/JsonHelper.h" +#include "logMsg/logMsg.h" + + + +/* **************************************************************************** +* +* getPyObjectType - +*/ +static orion::ValueType getPyObjectType(PyObject* obj) +{ + // PyBool_Check() has to be done before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be + // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 + if (PyBool_Check(obj)) + { + return orion::ValueTypeBoolean; + } + else if (PyLong_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyFloat_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyDict_Check(obj)) + { + return orion::ValueTypeObject; + } + else if (PyList_Check(obj)) + { + return orion::ValueTypeVector; + } + else if (obj == Py_None) + { + return orion::ValueTypeNull; + } + else + { + // For other types we use string (this is also a failsafe for types not being strings) + return orion::ValueTypeString; + } +} + + +static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); // forward declaration + + +/* **************************************************************************** +* +* processListItem - +* +*/ +void processListItem(orion::CompoundValueNode* parentP, PyObject* value) +{ + orion::CompoundValueNode* nodeP; + + const char* str; + double d; + bool b; + PyObject *keyAux, *valueAux; + Py_ssize_t pos, size; + + switch (getPyObjectType(value)) + { + case orion::ValueTypeString: + str = PyUnicode_AsUTF8(value); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + } + else + { + LM_T(LmtExpr, ("processListITem (string): %s", str)); + nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); + parentP->add(nodeP); + } + break; + + case orion::ValueTypeNumber: + d = PyFloat_AsDouble(value); + LM_T(LmtExpr, ("processList (double): %f", d)); + nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); + parentP->add(nodeP); + break; + + case orion::ValueTypeBoolean: + b = PyObject_IsTrue(value); + LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); + nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); + parentP->add(nodeP); + break; + + case orion::ValueTypeNull: + nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeNull); + parentP->add(nodeP); + break; + + case orion::ValueTypeVector: + nodeP = new orion::CompoundValueNode(orion::ValueTypeVector); + size = PyList_Size(value); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(nodeP, PyList_GetItem(value, ix)); + } + parentP->add(nodeP); + break; + + case orion::ValueTypeObject: + nodeP = new orion::CompoundValueNode(orion::ValueTypeObject); + pos = 0; + while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() + processDictItem(nodeP, keyAux, valueAux); + } + parentP->add(nodeP); + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given))")); + break; + + default: + LM_E(("Runtime Error (value type unknown))")); + } +} + + + +/* **************************************************************************** +* +* processDictItem - +* +*/ +void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +{ + const char * keyStr = PyUnicode_AsUTF8(key); + if (keyStr == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + return; + } + + orion::CompoundValueNode* nodeP; + const char* str; + double d; + bool b; + PyObject *keyAux, *valueAux; + Py_ssize_t pos, size; + + switch (getPyObjectType(value)) + { + case orion::ValueTypeString: + str = PyUnicode_AsUTF8(value); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + } + else + { + LM_T(LmtExpr, ("processListITem (string): %s", str)); + nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); + parentP->add(nodeP); + } + break; + + case orion::ValueTypeNumber: + d = PyFloat_AsDouble(value); + LM_T(LmtExpr, ("processList (double): %f", d)); + nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); + parentP->add(nodeP); + break; + + case orion::ValueTypeBoolean: + b = PyObject_IsTrue(value); + LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); + nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); + parentP->add(nodeP); + break; + + case orion::ValueTypeNull: + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeNull); + parentP->add(nodeP); + break; + + case orion::ValueTypeVector: + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeVector); + size = PyList_Size(value); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(nodeP, PyList_GetItem(value, ix)); + } + parentP->add(nodeP); + break; + + case orion::ValueTypeObject: + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeObject); + pos = 0; + while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() + processDictItem(nodeP, keyAux, valueAux); + } + parentP->add(nodeP); + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given))")); + break; + + default: + LM_E(("Runtime Error (value type unknown))")); + } +} + + + +/* **************************************************************************** +* +* fill - +* +*/ +void ExprResult::fill(std::string result) +{ + // If nothing changes, the returned value would be null (failsafe) + valueType = orion::ValueTypeNull; + + // Special case: expresion evalutes to None + if (result == Py_None) + { + LM_T(LmtExpr, ("ExprResult is null")); + valueType = orion::ValueTypeNull; + return; + } + + // Other not null types + valueType = getPyObjectType(result); + if (valueType == orion::ValueTypeNumber) + { + numberValue = PyFloat_AsDouble(result); + LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); + } + else if (valueType == orion::ValueTypeBoolean) + { + boolValue = PyObject_IsTrue(result); + LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); + } + else if (valueType == orion::ValueTypeObject) + { + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(result, &pos, &key, &value)) + { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() + processDictItem(compoundValueP, key, value); + } + } + else if (valueType == orion::ValueTypeVector) + { + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); + Py_ssize_t size = PyList_Size(result); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(compoundValueP, PyList_GetItem(result, ix)); + } + } + else if (valueType == orion::ValueTypeString) + { + const char* str = PyUnicode_AsUTF8(result); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtExpr, ("ExprResult (string): %s", str)); + stringValue = std::string(str); + } + } +} + + + +/* **************************************************************************** +* +* toString - +* +* Pretty similar to ContextAttribute::toJsonValue() +* +*/ +std::string ExprResult::toString(void) +{ + if (valueType == orion::ValueTypeNumber) + { + return double2string(numberValue); + } + else if (valueType == orion::ValueTypeBoolean) + { + return boolValue ? "true" : "false"; + } + else if (valueType == orion::ValueTypeString) + { + // FIXME PR: does this break the no legacy + //return "\"" + toJsonString(stringValue) + "\""; + return "\"" + stringValue + "\""; + } + else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) + { + if (compoundValueP != NULL) + { + return compoundValueP->toJson(); + } + else + { + LM_E(("Runtime Error (result is vector/object but compountValue is NULL)")); + return ""; + } + } + else if (valueType == orion::ValueTypeNull) + { + return "null"; + } + else + { + LM_E(("Runtime Error (not allowed type in ExprResult: %s)", valueTypeName(valueType))); + return ""; + } +} + + + +/* **************************************************************************** +* +* ExprResult::release - +*/ +void ExprResult::release(void) +{ + if (compoundValueP != NULL) + { + delete compoundValueP; + compoundValueP = NULL; + } +} \ No newline at end of file diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h new file mode 100644 index 0000000000..12dd928497 --- /dev/null +++ b/src/lib/expressions/ExprResult.h @@ -0,0 +1,60 @@ +#ifndef SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ +#define SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "orionTypes/OrionValueType.h" + +#include "parse/CompoundValueNode.h" + +#include + +/* **************************************************************************** +* +* ExprResult - +*/ +class ExprResult +{ +public: + // Similar to the fields used in ContextAttribute.h + + orion::ValueType valueType; // Type of value + std::string stringValue; // "value" as a String + double numberValue; // "value" as a Number + bool boolValue; // "value" as a Boolean + + // Use only when valueType is object or vector + orion::CompoundValueNode* compoundValueP; + + void fill(std::string result); + + std::string toString(void); + void release(void); +}; + + + +#endif // SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ From 9cc132a643efe567f3db7a096e89b14f362d292e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Apr 2024 16:54:21 +0200 Subject: [PATCH 049/104] FIX new cjexl function names and cjexl_version() usage --- src/lib/expressions/ExprManager.cpp | 12 ++++++------ src/lib/expressions/ExprResult.cpp | 11 +++++++++-- src/lib/serviceRoutines/versionTreat.cpp | 8 ++++++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index a9e16fa34b..195e3c480a 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -31,15 +31,15 @@ // Interface to use libcjexl.a extern "C" { - void* new_engine(); + void* cjexl_new_engine(); } extern "C" { - void free_engine(void* ptr); + void cjexl_free_engine(void* ptr); } extern "C" { - const char* eval(void* ptr, const char* script_ptr, const char* context_ptr); + const char* cjexl_eval(void* ptr, const char* script_ptr, const char* context_ptr); } @@ -55,7 +55,7 @@ void ExprManager::init(void) // LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); //} - jexlEngine = new_engine(); + jexlEngine = cjexl_new_engine(); } @@ -87,7 +87,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st { // JEXL based evaluation LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - const char* result = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); + const char* result = cjexl_eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); r.fill(result); } @@ -102,5 +102,5 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st */ void ExprManager::release(void) { - free_engine(jexlEngine); + cjexl_free_engine(jexlEngine); } \ No newline at end of file diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 8c3dd8d86d..92e40fb3f3 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -23,6 +23,11 @@ * Author: Fermin Galan */ +// The ExprResult class is used as return value in ExprManager::evaluate(). We could return std::string +// in that function and simplify (so avoiding the ExprResult class). But in that case float rounding is +// problematic (e.g. 1.999999 instead of 2), as they don't take advanage of the ad hoc logic implemented +// in ContextAttribute rendering + #include "expressions/ExprResult.h" #include "common/string.h" @@ -31,6 +36,7 @@ +#if 0 /* **************************************************************************** * * getPyObjectType - @@ -247,7 +253,7 @@ void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* LM_E(("Runtime Error (value type unknown))")); } } - +#endif /* **************************************************************************** @@ -257,6 +263,7 @@ void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* */ void ExprResult::fill(std::string result) { + /* // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; @@ -316,7 +323,7 @@ void ExprResult::fill(std::string result) LM_T(LmtExpr, ("ExprResult (string): %s", str)); stringValue = std::string(str); } - } + }*/ } diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index 30a42ff25a..eec99efedb 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -47,6 +47,11 @@ #include #include +// Interface to use libcjexl.a +extern "C" { + const char* cjexl_version(); +} + /* **************************************************************************** * * version - @@ -89,8 +94,7 @@ std::string libVersions(void) total += curl + "\"" + curlVersion + "\"" + ",\n"; total += mosq + "\"" + mosqVersion + "\"" + ",\n"; total += mhd + "\"" + MHD_get_version() + "\"" + ",\n"; - // FIXME PR - //total += cjexl + "\"" + cjexlVersion + "\"" + ",\n"; + total += cjexl + "\"" + cjexl_version() + "\"" + ",\n"; #ifdef OLD_SSL_VERSION_FORMAT // Needed by openssl 1.1.1n in Debian 11 and before total += ssl + "\"" + SHLIB_VERSION_NUMBER "\"" + ",\n"; From 8097a8d2dbb33a8dfb6c13a7d6905a6031f0a901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Apr 2024 18:05:46 +0200 Subject: [PATCH 050/104] FIX ExprResult implementation --- src/lib/common/JsonHelper.cpp | 18 +- src/lib/common/JsonHelper.h | 2 + src/lib/expressions/ExprManager.cpp | 9 +- src/lib/expressions/ExprResult.cpp | 332 +++++++----------- .../jexl_array_filtering.test | 29 +- 5 files changed, 165 insertions(+), 225 deletions(-) diff --git a/src/lib/common/JsonHelper.cpp b/src/lib/common/JsonHelper.cpp index 3897f011cf..3bd6fd5d2a 100644 --- a/src/lib/common/JsonHelper.cpp +++ b/src/lib/common/JsonHelper.cpp @@ -138,7 +138,7 @@ std::string objectToJson(std::map& list) * * JsonObjectHelper - */ -JsonObjectHelper::JsonObjectHelper(): empty(true) +JsonObjectHelper::JsonObjectHelper(): empty(true), closed(false) { ss += '{'; } @@ -282,7 +282,12 @@ void JsonObjectHelper::addNull(const std::string& key) */ std::string JsonObjectHelper::str() { - ss += '}'; + // This check allows to call str() several times (needed when this is used in ExprContext) + if (!closed) + { + ss += '}'; + closed = true; + } return ss; } @@ -292,7 +297,7 @@ std::string JsonObjectHelper::str() * * JsonVectorHelper - */ -JsonVectorHelper::JsonVectorHelper(): empty(true) +JsonVectorHelper::JsonVectorHelper(): empty(true), closed(false) { ss += '['; } @@ -423,6 +428,11 @@ void JsonVectorHelper::addNull(void) */ std::string JsonVectorHelper::str() { - ss += ']'; + // This check allows to call str() several times (needed when this is used in ExprContext) + if (!closed) + { + ss += ']'; + closed = true; + } return ss; } diff --git a/src/lib/common/JsonHelper.h b/src/lib/common/JsonHelper.h index 47a52c0ff0..bcb8aaeaf2 100644 --- a/src/lib/common/JsonHelper.h +++ b/src/lib/common/JsonHelper.h @@ -50,6 +50,7 @@ class JsonObjectHelper private: std::string ss; bool empty; + bool closed; }; @@ -72,6 +73,7 @@ class JsonVectorHelper private: std::string ss; bool empty; + bool closed; }; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 195e3c480a..86e24cbae0 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -72,7 +72,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st if (exprContextObjectP->isLegacy()) { // std::map based evaluation. Only pure replacement is supported - LM_T(LmtExpr, ("evaluating legacy expresion: <%s>", _expression.c_str())); + LM_T(LmtExpr, ("evaluating legacy expression: <%s>", _expression.c_str())); std::map* replacementsP = exprContextObjectP->getMap(); @@ -81,13 +81,16 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st { r.valueType = orion::ValueTypeString; r.stringValue = iter->second; + LM_T(LmtExpr, ("legacy evaluation result: <%s>", r.stringValue.c_str())); } } else { // JEXL based evaluation - LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - const char* result = cjexl_eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); + std::string context = exprContextObjectP->getJexlContext(); + LM_T(LmtExpr, ("evaluating JEXL expression <%s> with context <%s>", _expression.c_str(), context.c_str())); + const char* result = cjexl_eval(jexlEngine, _expression.c_str(), context.c_str()); + LM_T(LmtExpr, ("JEXL evaluation result: <%s>", result)); r.fill(result); } diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 92e40fb3f3..939c3ebfc8 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -28,56 +28,21 @@ // problematic (e.g. 1.999999 instead of 2), as they don't take advanage of the ad hoc logic implemented // in ContextAttribute rendering +#include "rapidjson/document.h" + #include "expressions/ExprResult.h" #include "common/string.h" #include "common/JsonHelper.h" #include "logMsg/logMsg.h" +#include "jsonParseV2/jsonParseTypeNames.h" +#include "jsonParseV2/utilsParse.h" -#if 0 -/* **************************************************************************** -* -* getPyObjectType - -*/ -static orion::ValueType getPyObjectType(PyObject* obj) -{ - // PyBool_Check() has to be done before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be - // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 - if (PyBool_Check(obj)) - { - return orion::ValueTypeBoolean; - } - else if (PyLong_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyFloat_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyDict_Check(obj)) - { - return orion::ValueTypeObject; - } - else if (PyList_Check(obj)) - { - return orion::ValueTypeVector; - } - else if (obj == Py_None) - { - return orion::ValueTypeNull; - } - else - { - // For other types we use string (this is also a failsafe for types not being strings) - return orion::ValueTypeString; - } -} +static void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstMemberIterator& iter); // forward declaration -static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); // forward declaration /* **************************************************************************** @@ -85,81 +50,58 @@ static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, Py * processListItem - * */ -void processListItem(orion::CompoundValueNode* parentP, PyObject* value) +void processListItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstValueIterator& iter) { orion::CompoundValueNode* nodeP; - const char* str; - double d; - bool b; - PyObject *keyAux, *valueAux; - Py_ssize_t pos, size; + std::string type = jsonParseTypeNames[iter->GetType()]; - switch (getPyObjectType(value)) + if (type == "String") { - case orion::ValueTypeString: - str = PyUnicode_AsUTF8(value); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - } - else - { - LM_T(LmtExpr, ("processListITem (string): %s", str)); - nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); - parentP->add(nodeP); - } - break; - - case orion::ValueTypeNumber: - d = PyFloat_AsDouble(value); - LM_T(LmtExpr, ("processList (double): %f", d)); - nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); + nodeP = new orion::CompoundValueNode("", iter->GetString(), orion::ValueTypeString); parentP->add(nodeP); - break; - - case orion::ValueTypeBoolean: - b = PyObject_IsTrue(value); - LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); - nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); + } + else if (type == "Number") + { + nodeP = new orion::CompoundValueNode("", iter->GetDouble(), orion::ValueTypeNumber); parentP->add(nodeP); - break; - - case orion::ValueTypeNull: + } + else if (type == "True") + { + nodeP = new orion::CompoundValueNode("", true, orion::ValueTypeBoolean); + parentP->add(nodeP); + } + else if (type == "False") + { + nodeP = new orion::CompoundValueNode("", false, orion::ValueTypeBoolean); + parentP->add(nodeP); + } + else if (type == "Null") + { nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeNull); parentP->add(nodeP); - break; - - case orion::ValueTypeVector: - nodeP = new orion::CompoundValueNode(orion::ValueTypeVector); - size = PyList_Size(value); - for (Py_ssize_t ix = 0; ix < size; ++ix) + } + else if (type == "Array") + { + nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeVector); + for (rapidjson::Value::ConstValueIterator iter2 = iter->Begin(); iter2 != iter->End(); ++iter2) { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(nodeP, PyList_GetItem(value, ix)); + processListItem(nodeP, iter2); } parentP->add(nodeP); - break; - - case orion::ValueTypeObject: - nodeP = new orion::CompoundValueNode(orion::ValueTypeObject); - pos = 0; - while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + } + else if (type == "Object") + { + nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeObject); + for (rapidjson::Value::ConstMemberIterator iter2 = iter->MemberBegin(); iter2 != iter->MemberEnd(); ++iter2) { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(nodeP, keyAux, valueAux); + processDictItem(nodeP, iter2); } parentP->add(nodeP); - break; - - case orion::ValueTypeNotGiven: - LM_E(("Runtime Error (value type not given))")); - break; - - default: - LM_E(("Runtime Error (value type unknown))")); + } + else + { + LM_E(("Runtime Error (unknown type: %s)", type.c_str())); } } @@ -170,90 +112,62 @@ void processListItem(orion::CompoundValueNode* parentP, PyObject* value) * processDictItem - * */ -void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstMemberIterator& iter) { - const char * keyStr = PyUnicode_AsUTF8(key); - if (keyStr == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - return; - } - orion::CompoundValueNode* nodeP; - const char* str; - double d; - bool b; - PyObject *keyAux, *valueAux; - Py_ssize_t pos, size; - switch (getPyObjectType(value)) - { - case orion::ValueTypeString: - str = PyUnicode_AsUTF8(value); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - } - else - { - LM_T(LmtExpr, ("processListITem (string): %s", str)); - nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); - parentP->add(nodeP); - } - break; + std::string name = iter->name.GetString(); + std::string type = jsonParseTypeNames[iter->value.GetType()]; - case orion::ValueTypeNumber: - d = PyFloat_AsDouble(value); - LM_T(LmtExpr, ("processList (double): %f", d)); - nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); + if (type == "String") + { + nodeP = new orion::CompoundValueNode(name, iter->value.GetString(), orion::ValueTypeString); parentP->add(nodeP); - break; - - case orion::ValueTypeBoolean: - b = PyObject_IsTrue(value); - LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); - nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); + } + else if (type == "Number") + { + nodeP = new orion::CompoundValueNode(name, iter->value.GetDouble(), orion::ValueTypeNumber); parentP->add(nodeP); - break; - - case orion::ValueTypeNull: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeNull); + } + else if (type == "True") + { + nodeP = new orion::CompoundValueNode(name, true, orion::ValueTypeBoolean); parentP->add(nodeP); - break; - - case orion::ValueTypeVector: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeVector); - size = PyList_Size(value); - for (Py_ssize_t ix = 0; ix < size; ++ix) + } + else if (type == "False") + { + nodeP = new orion::CompoundValueNode(name, false, orion::ValueTypeBoolean); + parentP->add(nodeP); + } + else if (type == "Null") + { + nodeP = new orion::CompoundValueNode(name, "", orion::ValueTypeNull); + parentP->add(nodeP); + } + else if (type == "Array") + { + nodeP = new orion::CompoundValueNode(name, "", orion::ValueTypeVector); + for (rapidjson::Value::ConstValueIterator iter2 = iter->value.Begin(); iter2 != iter->value.End(); ++iter2) { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(nodeP, PyList_GetItem(value, ix)); + processListItem(nodeP, iter2); } parentP->add(nodeP); - break; - - case orion::ValueTypeObject: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeObject); - pos = 0; - while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + } + else if (type == "Object") + { + nodeP = new orion::CompoundValueNode(name, "", orion::ValueTypeObject); + for (rapidjson::Value::ConstMemberIterator iter2 = iter->value.MemberBegin(); iter2 != iter->value.MemberEnd(); ++iter2) { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(nodeP, keyAux, valueAux); + processDictItem(nodeP, iter2); } parentP->add(nodeP); - break; - - case orion::ValueTypeNotGiven: - LM_E(("Runtime Error (value type not given))")); - break; - - default: - LM_E(("Runtime Error (value type unknown))")); + } + else + { + LM_E(("Runtime Error (unknown type: %s)", type.c_str())); } } -#endif + /* **************************************************************************** @@ -263,67 +177,67 @@ void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* */ void ExprResult::fill(std::string result) { - /* // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; - // Special case: expresion evalutes to None - if (result == Py_None) + rapidjson::Document document; + + document.Parse(result.c_str()); + + if (document.HasParseError()) { - LM_T(LmtExpr, ("ExprResult is null")); - valueType = orion::ValueTypeNull; + LM_E(("Runtime Error (parsing ExprResult: %s)", parseErrorString(document.GetParseError()).c_str())); return; } - // Other not null types - valueType = getPyObjectType(result); - if (valueType == orion::ValueTypeNumber) + std::string type = jsonParseTypeNames[document.GetType()]; + + if (type == "String") { - numberValue = PyFloat_AsDouble(result); - LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); + stringValue = document.GetString(); + valueType = orion::ValueTypeString; } - else if (valueType == orion::ValueTypeBoolean) + else if (type == "Number") { - boolValue = PyObject_IsTrue(result); - LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); + numberValue = document.GetDouble(); + valueType = orion::ValueTypeNumber; } - else if (valueType == orion::ValueTypeObject) + else if (type == "True") { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(result, &pos, &key, &value)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(compoundValueP, key, value); - } + boolValue = true; + valueType = orion::ValueTypeBoolean; } - else if (valueType == orion::ValueTypeVector) + else if (type == "False") { + boolValue = false; + valueType = orion::ValueTypeBoolean; + } + else if (type == "Null") + { + valueType = orion::ValueTypeNull; + } + else if (type == "Array") + { + valueType = orion::ValueTypeVector; compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); - Py_ssize_t size = PyList_Size(result); - for (Py_ssize_t ix = 0; ix < size; ++ix) + for (rapidjson::Value::ConstValueIterator iter = document.Begin(); iter != document.End(); ++iter) { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(compoundValueP, PyList_GetItem(result, ix)); + processListItem(compoundValueP, iter); } } - else if (valueType == orion::ValueTypeString) + else if (type == "Object") { - const char* str = PyUnicode_AsUTF8(result); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else + valueType = orion::ValueTypeObject; + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); + for (rapidjson::Value::ConstMemberIterator iter = document.MemberBegin(); iter != document.MemberEnd(); ++iter) { - LM_T(LmtExpr, ("ExprResult (string): %s", str)); - stringValue = std::string(str); + processDictItem(compoundValueP, iter); } - }*/ + } + else + { + LM_E(("Runtime Error (unknown type: %s)", type.c_str())); + } } diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test index 0a3bb36e7d..d6428e2074 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test @@ -36,7 +36,7 @@ accumulatorStart --pretty-print # 03. Update entity E1 with a=[{c:1},{b:-1},{b:2}] # 04. Update entity E1 with a=[{b:-1},{b:foo}] # 05. Update entity E1 with a=[{b:-1, c:1},{b:20, d:3}] -# 06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}]) +# 06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=[{b:-1},{b:2}], cal=[{b:-1}], cal=[{b:-1, c:1}]) # @@ -123,8 +123,8 @@ echo echo -echo "06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}])" -echo "=====================================================================================================" +echo "06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=[{b:-1},{b:2}], cal=[{b:-1}], cal=[{b:-1, c:1}])" +echo "===================================================================================================================" accumulatorDump echo echo @@ -175,8 +175,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}]) -===================================================================================================== +06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=[{b:-1},{b:2}], cal=[{b:-1}], cal=[{b:-1, c:1}]) +=================================================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 145 @@ -211,7 +211,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 132 +Content-Length: 146 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -225,7 +225,14 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "cal": { "metadata": {}, "type": "Calculated", - "value": null + "value": [ + { + "b": -1 + }, + { + "b": 2 + } + ] }, "id": "E1", "type": "T" @@ -236,7 +243,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 132 +Content-Length: 138 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -250,7 +257,11 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "cal": { "metadata": {}, "type": "Calculated", - "value": null + "value": [ + { + "b": -1 + } + ] }, "id": "E1", "type": "T" From c5411b32a7f8498519595a7cf5cbad2c5110b07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 18 Apr 2024 17:17:53 +0200 Subject: [PATCH 051/104] FIX cleanup --- src/lib/expressions/ExprContext.cpp | 33 ----------------------------- src/lib/expressions/ExprContext.h | 6 ------ src/lib/expressions/ExprManager.cpp | 6 ------ src/lib/expressions/ExprManager.h | 5 ----- src/lib/expressions/ExprResult.cpp | 2 +- src/lib/ngsiNotify/Notifier.cpp | 8 ------- 6 files changed, 1 insertion(+), 59 deletions(-) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 4b59840475..6f8421ca1e 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -200,28 +200,6 @@ bool ExprContextObject::isLegacy(void) -/* **************************************************************************** -* -* ExprContextObject::release - -*/ -void ExprContextObject::release(void) -{ - // FIXME PR: this method is probably no longer needed -} - - - -/* **************************************************************************** -* -* ExprContextList::ExprContextList - -*/ -ExprContextList::ExprContextList() -{ - // FIXME PR: this method is probably no longer needed -} - - - /* **************************************************************************** * * ExprContextList::get - @@ -306,14 +284,3 @@ void ExprContextList::add(ExprContextList exprContextList) LM_T(LmtExpr, ("adding to JEXL expression context list (list): %s", s.c_str())); jh.addRaw(s); } - - - -/* **************************************************************************** -* -* ExprContextList::relesase - -*/ -void ExprContextList::release(void) -{ - // FIXME PR: this method is probably no longer needed -} \ No newline at end of file diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 41a9acd980..aa92e3094a 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -58,8 +58,6 @@ class ExprContextObject void add(const std::string& key, ExprContextList exprContextList); bool isLegacy(void); - - void release(void); }; class ExprContextList @@ -68,8 +66,6 @@ class ExprContextList JsonVectorHelper jh; public: - ExprContextList(); - std::string get(void); void add(const std::string& value); void add(double value); @@ -77,8 +73,6 @@ class ExprContextList void add(void); void add(ExprContextObject exprContextObject); void add(ExprContextList exprContextList); - - void release(void); }; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 86e24cbae0..028b8808f7 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -49,12 +49,6 @@ extern "C" { */ void ExprManager::init(void) { - // FIXME PR: this is probably not needed - //if (sem_init(&sem, 0, 1) == -1) - //{ - // LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); - //} - jexlEngine = cjexl_new_engine(); } diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 821f7e9ad9..5d0892c078 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -26,9 +26,6 @@ * Author: Fermin Galan */ -// FIXME PR: probably this is no longer needed -//#include - #include "expressions/ExprContext.h" #include "expressions/ExprResult.h" @@ -39,8 +36,6 @@ class ExprManager { private: - // FIXME PR: probably this is no longer needed - //sem_t sem; void* jexlEngine; public: diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 939c3ebfc8..6bb5bf4e87 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -191,7 +191,7 @@ void ExprResult::fill(std::string result) } std::string type = jsonParseTypeNames[document.GetType()]; - + if (type == "String") { stringValue = document.GetString(); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 0c6fe587ff..b13d113b5a 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -372,7 +372,6 @@ static SenderThreadParams* buildSenderParamsCustom if (macroSubstitute(&url, notifUrl, &exprContext, "", true) == false) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } @@ -389,7 +388,6 @@ static SenderThreadParams* buildSenderParamsCustom if (!setPayload(includePayload, notifPayload, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } } @@ -406,7 +404,6 @@ static SenderThreadParams* buildSenderParamsCustom if (!setNgsiPayload(ngsi, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } mimeType = "application/json"; @@ -426,7 +423,6 @@ static SenderThreadParams* buildSenderParamsCustom if ((macroSubstitute(&key, it->first, &exprContext, "", true) == false) || (macroSubstitute(&value, it->second, &exprContext, "", true) == false)) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } @@ -453,7 +449,6 @@ static SenderThreadParams* buildSenderParamsCustom if ((macroSubstitute(&key, it->first, &exprContext, "", true) == false) || (macroSubstitute(&value, it->second, &exprContext, "", true) == false)) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } @@ -485,7 +480,6 @@ static SenderThreadParams* buildSenderParamsCustom if (!parseUrl(url, host, port, uriPath, protocol)) { LM_E(("Runtime Error (not sending notification: malformed URL: '%s')", url.c_str())); - exprContext.release(); return NULL; } @@ -520,7 +514,6 @@ static SenderThreadParams* buildSenderParamsCustom if (macroSubstitute(&topic, notification.mqttInfo.topic, &exprContext, "", true) == false) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } } @@ -556,7 +549,6 @@ static SenderThreadParams* buildSenderParamsCustom snprintf(suffix, sizeof(suffix), "%u", correlatorCounter); paramsP->fiwareCorrelator = fiwareCorrelator + "; cbnotif=" + suffix; - exprContext.release(); return paramsP; } From 2a20139fb3a8b2d640d9c037f825e56ee498ca25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 19 Apr 2024 16:24:58 +0200 Subject: [PATCH 052/104] FIX align version tests --- .../cases/0000_version_operation/version_via_rest.test | 2 +- test/functionalTest/cases/0501_cors/version_request.test | 6 +++--- .../cases/1916_fiware_correlator/fiware_correlator.test | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/functionalTest/cases/0000_version_operation/version_via_rest.test b/test/functionalTest/cases/0000_version_operation/version_via_rest.test index e4fd1517f7..1ff1b3f428 100644 --- a/test/functionalTest/cases/0000_version_operation/version_via_rest.test +++ b/test/functionalTest/cases/0000_version_operation/version_via_rest.test @@ -64,7 +64,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/functionalTest/cases/0501_cors/version_request.test b/test/functionalTest/cases/0501_cors/version_request.test index 57b963b40a..7c56566ec8 100644 --- a/test/functionalTest/cases/0501_cors/version_request.test +++ b/test/functionalTest/cases/0501_cors/version_request.test @@ -94,7 +94,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -131,7 +131,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -166,7 +166,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test index 01cd198b53..b7ec3feefb 100644 --- a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test +++ b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test @@ -73,7 +73,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -108,7 +108,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", From fb976cdcf590da1517d217ccc197fe678f9ae223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 22 Apr 2024 12:26:33 +0200 Subject: [PATCH 053/104] FIX unit tests --- test/unittests/serviceRoutines/versionTreat_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unittests/serviceRoutines/versionTreat_test.cpp b/test/unittests/serviceRoutines/versionTreat_test.cpp index fd8a4e2e4b..1ebdff883c 100644 --- a/test/unittests/serviceRoutines/versionTreat_test.cpp +++ b/test/unittests/serviceRoutines/versionTreat_test.cpp @@ -95,7 +95,7 @@ TEST(versionTreat, ok) EXPECT_TRUE(strstr(out.c_str(), "libcurl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "libmicrohttpd") != NULL); EXPECT_TRUE(strstr(out.c_str(), "libmosquitto") != NULL); - EXPECT_TRUE(strstr(out.c_str(), "libpython") != NULL); + EXPECT_TRUE(strstr(out.c_str(), "libcjexl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "openssl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "rapidjson") != NULL); EXPECT_TRUE(strstr(out.c_str(), "mongoc") != NULL); From 30d4556c5d57df1fd85dc0b9049f56ca6c475b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Apr 2024 16:56:52 +0200 Subject: [PATCH 054/104] FIX libcjexl download in CI build.sh --- ci/deb/build.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index aea9ed627d..25479aeb9b 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -172,6 +172,20 @@ echo "===================================== PREPARE ============================ echo "Builder: create temp folders" rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} +# FIXME PR: unhardwire release number +if [ -z "${REPO_ACCESS_TOKEN}" ]; then + echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" +else + res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/0.0.0) + if [ "$res_code" -eq 200 ]; then + echo "Builder: downloading cjexl lib" + ASSET_ID=$(curl -s -S -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/0.0.0 | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') + curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $REPO_ACCESS_TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID + else + echo "Builder: error $res_code accessing cjexl release. Maybe token is invalid?" + fi +fi + if [ -n "${branch}" ]; then echo "===================================== CLONE ============================================" From 9b02e231a01152541e4cbca6dfcbe26062256aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Apr 2024 17:59:23 +0200 Subject: [PATCH 055/104] FIX add token to gitactions that need it --- .github/workflows/functional.yml | 2 +- .github/workflows/valgrind-nocache.yml | 2 +- .github/workflows/valgrind.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 7b6b61eeea..61b91cfb66 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -60,4 +60,4 @@ jobs: - name: Run functional test run: | - docker run --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional + docker run --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional diff --git a/.github/workflows/valgrind-nocache.yml b/.github/workflows/valgrind-nocache.yml index ea86291e59..fa4edadef8 100644 --- a/.github/workflows/valgrind-nocache.yml +++ b/.github/workflows/valgrind-nocache.yml @@ -58,4 +58,4 @@ jobs: - name: Run valgrind test run: | - docker run --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind + docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 67940e7d17..36933d8ea3 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -55,4 +55,4 @@ jobs: - name: Run valgrind test run: | - docker run --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind + docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind From 30911873609e67dbd799a7ed1a149748cb41c5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Apr 2024 18:09:41 +0200 Subject: [PATCH 056/104] FIX syntax error in GitActions --- .github/workflows/functional.yml | 2 +- .github/workflows/valgrind-nocache.yml | 2 +- .github/workflows/valgrind.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 61b91cfb66..b8b4bcb597 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -60,4 +60,4 @@ jobs: - name: Run functional test run: | - docker run --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional + docker run --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional diff --git a/.github/workflows/valgrind-nocache.yml b/.github/workflows/valgrind-nocache.yml index fa4edadef8..e8ff06e31b 100644 --- a/.github/workflows/valgrind-nocache.yml +++ b/.github/workflows/valgrind-nocache.yml @@ -58,4 +58,4 @@ jobs: - name: Run valgrind test run: | - docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind + docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 36933d8ea3..33010b755d 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -55,4 +55,4 @@ jobs: - name: Run valgrind test run: | - docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind + docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind From 280d8f71900c88cd611e9313560e7cd98a8a63e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Apr 2024 18:31:49 +0200 Subject: [PATCH 057/104] FIX GitAction unit tests --- .github/workflows/unit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 722a7bb8db..fa1c722db5 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -31,4 +31,4 @@ jobs: - name: Run unit tests run: | - docker run --network host -t --rm -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts unit + docker run --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts unit From 21699835811039625f6ffd8aeb330410a348ff06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Apr 2024 16:18:50 +0200 Subject: [PATCH 058/104] FIX improve buid.sh --- ci/deb/build.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index 25479aeb9b..107cb11f30 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -173,14 +173,17 @@ echo "Builder: create temp folders" rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} # FIXME PR: unhardwire release number +CJEXL_VERSION="0.0.0" if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/0.0.0) + res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION) if [ "$res_code" -eq 200 ]; then echo "Builder: downloading cjexl lib" - ASSET_ID=$(curl -s -S -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/0.0.0 | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') + ASSET_ID=$(curl -s -S -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $REPO_ACCESS_TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID + MD5SUM=$(md5sum /usr/local/lib/libcjexl.a | awk -F ' ' '{print$1}') + echo "Builder: cjexl lib md5sum is $MD5SUM" else echo "Builder: error $res_code accessing cjexl release. Maybe token is invalid?" fi From 49aa74c1fe2e6d770a27b375835683f03134d0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Apr 2024 16:19:15 +0200 Subject: [PATCH 059/104] FIX ExprResult::release() in the proper places --- src/lib/common/macroSubstitute.cpp | 19 +++++++++++++------ src/lib/expressions/ExprManager.cpp | 3 +++ src/lib/expressions/ExprResult.cpp | 4 ++-- src/lib/expressions/ExprResult.h | 2 +- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 23b631a710..cc994b70cd 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -54,22 +54,25 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e std::string macroName = stringValue.substr(2, stringValue.size() - 3); ExprResult r = exprMgr.evaluate(exprContextObjectP, macroName); + std::string result; if (r.valueType == orion::ValueTypeNull) { - return notFoundDefault; + result = notFoundDefault; } else { // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - return removeQuotes(r.toString()); + result = removeQuotes(r.toString()); } else { - return r.toString(); + result = r.toString(); } } + r.release(); + return result; } else if (exprContextObjectP != NULL) { @@ -101,10 +104,11 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, const std::string key, const std::string& notFoundDefault, bool raw) { ExprResult r = exprMgr.evaluate(exprContextObjectP, key); + std::string result; if (r.valueType == orion::ValueTypeNull) { - return notFoundDefault; + result = notFoundDefault; } else { @@ -120,13 +124,16 @@ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, c { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), // so double quotes have to be be removed - return removeQuotes(s); + result = removeQuotes(s); } else { - return s; + result = s; } } + + r.release(); + return result; } diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 028b8808f7..0ee2fcb23d 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -85,6 +85,9 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st LM_T(LmtExpr, ("evaluating JEXL expression <%s> with context <%s>", _expression.c_str(), context.c_str())); const char* result = cjexl_eval(jexlEngine, _expression.c_str(), context.c_str()); LM_T(LmtExpr, ("JEXL evaluation result: <%s>", result)); + + // The ExprResult::fill() method allocates dynamic memory. So, the callers to evaluate() are supposed to invoke ExprResult::release() + // method to free it r.fill(result); } diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 6bb5bf4e87..e071a7d2ea 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -25,7 +25,7 @@ // The ExprResult class is used as return value in ExprManager::evaluate(). We could return std::string // in that function and simplify (so avoiding the ExprResult class). But in that case float rounding is -// problematic (e.g. 1.999999 instead of 2), as they don't take advanage of the ad hoc logic implemented +// problematic (e.g. 1.999999 instead of 2), as they don't take advantage of the ad hoc logic implemented // in ContextAttribute rendering #include "rapidjson/document.h" @@ -175,7 +175,7 @@ void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value:: * fill - * */ -void ExprResult::fill(std::string result) +void ExprResult::fill(const std::string& result) { // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index 12dd928497..8b9eeb8233 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -49,7 +49,7 @@ class ExprResult // Use only when valueType is object or vector orion::CompoundValueNode* compoundValueP; - void fill(std::string result); + void fill(const std::string& result); std::string toString(void); void release(void); From c9a0e5866ccbb8be0defd0bfdeba230de55ab876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Apr 2024 16:56:04 +0200 Subject: [PATCH 060/104] FIX add ExprResult constructor to initialize compoundValueP to NULL --- src/lib/expressions/ExprResult.cpp | 20 ++++++++++++++++---- src/lib/expressions/ExprResult.h | 2 ++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index e071a7d2ea..5f34023565 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -50,7 +50,7 @@ static void processDictItem(orion::CompoundValueNode* parentP, const rapidjson:: * processListItem - * */ -void processListItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstValueIterator& iter) +static void processListItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstValueIterator& iter) { orion::CompoundValueNode* nodeP; @@ -112,7 +112,7 @@ void processListItem(orion::CompoundValueNode* parentP, const rapidjson::Value:: * processDictItem - * */ -void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstMemberIterator& iter) +static void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstMemberIterator& iter) { orion::CompoundValueNode* nodeP; @@ -172,7 +172,19 @@ void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value:: /* **************************************************************************** * -* fill - +* ExprResult::ExprResult - +* +*/ +ExprResult::ExprResult() +{ + compoundValueP = NULL; +} + + + +/* **************************************************************************** +* +* ExprResult::fill - * */ void ExprResult::fill(const std::string& result) @@ -244,7 +256,7 @@ void ExprResult::fill(const std::string& result) /* **************************************************************************** * -* toString - +* ExprResult::toString - * * Pretty similar to ContextAttribute::toJsonValue() * diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index 8b9eeb8233..9814d3073b 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -49,6 +49,8 @@ class ExprResult // Use only when valueType is object or vector orion::CompoundValueNode* compoundValueP; + ExprResult(void); + void fill(const std::string& result); std::string toString(void); From ecf7fbd2117e6610bc428ead4c5ea9ca58ffb838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Apr 2024 13:40:51 +0200 Subject: [PATCH 061/104] FIX add expr fields in timing section in GET /statistics --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/admin/statistics.md | 10 +++ src/lib/common/statistics.cpp | 16 +++++ src/lib/common/statistics.h | 101 ++++++++++++++++++++++++++++ src/lib/expressions/ExprManager.cpp | 7 ++ src/lib/ngsiNotify/Notifier.cpp | 2 + src/lib/rest/rest.cpp | 4 ++ 7 files changed, 141 insertions(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 8cb6e6e302..9c42cb5f02 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ - Add: exprLang field in custom notifications (#4004) +- Add: expression context build and evaluation counter in timing section in GET /statistics (#4004) - Fix: lighter operation to get databases list from MongoDB (#4517) \ No newline at end of file diff --git a/doc/manuals/admin/statistics.md b/doc/manuals/admin/statistics.md index 768368691f..a2760412f2 100644 --- a/doc/manuals/admin/statistics.md +++ b/doc/manuals/admin/statistics.md @@ -133,6 +133,10 @@ Provides timing information, i.e. the time that CB passes executing in different "mongoReadWait": 4656.924425628, "mongoWriteWait": 259.347915990, "mongoCommandWait": 0.514811318, + "exprLegacyCtxBld": FIXME PR, + "exprLegacyEval": FIXME PR, + "exprJexlCtxBld": FIXME PR, + "exprJexlEval": FIXME PR, "render": 108.162782114, "total": 6476.593504743 }, @@ -140,6 +144,10 @@ Provides timing information, i.e. the time that CB passes executing in different "mongoBackend": 0.014752309, "mongoReadWait": 0.012018445, "mongoWriteWait": 0.000574611, + "exprLegacyCtxBld": FIXME PR, + "exprLegacyEval": FIXME PR, + "exprJexlCtxBld": FIXME PR, + "exprJexlEval": FIXME PR, "render": 0.000019136, "total": 0.015148915 } @@ -156,6 +164,8 @@ The block includes two main sections: The particular counters are as follows: +FIXME PR: explain expr* fields + * `total`: processing time for the whole request, excluding the time that the HTTP library takes for request/response dispatching (pseudo end-to-end time) * `jsonV1Parse`: time passed in NGSIv1 JSON parsing module (pseudo self-time) diff --git a/src/lib/common/statistics.cpp b/src/lib/common/statistics.cpp index 794eafee2b..1eb5dde069 100644 --- a/src/lib/common/statistics.cpp +++ b/src/lib/common/statistics.cpp @@ -195,6 +195,10 @@ std::string renderTimingStatistics(void) bool accMongoReadWaitTime = (accTimeStat.mongoReadWaitTime.tv_sec != 0) || (accTimeStat.mongoReadWaitTime.tv_nsec != 0); bool accMongoWriteWaitTime = (accTimeStat.mongoWriteWaitTime.tv_sec != 0) || (accTimeStat.mongoWriteWaitTime.tv_nsec != 0); bool accMongoCommandWaitTime = (accTimeStat.mongoCommandWaitTime.tv_sec != 0) || (accTimeStat.mongoCommandWaitTime.tv_nsec != 0); + bool accExprLegacyCtxBldTime = (accTimeStat.exprLegacyCtxBldTime.tv_sec != 0) || (accTimeStat.exprLegacyCtxBldTime.tv_nsec != 0); + bool accExprLegacyEvalTime = (accTimeStat.exprLegacyEvalTime.tv_sec != 0) || (accTimeStat.exprLegacyEvalTime.tv_nsec != 0); + bool accExprJexlCtxBldTime = (accTimeStat.exprJexlCtxBldTime.tv_sec != 0) || (accTimeStat.exprJexlCtxBldTime.tv_nsec != 0); + bool accExprJexlEvalTime = (accTimeStat.exprJexlEvalTime.tv_sec != 0) || (accTimeStat.exprJexlEvalTime.tv_nsec != 0); bool accRenderTime = (accTimeStat.renderTime.tv_sec != 0) || (accTimeStat.renderTime.tv_nsec != 0); bool accReqTime = (accTimeStat.reqTime.tv_sec != 0) || (accTimeStat.reqTime.tv_nsec != 0); @@ -204,6 +208,10 @@ std::string renderTimingStatistics(void) bool lastMongoReadWaitTime = (lastTimeStat.mongoReadWaitTime.tv_sec != 0) || (lastTimeStat.mongoReadWaitTime.tv_nsec != 0); bool lastMongoWriteWaitTime = (lastTimeStat.mongoWriteWaitTime.tv_sec != 0) || (lastTimeStat.mongoWriteWaitTime.tv_nsec != 0); bool lastMongoCommandWaitTime = (lastTimeStat.mongoCommandWaitTime.tv_sec != 0) || (lastTimeStat.mongoCommandWaitTime.tv_nsec != 0); + bool lastExprLegacyCtxBldTime = (lastTimeStat.exprLegacyCtxBldTime.tv_sec != 0) || (lastTimeStat.exprLegacyCtxBldTime.tv_nsec != 0); + bool lastExprLegacyEvalTime = (lastTimeStat.exprLegacyEvalTime.tv_sec != 0) || (lastTimeStat.exprLegacyEvalTime.tv_nsec != 0); + bool lastExprJexlCtxBldTime = (lastTimeStat.exprJexlCtxBldTime.tv_sec != 0) || (lastTimeStat.exprJexlCtxBldTime.tv_nsec != 0); + bool lastExprJexlEvalTime = (lastTimeStat.exprJexlEvalTime.tv_sec != 0) || (lastTimeStat.exprJexlEvalTime.tv_nsec != 0); bool lastRenderTime = (lastTimeStat.renderTime.tv_sec != 0) || (lastTimeStat.renderTime.tv_nsec != 0); bool lastReqTime = (lastTimeStat.reqTime.tv_sec != 0) || (lastTimeStat.reqTime.tv_nsec != 0); @@ -228,6 +236,10 @@ std::string renderTimingStatistics(void) if (accMongoReadWaitTime) accJh.addNumber("mongoReadWait", timeSpecToFloat(accTimeStat.mongoReadWaitTime)); if (accMongoWriteWaitTime) accJh.addNumber("mongoWriteWait", timeSpecToFloat(accTimeStat.mongoWriteWaitTime)); if (accMongoCommandWaitTime) accJh.addNumber("mongoCommandWait", timeSpecToFloat(accTimeStat.mongoCommandWaitTime)); + if (accExprLegacyCtxBldTime) accJh.addNumber("exprLegacyCtxBld", timeSpecToFloat(accTimeStat.exprLegacyCtxBldTime)); + if (accExprLegacyEvalTime) accJh.addNumber("exprLegacyEval", timeSpecToFloat(accTimeStat.exprLegacyEvalTime)); + if (accExprJexlCtxBldTime) accJh.addNumber("exprJexlCtxBld", timeSpecToFloat(accTimeStat.exprJexlCtxBldTime)); + if (accExprJexlEvalTime) accJh.addNumber("exprJexlEval", timeSpecToFloat(accTimeStat.exprJexlEvalTime)); if (accRenderTime) accJh.addNumber("render", timeSpecToFloat(accTimeStat.renderTime)); if (accReqTime) accJh.addNumber("total", timeSpecToFloat(accTimeStat.reqTime)); @@ -243,6 +255,10 @@ std::string renderTimingStatistics(void) if (lastMongoReadWaitTime) lastJh.addNumber("mongoReadWait", timeSpecToFloat(lastTimeStat.mongoReadWaitTime)); if (lastMongoWriteWaitTime) lastJh.addNumber("mongoWriteWait", timeSpecToFloat(lastTimeStat.mongoWriteWaitTime)); if (lastMongoCommandWaitTime) lastJh.addNumber("mongoCommandWait", timeSpecToFloat(lastTimeStat.mongoCommandWaitTime)); + if (lastExprLegacyCtxBldTime) lastJh.addNumber("exprLegacyCtxBld", timeSpecToFloat(lastTimeStat.exprLegacyCtxBldTime)); + if (lastExprLegacyEvalTime) lastJh.addNumber("exprLegacyEval", timeSpecToFloat(lastTimeStat.exprLegacyEvalTime)); + if (lastExprJexlCtxBldTime) lastJh.addNumber("exprJexlCtxBld", timeSpecToFloat(lastTimeStat.exprJexlCtxBldTime)); + if (lastExprJexlEvalTime) lastJh.addNumber("exprJexlEval", timeSpecToFloat(lastTimeStat.exprJexlEvalTime)); if (lastRenderTime) lastJh.addNumber("render", timeSpecToFloat(lastTimeStat.renderTime)); if (lastReqTime) lastJh.addNumber("total", timeSpecToFloat(lastTimeStat.reqTime)); diff --git a/src/lib/common/statistics.h b/src/lib/common/statistics.h index ab4034f5f2..623e074dcf 100644 --- a/src/lib/common/statistics.h +++ b/src/lib/common/statistics.h @@ -208,6 +208,103 @@ +/* **************************************************************************** +* +* TIME_EXPR_CTXBLD_START - +*/ +#define TIME_EXPR_CTXBLD_START() \ + struct timespec exprCtxBldStart; \ + struct timespec exprCtxBldEnd; \ + \ + if (timingStatistics) \ + { \ + clock_gettime(CLOCK_REALTIME, &exprCtxBldStart); \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_CTXBLD_STOP - +*/ +#define TIME_EXPR_CTXBLD_STOP() \ + if (timingStatistics) \ + { \ + struct timespec diff; \ + clock_gettime(CLOCK_REALTIME, &exprCtxBldEnd); \ + clock_difftime(&exprCtxBldEnd, &exprCtxBldStart, &diff); \ + if (legacy) \ + { \ + clock_addtime(&threadLastTimeStat.exprLegacyCtxBldTime, &diff); \ + } \ + else \ + { \ + clock_addtime(&threadLastTimeStat.exprJexlCtxBldTime, &diff); \ + } \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_LEGACY_EVAL_START - +*/ +#define TIME_EXPR_LEGACY_EVAL_START() \ + struct timespec exprLegacyEvalStart; \ + struct timespec exprLegacyEvalEnd; \ + \ + if (timingStatistics) \ + { \ + clock_gettime(CLOCK_REALTIME, &exprLegacyEvalStart); \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_LEGACY_EVAL_STOP - +*/ +#define TIME_EXPR_LEGACY_EVAL_STOP() \ + if (timingStatistics) \ + { \ + struct timespec diff; \ + clock_gettime(CLOCK_REALTIME, &exprLegacyEvalEnd); \ + clock_difftime(&exprLegacyEvalEnd, &exprLegacyEvalStart, &diff); \ + clock_addtime(&threadLastTimeStat.exprLegacyEvalTime, &diff); \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_JEXL_EVAL_START - +*/ +#define TIME_EXPR_JEXL_EVAL_START() \ + struct timespec exprJexlEvalStart; \ + struct timespec exprJexlEvalEnd; \ + \ + if (timingStatistics) \ + { \ + clock_gettime(CLOCK_REALTIME, &exprJexlEvalStart); \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_JEXL_EVAL_STOP - +*/ +#define TIME_EXPR_JEXL_EVAL_STOP() \ + if (timingStatistics) \ + { \ + struct timespec diff; \ + clock_gettime(CLOCK_REALTIME, &exprJexlEvalEnd); \ + clock_difftime(&exprJexlEvalEnd, &exprJexlEvalStart, &diff); \ + clock_addtime(&threadLastTimeStat.exprJexlEvalTime, &diff); \ + } + + + /* **************************************************************************** * * TimeStat - @@ -220,6 +317,10 @@ typedef struct TimeStat struct timespec mongoReadWaitTime; struct timespec mongoWriteWaitTime; struct timespec mongoCommandWaitTime; + struct timespec exprLegacyCtxBldTime; + struct timespec exprLegacyEvalTime; + struct timespec exprJexlCtxBldTime; + struct timespec exprJexlEvalTime; struct timespec renderTime; struct timespec reqTime; } TimeStat; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 0ee2fcb23d..0e59f59bb6 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -28,6 +28,7 @@ #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" +#include "common/statistics.h" // Interface to use libcjexl.a extern "C" { @@ -66,6 +67,8 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st if (exprContextObjectP->isLegacy()) { // std::map based evaluation. Only pure replacement is supported + + TIME_EXPR_LEGACY_EVAL_START(); LM_T(LmtExpr, ("evaluating legacy expression: <%s>", _expression.c_str())); std::map* replacementsP = exprContextObjectP->getMap(); @@ -77,10 +80,13 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st r.stringValue = iter->second; LM_T(LmtExpr, ("legacy evaluation result: <%s>", r.stringValue.c_str())); } + TIME_EXPR_LEGACY_EVAL_STOP(); } else { // JEXL based evaluation + + TIME_EXPR_JEXL_EVAL_START(); std::string context = exprContextObjectP->getJexlContext(); LM_T(LmtExpr, ("evaluating JEXL expression <%s> with context <%s>", _expression.c_str(), context.c_str())); const char* result = cjexl_eval(jexlEngine, _expression.c_str(), context.c_str()); @@ -89,6 +95,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st // The ExprResult::fill() method allocates dynamic memory. So, the callers to evaluate() are supposed to invoke ExprResult::release() // method to free it r.fill(result); + TIME_EXPR_JEXL_EVAL_STOP(); } return r; diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index b13d113b5a..2877049fd7 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -326,6 +326,7 @@ static SenderThreadParams* buildSenderParamsCustom // updated. In legacy model, if the key already exists, the operation is ignored (so previous value is preserved). Taking // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence // over macros comming from macros of the same name we conditionally add them depending the case + TIME_EXPR_CTXBLD_START(); exprContext.add("id", en.id); exprContext.add("type", en.type); if (!legacy) @@ -344,6 +345,7 @@ static SenderThreadParams* buildSenderParamsCustom exprContext.add("servicePath", en.servicePath); exprContext.add("authToken", xauthToken); } + TIME_EXPR_CTXBLD_STOP(); // // 1. Verb/Method diff --git a/src/lib/rest/rest.cpp b/src/lib/rest/rest.cpp index cc35ff57fd..3ddd0a85e7 100644 --- a/src/lib/rest/rest.cpp +++ b/src/lib/rest/rest.cpp @@ -618,6 +618,10 @@ static void requestCompleted clock_addtime(&accTimeStat.mongoWriteWaitTime, &threadLastTimeStat.mongoWriteWaitTime); clock_addtime(&accTimeStat.mongoReadWaitTime, &threadLastTimeStat.mongoReadWaitTime); clock_addtime(&accTimeStat.mongoCommandWaitTime, &threadLastTimeStat.mongoCommandWaitTime); + clock_addtime(&accTimeStat.exprLegacyCtxBldTime, &threadLastTimeStat.exprLegacyCtxBldTime); + clock_addtime(&accTimeStat.exprLegacyEvalTime, &threadLastTimeStat.exprLegacyEvalTime); + clock_addtime(&accTimeStat.exprJexlCtxBldTime, &threadLastTimeStat.exprJexlCtxBldTime); + clock_addtime(&accTimeStat.exprJexlEvalTime, &threadLastTimeStat.exprJexlEvalTime); clock_addtime(&accTimeStat.renderTime, &threadLastTimeStat.renderTime); clock_addtime(&accTimeStat.reqTime, &threadLastTimeStat.reqTime); From 4db6066af441002a20ea0318e222533ce2846234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Apr 2024 16:58:23 +0200 Subject: [PATCH 062/104] FIX PoC to measure times CJEXL --- src/lib/expressions/ExprManager.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 0e59f59bb6..7bd204ccce 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -51,6 +51,23 @@ extern "C" { void ExprManager::init(void) { jexlEngine = cjexl_new_engine(); + LM_T(LmtExpr, ("JEXL engine has been initialized")); + + // Create context + ExprContextObject eco; + eco.add("x", 4.0); + eco.add("y", 11.0); + LM_T(LmtExpr, ("jexl context has been built")); + + // Call evaluate method + ExprResult r = evaluate(&eco, "x + y"); + LM_T(LmtExpr, ("jexl evaluation is done")); + + // Print result + LM_T(LmtExpr, ("jexl evaluation result is obtainted")); + LM_T(LmtExpr, ("jexl result: %f", r.numberValue)); + + cjexl_free_engine(jexlEngine); } From b95dd16e1ec8a4011f5c8c4c3a4b886abc9aa7e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Apr 2024 17:26:48 +0200 Subject: [PATCH 063/104] FIX PoC to compare CJEXL with std::map --- src/lib/expressions/ExprManager.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 7bd204ccce..40e86a69fd 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -54,13 +54,12 @@ void ExprManager::init(void) LM_T(LmtExpr, ("JEXL engine has been initialized")); // Create context - ExprContextObject eco; + ExprContextObject eco(false); eco.add("x", 4.0); - eco.add("y", 11.0); LM_T(LmtExpr, ("jexl context has been built")); // Call evaluate method - ExprResult r = evaluate(&eco, "x + y"); + ExprResult r = evaluate(&eco, "x"); LM_T(LmtExpr, ("jexl evaluation is done")); // Print result From 64ed3693d5b93da572d001c704d913fe315c4277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Apr 2024 17:27:01 +0200 Subject: [PATCH 064/104] Revert last two commits with PoC This reverts commit b95dd16e1ec8a4011f5c8c4c3a4b886abc9aa7e3. --- src/lib/expressions/ExprManager.cpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 40e86a69fd..0e59f59bb6 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -51,22 +51,6 @@ extern "C" { void ExprManager::init(void) { jexlEngine = cjexl_new_engine(); - LM_T(LmtExpr, ("JEXL engine has been initialized")); - - // Create context - ExprContextObject eco(false); - eco.add("x", 4.0); - LM_T(LmtExpr, ("jexl context has been built")); - - // Call evaluate method - ExprResult r = evaluate(&eco, "x"); - LM_T(LmtExpr, ("jexl evaluation is done")); - - // Print result - LM_T(LmtExpr, ("jexl evaluation result is obtainted")); - LM_T(LmtExpr, ("jexl result: %f", r.numberValue)); - - cjexl_free_engine(jexlEngine); } From 9434b9c95a6a9a644f25fa7b2de41fc5bda01bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 09:01:35 +0200 Subject: [PATCH 065/104] FIX revert exprLang --- CHANGES_NEXT_RELEASE | 1 - doc/manuals/admin/database_model.md | 1 - doc/manuals/orion-api.md | 2 - src/lib/apiTypesV2/HttpInfo.cpp | 6 - src/lib/apiTypesV2/HttpInfo.h | 2 - src/lib/apiTypesV2/MqttInfo.cpp | 6 - src/lib/apiTypesV2/MqttInfo.h | 2 - src/lib/expressions/ExprManager.cpp | 1 + src/lib/jsonParseV2/parseSubscription.cpp | 40 - .../mongoBackend/MongoCommonSubscription.cpp | 7 - src/lib/mongoBackend/dbConstants.h | 1 - src/lib/ngsiNotify/Notifier.cpp | 5 +- .../notification_templates_cache_refresh.test | 3 +- ...n_templates_many_notifications_legacy.test | 449 ----------- ...s_with_decoded_chars_in_notifications.test | 3 +- .../http_info_for_sub_update.test | 6 +- .../custom_url_validity_check.test | 3 +- ...tom_notification_http_json_basic_crud.test | 25 +- ...stom_notification_http_json_constants.test | 4 +- ...m_notification_http_json_replacements.test | 4 +- ...ication_http_json_replacements_inside.test | 4 +- ...n_http_json_replacements_inside_multi.test | 4 +- ...n_http_json_replacements_intermediate.test | 4 +- ...tom_notification_mqtt_json_basic_crud.test | 25 +- ...stom_notification_mqtt_json_constants.test | 4 +- ...m_notification_mqtt_json_replacements.test | 4 +- ...ication_mqtt_json_replacements_inside.test | 4 +- ...n_mqtt_json_replacements_inside_multi.test | 4 +- ...n_mqtt_json_replacements_intermediate.test | 4 +- .../self_notification_multi_hop.test | 9 +- .../self_notification_one_hop.test | 3 +- .../mqtt_custom_subscription_basic_crud.test | 8 +- .../mqtt_custom_subscription_update.test | 12 +- .../3001_mqtt/mqtt_notification_custom.test | 3 +- ...tification_problem_with_custom_header.test | 4 +- .../null_payload_get.test | 6 +- .../covered_custom_notification_legacy.test | 217 ------ .../per_sub_httpTimeout_crud_custom.test | 15 +- ...r_sub_httpTimeout_notification_custom.test | 6 +- .../mqtt_custom_notifications_auth.test | 12 +- ...http_custom_exprlang_field_basic_crud.test | 664 ----------------- .../jexl_expr_attrs_weird_syntax.test | 12 +- .../legacy_expr_attrs_weird_syntax.test | 216 ------ ...mqtt_custom_exprlang_field_basic_crud.test | 699 ------------------ ...m_notification_http_ngsi_attr_no_type.test | 3 +- ...tom_notification_http_ngsi_basic_crud.test | 25 +- ...cation_http_ngsi_basic_crud_compounds.test | 21 +- ...fication_http_ngsi_basic_crud_partial.test | 21 +- ...m_notification_mqtt_ngsi_attr_no_type.test | 3 +- ...tom_notification_mqtt_ngsi_basic_crud.test | 25 +- ...cation_mqtt_ngsi_basic_crud_compounds.test | 21 +- ...fication_mqtt_ngsi_basic_crud_partial.test | 21 +- .../mqttCustom_retain_basic_crud.test | 30 +- .../ngsi_patching_special_attr_types.test | 6 +- 54 files changed, 120 insertions(+), 2570 deletions(-) delete mode 100644 test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test delete mode 100644 test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test delete mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test delete mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test delete mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 9c42cb5f02..8876c3fb18 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,2 @@ -- Add: exprLang field in custom notifications (#4004) - Add: expression context build and evaluation counter in timing section in GET /statistics (#4004) - Fix: lighter operation to get databases list from MongoDB (#4517) \ No newline at end of file diff --git a/doc/manuals/admin/database_model.md b/doc/manuals/admin/database_model.md index 1c3ad517c6..ae438dbef6 100644 --- a/doc/manuals/admin/database_model.md +++ b/doc/manuals/admin/database_model.md @@ -329,7 +329,6 @@ notifications. It is a number between 0 and 1800000. If defined to 0 or omitted, in the Orion API. More detail of this functionality [here](../orion-api.md#ngsi-payload-patching). The value of this field is an object with a `attrs` key which value is a simplified version of `attrs` in [the entities collection](#entities-collection). -- **exprLang**: optional field to store the expression language. Only for custom subscriptions. If omitted `legacy` is assumed. - **lastFailure**: the time (as integer number, meaning seconds) when last notification failure occurred. Not present if the subscription has never failed. - **lastFailureReason**: text describing the cause of the last failure. diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 50493f53c1..5e8e6e9bef 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -3947,7 +3947,6 @@ A `httpCustom` object contains the following subfields. | `payload` | ✓ | string | Text-based payload to be used in notifications. In case of empty string or omitted, the default payload (see [Notification Messages](#notification-messages) sections) is used. If `null`, notification will not include any payload. | | `json` | ✓ | object | JSON-based payload to be used in notifications. See [JSON Payloads](#json-payloads) section for more details. | | `ngsi` | ✓ | object | NGSI patching for payload to be used in notifications. See [NGSI payload patching](#ngsi-payload-patching) section for more details. | -| `exprLang` | ✓ | string | Expression language to use, either `legacy` or `jexl` (default is `jexl`) | | `timeout` | ✓ | number | Maximum time (in milliseconds) the subscription waits for the response. The maximum value allowed for this parameter is 1800000 (30 minutes). If `timeout` is defined to 0 or omitted, then the value passed as `-httpTimeout` CLI parameter is used. See section in the [Command line options](admin/cli.md#command-line-options) for more details. | `payload`, `json` or `ngsi` cannot be used at the same time, they are mutually exclusive. @@ -3969,7 +3968,6 @@ A `mqttCustom` object contains the following subfields. | `payload` | ✓ | string | Text-based payload to be used in notifications. In case of empty string or omitted, the default payload (see [Notification Messages](#notification-messages) sections) is used. If `null`, notification will not include any payload. | | `json` | ✓ | object | JSON-based payload to be used in notifications. See [JSON Payloads](#json-payloads) section for more details. | | `ngsi` | ✓ | object | NGSI patching for payload to be used in notifications. See [NGSI payload patching](#ngsi-payload-patching) section for more details. | -| `exprLang` | ✓ | string | Expression language to use, either `legacy` or `jexl` (default is `jexl`) | `payload`, `json` or `ngsi` cannot be used at the same time, they are mutually exclusive. diff --git a/src/lib/apiTypesV2/HttpInfo.cpp b/src/lib/apiTypesV2/HttpInfo.cpp index cc4c000283..d4eb24114c 100644 --- a/src/lib/apiTypesV2/HttpInfo.cpp +++ b/src/lib/apiTypesV2/HttpInfo.cpp @@ -106,8 +106,6 @@ std::string HttpInfo::toJson() { jh.addRaw("headers", objectToJson(headers)); } - - jh.addString("exprLang", this->exprLang); } return jh.str(); @@ -234,9 +232,6 @@ void HttpInfo::fill(const orion::BSONObj& bo) orion::BSONObj headers = getObjectFieldF(bo, CSUB_HEADERS); headers.toStringMap(&this->headers); } - - // expression language used in custom notifications - this->exprLang = bo.hasField(CSUB_EXPRLANG) ? getStringFieldF(bo, CSUB_EXPRLANG) : "legacy"; } } @@ -257,7 +252,6 @@ void HttpInfo::fill(const HttpInfo& _httpInfo) this->custom = _httpInfo.custom; this->includePayload = _httpInfo.includePayload; this->timeout = _httpInfo.timeout; - this->exprLang = _httpInfo.exprLang; this->json = _httpInfo.json == NULL? NULL : _httpInfo.json->clone(); diff --git a/src/lib/apiTypesV2/HttpInfo.h b/src/lib/apiTypesV2/HttpInfo.h index 8d32651374..0af3d273ef 100644 --- a/src/lib/apiTypesV2/HttpInfo.h +++ b/src/lib/apiTypesV2/HttpInfo.h @@ -57,8 +57,6 @@ struct HttpInfo bool includePayload; long long timeout; - std::string exprLang; - HttpInfo(); std::string toJson(); diff --git a/src/lib/apiTypesV2/MqttInfo.cpp b/src/lib/apiTypesV2/MqttInfo.cpp index c8829b0bb2..e604f21e1a 100644 --- a/src/lib/apiTypesV2/MqttInfo.cpp +++ b/src/lib/apiTypesV2/MqttInfo.cpp @@ -102,8 +102,6 @@ std::string MqttInfo::toJson() jh.addRaw("ngsi", this->ngsi.toJson(NGSI_V2_NORMALIZED, true)); break; } - - jh.addString("exprLang", this->exprLang); } return jh.str(); @@ -218,9 +216,6 @@ void MqttInfo::fill(const orion::BSONObj& bo) this->ngsi.attributeVector.fill(getObjectFieldF(ngsiObj, ENT_ATTRS)); } } - - // expression language used in custom notifications - this->exprLang = bo.hasField(CSUB_EXPRLANG) ? getStringFieldF(bo, CSUB_EXPRLANG) : "legacy"; } } @@ -243,7 +238,6 @@ void MqttInfo::fill(const MqttInfo& _mqttInfo) this->providedAuth = _mqttInfo.providedAuth; this->user = _mqttInfo.user; this->passwd = _mqttInfo.passwd; - this->exprLang = _mqttInfo.exprLang; this->json = _mqttInfo.json == NULL ? NULL : _mqttInfo.json->clone(); diff --git a/src/lib/apiTypesV2/MqttInfo.h b/src/lib/apiTypesV2/MqttInfo.h index bbc3b9260d..738e73c6a9 100644 --- a/src/lib/apiTypesV2/MqttInfo.h +++ b/src/lib/apiTypesV2/MqttInfo.h @@ -53,8 +53,6 @@ struct MqttInfo CustomPayloadType payloadType; bool includePayload; - std::string exprLang; - bool providedAuth; std::string user; std::string passwd; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 0e59f59bb6..f54f91ef7c 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -55,6 +55,7 @@ void ExprManager::init(void) + /* **************************************************************************** * * ExprManager::evaluate - diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index 5fbe2faa38..593161b202 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -1023,26 +1023,6 @@ static std::string parseNotification(ConnectionInfo* ciP, SubscriptionUpdate* su return r; } - // exprLang - Opt exprLangOpt = getStringOpt(httpCustom, "exprLang", "exprLang httpCustom notification"); - std::string exprLang = "jexl"; - - if (!exprLangOpt.ok()) - { - return badInput(ciP, exprLangOpt.error); - } - - if (exprLangOpt.given) - { - exprLang = exprLangOpt.value; - if ((exprLang != "jexl") && (exprLang != "legacy")) - { - return badInput(ciP, "not valid exprLang, valid ones are jexl or legacy"); - } - } - - subsP->notification.httpInfo.exprLang = exprLang; - subsP->notification.httpInfo.custom = true; } else if (notification.HasMember("mqtt")) @@ -1153,26 +1133,6 @@ static std::string parseNotification(ConnectionInfo* ciP, SubscriptionUpdate* su return r; } - // exprLang - Opt exprLangOpt = getStringOpt(mqttCustom, "exprLang", "exprLang mqttCustom notification"); - std::string exprLang = "jexl"; - - if (!exprLangOpt.ok()) - { - return badInput(ciP, exprLangOpt.error); - } - - if (exprLangOpt.given) - { - exprLang = exprLangOpt.value; - if ((exprLang != "jexl") && (exprLang != "legacy")) - { - return badInput(ciP, "not valid exprLang, valid ones are jexl or legacy"); - } - } - - subsP->notification.mqttInfo.exprLang = exprLang; - subsP->notification.mqttInfo.custom = true; } diff --git a/src/lib/mongoBackend/MongoCommonSubscription.cpp b/src/lib/mongoBackend/MongoCommonSubscription.cpp index a58ea5f55b..dd6bcf3101 100644 --- a/src/lib/mongoBackend/MongoCommonSubscription.cpp +++ b/src/lib/mongoBackend/MongoCommonSubscription.cpp @@ -183,9 +183,6 @@ static void setCustomHttpInfo(const HttpInfo& httpInfo, orion::BSONObjBuilder* b b->append(CSUB_NGSI, bob.obj()); } - - b->append(CSUB_EXPRLANG, httpInfo.exprLang); - LM_T(LmtMongo, ("Subscription exprLang: %s", httpInfo.exprLang.c_str())); } @@ -258,9 +255,6 @@ static void setCustomMqttInfo(const ngsiv2::MqttInfo& mqttInfo, orion::BSONObjBu b->append(CSUB_NGSI, bob.obj()); } - - b->append(CSUB_EXPRLANG, mqttInfo.exprLang); - LM_T(LmtMongo, ("Subscription exprLang: %s", mqttInfo.exprLang.c_str())); } @@ -632,7 +626,6 @@ void setOnlyChanged(const Subscription& sub, orion::BSONObjBuilder* b) } - /* **************************************************************************** * * setCovered - diff --git a/src/lib/mongoBackend/dbConstants.h b/src/lib/mongoBackend/dbConstants.h index 241507efd9..b55cf29925 100644 --- a/src/lib/mongoBackend/dbConstants.h +++ b/src/lib/mongoBackend/dbConstants.h @@ -119,7 +119,6 @@ #define CSUB_NGSI "ngsi" #define CSUB_BLACKLIST "blacklist" #define CSUB_ONLYCHANGED "onlyChanged" -#define CSUB_EXPRLANG "exprLang" #define CSUB_COVERED "covered" #define CSUB_NOTIFYONMETADATACHANGE "notifyOnMetadataChange" #define CSUB_LASTFAILURE "lastFailure" diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 2877049fd7..bd8c66738d 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -315,10 +315,9 @@ static SenderThreadParams* buildSenderParamsCustom std::map headers; Entity& en = notifyCerP->entity; - const std::string exprLang = notification.type == ngsiv2::HttpNotification ? notification.httpInfo.exprLang : notification.mqttInfo.exprLang; - // Used by several macroSubstitute() calls along this function - bool legacy = (exprLang == "legacy"); + // FIXME PR: unhardwire legacy == false + bool legacy = false; ExprContextObject exprContext(legacy); // FIXME PR: no-legacy context is now based in JsonHelper, which may mess with key repetition. Check this diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test index f1b0c00cbb..853d3e6abd 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test @@ -157,7 +157,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 713 +Content-Length: 695 [ { @@ -167,7 +167,6 @@ Content-Length: 713 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "A1": "${A1}", "A2": "${A2}", diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test deleted file mode 100644 index d624115962..0000000000 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Notification Templates - many notifications (legacy expressions - ---SHELL-INIT-- -dbInit CB -brokerStart CB -accumulatorStart --pretty-print - ---SHELL-- - -# -# 01. Create regular subscription for E1 -# 02. Create same subscription but with 'httpCustom' instead of 'http' -# 03. Create custom subscription for E1 -# 04. Create E1/T1 -# 05. Create E1/T2 -# 06. Create E1/T3 -# 07. Dump accumulator and see 9 notifications -# - -echo "01. Create regular subscription for E1" -echo "======================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1" - } - ], - "condition": { - "attrs": [] - } - }, - "notification": { - "http": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" - } - }, - "throttling": 0 -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create same subscription but with 'httpCustom' instead of 'http'" -echo "====================================================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1" - } - ], - "condition": { - "attrs": [] - } - }, - "notification": { - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" - }, - "exprLang": "legacy" - }, - "throttling": 0 -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "03. Create custom subscription for E1" -echo "=====================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1" - } - ], - "condition": { - "attrs": [] - } - }, - "notification": { - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "method": "PUT", - "payload": "{ %22A1%22: %22${A1}%22, %22A2%22: %22${A2}%22, %22A3%22: %22${A3}%22 }", - "qs": { "id": "${id}", "type": "${type}", "a1": "${A1}", "a2": "${A2}", "a3": "${A3}" }, - "headers": { "entity-id": "${id}", "entity-type": "${type}", "A1": "${A1}", "A2": "${A2}", "A3": "${A3}" }, - "exprLang": "legacy" - } - }, - "throttling": 0 -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "04. Create E1/T1" -echo "================" -payload='{ - "id": "E1", - "type": "T1", - "A1": true -}' -orionCurl --url /v2/entities?options=keyValues --payload "$payload" -echo -echo - - -echo "05. Create E1/T2" -echo "================" -payload='{ - "id": "E1", - "type": "T2", - "A2": null -}' -orionCurl --url /v2/entities?options=keyValues --payload "$payload" -echo -echo - - -echo "06. Create E1/T3" -echo "================" -payload='{ - "id": "E1", - "type": "T3", - "A3": 3 -}' -orionCurl --url /v2/entities?options=keyValues --payload "$payload" -echo -echo - - -echo "07. Dump accumulator and see 9 notifications" -echo "============================================" -sleep 0.5 # Without this sleep, sometimes the last notification is missing -accumulatorDump -echo -echo - - ---REGEXPECT-- -01. Create regular subscription for E1 -====================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -02. Create same subscription but with 'httpCustom' instead of 'http' -==================================================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -03. Create custom subscription for E1 -===================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -04. Create E1/T1 -================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T1 -Content-Length: 0 - - - -05. Create E1/T2 -================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T2 -Content-Length: 0 - - - -06. Create E1/T3 -================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T3 -Content-Length: 0 - - - -07. Dump accumulator and see 9 notifications -#SORT_START -============================================ -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 129 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A1": { - "metadata": {}, - "type": "Boolean", - "value": true - }, - "id": "E1", - "type": "T1" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 129 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A1": { - "metadata": {}, - "type": "Boolean", - "value": true - }, - "id": "E1", - "type": "T1" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 -Fiware-Servicepath: / -Entity-Id: E1 -Content-Length: 36 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -A1: true -Entity-Type: T1 -Content-Type: text/plain; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "A1": "true", - "A2": "", - "A3": "" -} -#SORT_END -#SORT_START -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 126 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A2": { - "metadata": {}, - "type": "None", - "value": null - }, - "id": "E1", - "type": "T2" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 126 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A2": { - "metadata": {}, - "type": "None", - "value": null - }, - "id": "E1", - "type": "T2" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a2=null&id=E1&type=T2 -Fiware-Servicepath: / -Entity-Id: E1 -Content-Length: 36 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -A2: null -Entity-Type: T2 -Content-Type: text/plain; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "A1": "", - "A2": "null", - "A3": "" -} -#SORT_END -#SORT_START -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 125 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A3": { - "metadata": {}, - "type": "Number", - "value": 3 - }, - "id": "E1", - "type": "T3" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 125 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A3": { - "metadata": {}, - "type": "Number", - "value": 3 - }, - "id": "E1", - "type": "T3" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 -Fiware-Servicepath: / -A3: 3 -Entity-Id: E1 -Content-Length: 33 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Entity-Type: T3 -Content-Type: text/plain; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "A1": "", - "A2": "", - "A3": "3" -} -======================================= -#SORT_END - - ---TEARDOWN-- -brokerStop CB -dbDrop CB -accumulatorStop diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test index a31092de4a..9a548cec40 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test @@ -137,7 +137,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 568 +Content-Length: 550 { "id": "REGEX([0-9a-f]{24})", @@ -146,7 +146,6 @@ Content-Length: 568 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "hs1": "1", "hs2": "${A1}" diff --git a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test index 2bdcc9c72c..7a4956a172 100644 --- a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test +++ b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test @@ -173,7 +173,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 513 +Content-Length: 495 [ { @@ -183,7 +183,6 @@ Content-Length: 513 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-Type": "application/json", "H1": "${A1}" @@ -232,7 +231,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 515 +Content-Length: 497 [ { @@ -242,7 +241,6 @@ Content-Length: 515 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-Type": "application/json", "H2": "${A2}" diff --git a/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test b/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test index 5969637f5d..57a7a22c99 100644 --- a/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test +++ b/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test @@ -142,7 +142,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 304 +Content-Length: 286 [ { @@ -152,7 +152,6 @@ Content-Length: 304 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "method": "PUT", "url": "${abc}" }, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test index 89828afff7..c50d612856 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test @@ -254,7 +254,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 343 +Content-Length: 325 { "id": "REGEX([0-9a-f]{24})", @@ -263,7 +263,6 @@ Content-Length: 343 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "x1": 1, "x2": 2 @@ -302,7 +301,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -311,7 +310,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -347,7 +345,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 339 +Content-Length: 321 { "id": "REGEX([0-9a-f]{24})", @@ -356,7 +354,6 @@ Content-Length: 339 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "yy", "zz" @@ -395,7 +392,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 335 +Content-Length: 317 { "id": "REGEX([0-9a-f]{24})", @@ -404,7 +401,6 @@ Content-Length: 335 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ 1, 2, @@ -444,7 +440,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -453,7 +449,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -491,7 +486,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -500,7 +495,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -536,7 +530,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 352 +Content-Length: 334 { "id": "REGEX([0-9a-f]{24})", @@ -545,7 +539,6 @@ Content-Length: 352 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "z4": true, "z5": [ @@ -579,7 +572,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 691 +Content-Length: 655 [ { @@ -589,7 +582,6 @@ Content-Length: 691 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -616,7 +608,6 @@ Content-Length: 691 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "z4": true, "z5": [ diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test index d49ac8a108..2bb34d3d32 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test @@ -180,7 +180,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 916 +Content-Length: 880 [ { @@ -190,7 +190,6 @@ Content-Length: 916 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "array": [ "22", @@ -246,7 +245,6 @@ Content-Length: 916 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "foo", 10, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test index e0a2d267ec..08f26a3682 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 838 +Content-Length: 802 [ { @@ -197,7 +197,6 @@ Content-Length: 838 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "array": "${array}", "bool": "${bool}", @@ -231,7 +230,6 @@ Content-Length: 838 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "${text}", "${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test index e1b86b3f9f..758775b4e3 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 851 +Content-Length: 815 [ { @@ -197,7 +197,6 @@ Content-Length: 851 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "A": "ns:${text}", "B": "ns:${number}", @@ -231,7 +230,6 @@ Content-Length: 851 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "ns:${text}", "ns:${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test index 191994521c..95f7928e31 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 957 +Content-Length: 921 [ { @@ -197,7 +197,6 @@ Content-Length: 957 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "A": "ns:${text}:${object}", "B": "ns:${number}:${array}", @@ -231,7 +230,6 @@ Content-Length: 957 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "ns:${text}:${object}", "ns:${number}:${array}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test index 568d36fc19..7a19421ab8 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test @@ -157,7 +157,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 751 +Content-Length: 715 [ { @@ -167,7 +167,6 @@ Content-Length: 751 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "x": { "x1": "${A}", @@ -203,7 +202,6 @@ Content-Length: 751 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "22", { diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test index dccebf8230..0c24073d89 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test @@ -261,7 +261,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 376 +Content-Length: 358 { "id": "REGEX([0-9a-f]{24})", @@ -270,7 +270,6 @@ Content-Length: 376 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "x1": 1, "x2": 2 @@ -312,7 +311,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -321,7 +320,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -360,7 +358,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 372 +Content-Length: 354 { "id": "REGEX([0-9a-f]{24})", @@ -369,7 +367,6 @@ Content-Length: 372 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "yy", "zz" @@ -411,7 +408,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 368 +Content-Length: 350 { "id": "REGEX([0-9a-f]{24})", @@ -420,7 +417,6 @@ Content-Length: 368 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ 1, 2, @@ -463,7 +459,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -472,7 +468,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -513,7 +508,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -522,7 +517,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -561,7 +555,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 385 +Content-Length: 367 { "id": "REGEX([0-9a-f]{24})", @@ -570,7 +564,6 @@ Content-Length: 385 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "z4": true, "z5": [ @@ -607,7 +600,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 757 +Content-Length: 721 [ { @@ -617,7 +610,6 @@ Content-Length: 757 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -647,7 +639,6 @@ Content-Length: 757 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "z4": true, "z5": [ diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test index e84b665130..9428ed0e64 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test @@ -186,7 +186,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 982 +Content-Length: 946 [ { @@ -196,7 +196,6 @@ Content-Length: 982 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "array": [ "22", @@ -255,7 +254,6 @@ Content-Length: 982 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "foo", 10, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test index 993180a8ce..954ed3bdba 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test @@ -193,7 +193,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 904 +Content-Length: 868 [ { @@ -203,7 +203,6 @@ Content-Length: 904 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "array": "${array}", "bool": "${bool}", @@ -240,7 +239,6 @@ Content-Length: 904 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "${text}", "${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test index 7133b2e675..3a6f4a15c5 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test @@ -192,7 +192,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 917 +Content-Length: 881 [ { @@ -202,7 +202,6 @@ Content-Length: 917 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "A": "ns:${text}", "B": "ns:${number}", @@ -239,7 +238,6 @@ Content-Length: 917 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "ns:${text}", "ns:${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test index 58c287e5d5..940492dc04 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test @@ -193,7 +193,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1023 +Content-Length: 987 [ { @@ -203,7 +203,6 @@ Content-Length: 1023 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "A": "ns:${text}:${object}", "B": "ns:${number}:${array}", @@ -240,7 +239,6 @@ Content-Length: 1023 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "ns:${text}:${object}", "ns:${number}:${array}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test index 2fc44f8b6b..020245e27b 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test @@ -163,7 +163,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 817 +Content-Length: 781 [ { @@ -173,7 +173,6 @@ Content-Length: 817 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "x": { "x1": "${A}", @@ -212,7 +211,6 @@ Content-Length: 817 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "22", { diff --git a/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test b/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test index 6fa972f30b..2fe079ec78 100644 --- a/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test +++ b/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test @@ -373,7 +373,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 582 +Content-Length: 564 { "id": "REGEX([0-9a-f]{24})", @@ -382,7 +382,6 @@ Content-Length: 582 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-type": "application/json" }, @@ -416,7 +415,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 582 +Content-Length: 564 { "id": "REGEX([0-9a-f]{24})", @@ -425,7 +424,6 @@ Content-Length: 582 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-type": "application/json" }, @@ -459,7 +457,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 582 +Content-Length: 564 { "id": "REGEX([0-9a-f]{24})", @@ -468,7 +466,6 @@ Content-Length: 582 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-type": "application/json" }, diff --git a/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test b/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test index 12727d8b09..ffaffdb044 100644 --- a/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test +++ b/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test @@ -128,7 +128,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 584 +Content-Length: 566 [ { @@ -138,7 +138,6 @@ Content-Length: 584 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-type": "application/json" }, diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test index 36ca87149f..d482101617 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test @@ -282,7 +282,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 2536 +Content-Length: 2428 [ { @@ -295,7 +295,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "temperature:${temperature}", "qos": 0, "retain": false, @@ -328,7 +327,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "temperature:${temperature}", "qos": 2, "retain": false, @@ -361,7 +359,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "/orion/${dest}", @@ -393,7 +390,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 2, "retain": false, "topic": "/orion/${dest}", @@ -425,7 +421,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": null, "qos": 0, "retain": false, @@ -458,7 +453,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": null, "qos": 2, "retain": false, diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test index 1d65654747..531e251861 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test @@ -204,7 +204,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 407 +Content-Length: 389 { "description": "Original sub", @@ -214,7 +214,6 @@ Content-Length: 407 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "t:${t}", "qos": 2, "retain": false, @@ -253,7 +252,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 407 +Content-Length: 389 { "description": "Modified sub", @@ -263,7 +262,6 @@ Content-Length: 407 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "t:${t}", "qos": 2, "retain": false, @@ -302,7 +300,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 400 +Content-Length: 382 { "description": "Modified sub", @@ -312,7 +310,6 @@ Content-Length: 400 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": null, "qos": 0, "retain": false, @@ -395,7 +392,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 390 +Content-Length: 372 { "description": "Modified sub", @@ -405,7 +402,6 @@ Content-Length: 390 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 1, "retain": false, "topic": "/orionbk/${d}", diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test b/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test index 597cfa5457..ecc29badba 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test @@ -245,7 +245,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 527 +Content-Length: 509 [ { @@ -258,7 +258,6 @@ Content-Length: 527 "lastNotification": "REGEX(.*)", "lastSuccess": "REGEX(.*)", "mqttCustom": { - "exprLang": "jexl", "payload": "{ %22A%22: %22${A}%22 }", "qos": 0, "retain": false, diff --git a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test b/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test index 9e63211726..b4e9cc5695 100644 --- a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test +++ b/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test @@ -159,7 +159,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1048 +Content-Length: 1012 [ { @@ -171,7 +171,6 @@ Content-Length: 1048 "attrsFormat": "legacy", "covered": false, "httpCustom": { - "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "lastNotification": "REGEX(.*)", @@ -202,7 +201,6 @@ Content-Length: 1048 "attrsFormat": "legacy", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "{ %22A1%22: %22Value of A1: ${A1}%22 }", "url": "http://localhost:9997/notify" }, diff --git a/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test b/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test index 13e3163639..0955bd6c86 100644 --- a/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test +++ b/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test @@ -189,7 +189,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1272 +Content-Length: 1200 [ { @@ -199,7 +199,6 @@ Content-Length: 1272 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "abc", "url": "http://localhost:9997/notify" }, @@ -225,7 +224,6 @@ Content-Length: 1272 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "onlyChangedAttrs": false @@ -250,7 +248,6 @@ Content-Length: 1272 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": null, "url": "http://localhost:9997/notify" }, @@ -276,7 +273,6 @@ Content-Length: 1272 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "onlyChangedAttrs": false diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test deleted file mode 100644 index 8321f78322..0000000000 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test +++ /dev/null @@ -1,217 +0,0 @@ -# Copyright 2022 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Covered custom notifictions (legacy expressions) - ---SHELL-INIT-- -dbInit CB -brokerStart CB -accumulatorStart - ---SHELL-- - -# -# 01. Create covered custom subscriptions for E1 covering attributes A1 and A2 -# 02. Create not covered custom subscriptions for E2 -# 03. Create E1 with attribute A1=1 -# 04. Dump & reset, see notifications A1=1 and A2=null in custom payload -# 05. Create E2 with attribute A1=1 -# 06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) -# - -echo "01. Create covered custom subscriptions for E1 covering attributes A1 and A2" -echo "============================================================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1", - "type": "T" - } - ] - }, - "notification": { - "covered": true, - "attrs": [ "A1", "A2" ], - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "payload": "{ %22A1-value%22: ${A1}, %22A2-value%22: ${A2} }", - "headers": { - "content-type": "application/json%3B charset%3Dutf-8" - }, - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create not covered custom subscriptions for E2" -echo "==================================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E2", - "type": "T" - } - ] - }, - "notification": { - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "payload": "{ %22A1-value%22: ${A1}, %22A2-value%22: ${A2} }", - "headers": { - "content-type": "application/json%3B charset%3Dutf-8" - } - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "03. Create E1 with attribute A1=1" -echo "=================================" -payload='{ - "id": "E1", - "type": "T", - "A1": { - "value": 1, - "type": "Number" - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "04. Dump & reset, see notifications A1=1 and A2=null in custom payload" -echo "======================================================================" -accumulatorDump -accumulatorReset -echo -echo - - -echo "05. Create E2 with attribute A1=1" -echo "=================================" -payload='{ - "id": "E2", - "type": "T", - "A1": { - "value": 1, - "type": "Number" - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON)" -echo "========================================================================================" -accumulatorDump -accumulatorReset -echo -echo - - ---REGEXPECT-- -01. Create covered custom subscriptions for E1 covering attributes A1 and A2 -============================================================================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -02. Create not covered custom subscriptions for E2 -================================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -03. Create E1 with attribute A1=1 -================================= -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T -Content-Length: 0 - - - -04. Dump & reset, see notifications A1=1 and A2=null in custom payload -====================================================================== -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 35 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ "A1-value": 1, "A2-value": null }======================================= - - -05. Create E2 with attribute A1=1 -================================= -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E2?type=T -Content-Length: 0 - - - -06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) -======================================================================================== -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 31 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ "A1-value": 1, "A2-value": }======================================= - - ---TEARDOWN-- -brokerStop CB -dbDrop CB -accumulatorStop diff --git a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test index badbe98993..d874dbefdd 100644 --- a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test +++ b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test @@ -327,7 +327,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 379 +Content-Length: 361 { "description": "HTTP sub", @@ -337,7 +337,6 @@ Content-Length: 379 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 1000, "url": "http://localhost:1234" @@ -374,7 +373,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 379 +Content-Length: 361 { "description": "HTTP sub", @@ -384,7 +383,6 @@ Content-Length: 379 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 2000, "url": "http://localhost:1234" @@ -421,7 +419,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 364 +Content-Length: 346 { "description": "HTTP sub", @@ -431,7 +429,6 @@ Content-Length: 364 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "url": "http://localhost:1234" }, @@ -467,7 +464,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 379 +Content-Length: 361 { "description": "HTTP sub", @@ -477,7 +474,6 @@ Content-Length: 379 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 1000, "url": "http://localhost:1234" @@ -524,7 +520,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 364 +Content-Length: 346 { "description": "HTTP sub", @@ -534,7 +530,6 @@ Content-Length: 364 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "url": "http://localhost:1234" }, diff --git a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test index bbdcdaebb0..0937cb5cec 100644 --- a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test +++ b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test @@ -175,7 +175,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 514 +Content-Length: 496 { "description": "HTTP sub", @@ -185,7 +185,6 @@ Content-Length: 514 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 12000, "url": "http://localhost:9997/noresponse" @@ -218,7 +217,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 550 +Content-Length: 532 { "description": "HTTP sub", @@ -229,7 +228,6 @@ Content-Length: 550 "covered": false, "failsCounter": 1, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 8000, "url": "http://localhost:9997/noresponse" diff --git a/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test b/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test index c50332dc8f..59a65b45f0 100644 --- a/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test +++ b/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test @@ -317,7 +317,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 384 +Content-Length: 366 { "id": "REGEX([0-9a-f\-]{24})", @@ -326,7 +326,6 @@ Content-Length: 384 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, @@ -372,7 +371,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 385 +Content-Length: 367 { "id": "REGEX([0-9a-f\-]{24})", @@ -381,7 +380,6 @@ Content-Length: 385 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, @@ -427,7 +425,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 352 +Content-Length: 334 { "id": "REGEX([0-9a-f\-]{24})", @@ -436,7 +434,6 @@ Content-Length: 352 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "/orion", @@ -480,7 +477,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 384 +Content-Length: 366 { "id": "REGEX([0-9a-f\-]{24})", @@ -489,7 +486,6 @@ Content-Length: 384 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test deleted file mode 100644 index 656e55da9b..0000000000 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test +++ /dev/null @@ -1,664 +0,0 @@ -# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Basic CRUD for exprLang field (http variant) - ---SHELL-INIT-- -dbInit CB -brokerStart CB 0-255 - ---SHELL-- - -# -# 01. Create subscription with invalid exprLang, see error -# 02. Create subscription with exprLang jexl -# 03a. Get subscription and see exprLang jexl -# 03b. Get subscription (as a list) and see exprLang jexl -# 04. Update subscription with exprLang legacy -# 05a. Get subscription and see exprLang legacy -# 05b. Get subscription (as a list) and see exprLang legacy -# 06. Update subscription without exprLang -# 07a. Get subscription and see exprLang jexl -# 07b. Get subscription (as a list) and see exprLang jexl -# 08. Update subscription with exprLang legacy -# 09a. Get subscription and see exprLang legacy -# 09b. Get subscription (as a list) and see exprLang legacy -# 10. Update subscription with invalid exprLang, see error -# 11a. Get subscription and see exprLang legacy -# 11b. Get subscription (as a list) and see exprLang legacy -# - -echo "01. Create subscription with invalid exprLang, see error" -echo "========================================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern" : ".*", - "type": "T" - } - ] - }, - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": "foo" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create subscription with exprLang jexl" -echo "==========================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern" : ".*", - "type": "T" - } - ] - }, - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": "jexl" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") - - -echo "03a. Get subscription and see exprLang jexl" -echo "===========================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "03b. Get subscription (as a list) and see exprLang jexl" -echo "=======================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "04. Update subscription with exprLang legacy" -echo "============================================" -payload='{ - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "05a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "05b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "06. Update subscription without exprLang" -echo "========================================" -payload='{ - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "topic": "orion" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "07a. Get subscription and see exprLang jexl" -echo "===========================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "07b. Get subscription (as a list) and see exprLang jexl" -echo "=======================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "08. Update subscription with exprLang legacy" -echo "============================================" -payload='{ - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "09a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "09b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "10. Update subscription with invalid exprLang, see error" -echo "========================================================" -payload='{ - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": 42 - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "11a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "11b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - ---REGEXPECT-- -01. Create subscription with invalid exprLang, see error -======================================================== -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 88 - -{ - "description": "not valid exprLang, valid ones are jexl or legacy", - "error": "BadRequest" -} - - -02. Create subscription with exprLang jexl -========================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -03a. Get subscription and see exprLang jexl -=========================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 320 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "jexl", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -03b. Get subscription (as a list) and see exprLang jexl -======================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "jexl", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -04. Update subscription with exprLang legacy -============================================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -05a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -05b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 324 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -06. Update subscription without exprLang -======================================== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -07a. Get subscription and see exprLang jexl -=========================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 320 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "jexl", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -07b. Get subscription (as a list) and see exprLang jexl -======================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "jexl", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -08. Update subscription with exprLang legacy -============================================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -09a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -09b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 324 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -10. Update subscription with invalid exprLang, see error -======================================================== -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 87 - -{ - "description": "exprLang httpCustom notification is not a string", - "error": "BadRequest" -} - - -11a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -11b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 324 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test index ac53119941..79ecfd9eeb 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -31,17 +31,15 @@ accumulatorStart --pretty-print --SHELL-- # -# Same as legacy_expr_attrs_weird_syntax.test, but using jexl expression language -# -# 01. 01. Create custom sub with custom expression: R1=A-B and R2=A:B +# 01. Create custom sub with custom expression: A-B and A:B # 02. Create entity E1 with A-B 1 and A:B 2 # 03. Update entity E1 with A-B 3 and A:B 4 # 04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null) # -echo "01. Create custom sub with custom expression: R1=A-B and R2=A:B" -echo "===============================================================" +echo "01. Create custom sub with custom expression: A-B and A:B" +echo "=========================================================" payload='{ "subject": { "entities": [ @@ -117,8 +115,8 @@ echo --REGEXPECT-- -01. Create custom sub with custom expression: R1=A-B and R2=A:B -=============================================================== +01. Create custom sub with custom expression: A-B and A:B +========================================================= HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test deleted file mode 100644 index 15da6fc5bb..0000000000 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Legacy expression using attributes with weird syntax - ---SHELL-INIT-- -dbInit CB -brokerStart CB -accumulatorStart --pretty-print - ---SHELL-- - -# -# Same as jexl_expr_attrs_weird_syntax.test, but using legacy expression language -# -# 01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B -# 02. Create entity E1 with A-B 1 and A:B 2 -# 03. Update entity E1 with A-B 3 and A:B 4 -# 04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4) -# - - -echo "01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B" -echo "======================================================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1", - "type": "T" - } - ] - }, - "notification": { - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "exprLang": "legacy", - "ngsi": { - "R1": { - "value": "${A-B}", - "type": "Calculated" - }, - "R2": { - "value": "${A:B}", - "type": "Calculated" - } - } - }, - "attrs": [ "R1", "R2" ] - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create entity E1 with A-B 1 and A:B 2" -echo "=========================================" -payload='{ - "id": "E1", - "type": "T", - "A-B": { - "value": 1, - "type": "Number" - }, - "A:B": { - "value": 2, - "type": "Number" - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "03. Update entity E1 with A-B 3 and A:B 4" -echo "=========================================" -payload='{ - "A-B": { - "value": 3, - "type": "Number" - }, - "A:B": { - "value": 4, - "type": "Number" - } -}' -orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" -echo -echo - - -echo "04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4)" -echo "======================================================================" -accumulatorDump -echo -echo - - ---REGEXPECT-- -01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B -====================================================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -02. Create entity E1 with A-B 1 and A:B 2 -========================================= -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T -Content-Length: 0 - - - -03. Update entity E1 with A-B 3 and A:B 4 -========================================= -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4) -====================================================================== -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 179 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "data": [ - { - "R1": { - "metadata": {}, - "type": "Calculated", - "value": 1 - }, - "R2": { - "metadata": {}, - "type": "Calculated", - "value": 2 - }, - "id": "E1", - "type": "T" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 179 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "data": [ - { - "R1": { - "metadata": {}, - "type": "Calculated", - "value": 3 - }, - "R2": { - "metadata": {}, - "type": "Calculated", - "value": 4 - }, - "id": "E1", - "type": "T" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - ---TEARDOWN-- -brokerStop CB -dbDrop CB -accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test deleted file mode 100644 index b155766fea..0000000000 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test +++ /dev/null @@ -1,699 +0,0 @@ -# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Basic CRUD for exprLang field (mqtt variant) - ---SHELL-INIT-- -dbInit CB -brokerStart CB 0-255 - ---SHELL-- - -# -# 01. Create subscription with invalid exprLang, see error -# 02. Create subscription with exprLang jexl -# 03a. Get subscription and see exprLang jexl -# 03b. Get subscription (as a list) and see exprLang jexl -# 04. Update subscription with exprLang legacy -# 05a. Get subscription and see exprLang legacy -# 05b. Get subscription (as a list) and see exprLang legacy -# 06. Update subscription without exprLang -# 07a. Get subscription and see exprLang jexl -# 07b. Get subscription (as a list) and see exprLang jexl -# 08. Update subscription with exprLang legacy -# 09a. Get subscription and see exprLang legacy -# 09b. Get subscription (as a list) and see exprLang legacy -# 10. Update subscription with invalid exprLang, see error -# 11a. Get subscription and see exprLang legacy -# 11b. Get subscription (as a list) and see exprLang legacy -# - -echo "01. Create subscription with invalid exprLang, see error" -echo "========================================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern" : ".*", - "type": "T" - } - ] - }, - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": "foo" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create subscription with exprLang jexl" -echo "==========================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern" : ".*", - "type": "T" - } - ] - }, - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": "jexl" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") - - -echo "03a. Get subscription and see exprLang jexl" -echo "===========================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "03b. Get subscription (as a list) and see exprLang jexl" -echo "=======================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "04. Update subscription with exprLang legacy" -echo "============================================" -payload='{ - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "05a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "05b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "06. Update subscription without exprLang" -echo "========================================" -payload='{ - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "07a. Get subscription and see exprLang jexl" -echo "===========================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "07b. Get subscription (as a list) and see exprLang jexl" -echo "=======================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "08. Update subscription with exprLang legacy" -echo "============================================" -payload='{ - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "09a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "09b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "10. Update subscription with invalid exprLang, see error" -echo "========================================================" -payload='{ - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": 42 - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "11a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "11b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - ---REGEXPECT-- -01. Create subscription with invalid exprLang, see error -======================================================== -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 88 - -{ - "description": "not valid exprLang, valid ones are jexl or legacy", - "error": "BadRequest" -} - - -02. Create subscription with exprLang jexl -========================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -03a. Get subscription and see exprLang jexl -=========================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 359 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "jexl", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -03b. Get subscription (as a list) and see exprLang jexl -======================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "jexl", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -04. Update subscription with exprLang legacy -============================================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -05a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -05b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 363 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -06. Update subscription without exprLang -======================================== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -07a. Get subscription and see exprLang jexl -=========================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 359 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "jexl", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -07b. Get subscription (as a list) and see exprLang jexl -======================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "jexl", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -08. Update subscription with exprLang legacy -============================================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -09a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -09b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 363 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -10. Update subscription with invalid exprLang, see error -======================================================== -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 87 - -{ - "description": "exprLang mqttCustom notification is not a string", - "error": "BadRequest" -} - - -11a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -11b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 363 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test index ed8fee4246..4f2b9778b6 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test @@ -110,7 +110,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 585 +Content-Length: 567 { "id": "REGEX([0-9a-f]{24})", @@ -119,7 +119,6 @@ Content-Length: 585 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "SomeType", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test index 71d19596e5..9d671877f0 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test @@ -275,7 +275,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 383 +Content-Length: 365 { "id": "REGEX([0-9a-f]{24})", @@ -284,7 +284,6 @@ Content-Length: 383 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -327,7 +326,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -336,7 +335,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -372,7 +370,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 383 +Content-Length: 365 { "id": "REGEX([0-9a-f]{24})", @@ -381,7 +379,6 @@ Content-Length: 383 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -424,7 +421,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 383 +Content-Length: 365 { "id": "REGEX([0-9a-f]{24})", @@ -433,7 +430,6 @@ Content-Length: 383 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "C": { "type": "Number", @@ -476,7 +472,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -485,7 +481,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -523,7 +518,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -532,7 +527,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -568,7 +562,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 383 +Content-Length: 365 { "id": "REGEX([0-9a-f]{24})", @@ -577,7 +571,6 @@ Content-Length: 383 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", @@ -612,7 +605,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 722 +Content-Length: 686 [ { @@ -622,7 +615,6 @@ Content-Length: 722 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -649,7 +641,6 @@ Content-Length: 722 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test index 3ee0ec39fb..a34bfa6d96 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test @@ -293,7 +293,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 432 +Content-Length: 414 { "id": "REGEX([0-9a-f]{24})", @@ -302,7 +302,6 @@ Content-Length: 432 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "StructuredValue", @@ -354,7 +353,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -363,7 +362,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -399,7 +397,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 433 +Content-Length: 415 { "id": "REGEX([0-9a-f]{24})", @@ -408,7 +406,6 @@ Content-Length: 433 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "StructuredValue", @@ -464,7 +461,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 429 +Content-Length: 411 { "id": "REGEX([0-9a-f]{24})", @@ -473,7 +470,6 @@ Content-Length: 429 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "C": { "type": "StructuredValue", @@ -525,7 +521,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -534,7 +530,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -572,7 +567,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -581,7 +576,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -617,7 +611,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 435 +Content-Length: 417 { "id": "REGEX([0-9a-f]{24})", @@ -626,7 +620,6 @@ Content-Length: 435 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "StructuredValue", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test index d6330c8c1f..2e60112a6e 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test @@ -259,7 +259,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 371 +Content-Length: 353 { "id": "REGEX([0-9a-f]{24})", @@ -268,7 +268,6 @@ Content-Length: 371 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -310,7 +309,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -319,7 +318,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -355,7 +353,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 343 { "id": "REGEX([0-9a-f]{24})", @@ -364,7 +362,6 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -405,7 +402,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 333 { "id": "REGEX([0-9a-f]{24})", @@ -414,7 +411,6 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "id": "E3", "type": "T3" @@ -453,7 +449,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -462,7 +458,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -500,7 +495,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -509,7 +504,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -545,7 +539,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 373 +Content-Length: 355 { "id": "REGEX([0-9a-f]{24})", @@ -554,7 +548,6 @@ Content-Length: 373 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test index 0f62f2759f..8043fd7c4f 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test @@ -111,7 +111,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 618 +Content-Length: 600 { "id": "REGEX([0-9a-f]{24})", @@ -120,7 +120,6 @@ Content-Length: 618 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "SomeType", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test index d8421cc232..c08b1ab6ca 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test @@ -282,7 +282,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 416 +Content-Length: 398 { "id": "REGEX([0-9a-f]{24})", @@ -291,7 +291,6 @@ Content-Length: 416 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -337,7 +336,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -346,7 +345,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -385,7 +383,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 416 +Content-Length: 398 { "id": "REGEX([0-9a-f]{24})", @@ -394,7 +392,6 @@ Content-Length: 416 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -440,7 +437,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 416 +Content-Length: 398 { "id": "REGEX([0-9a-f]{24})", @@ -449,7 +446,6 @@ Content-Length: 416 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "C": { "type": "Number", @@ -495,7 +491,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -504,7 +500,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -545,7 +540,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -554,7 +549,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -593,7 +587,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 416 +Content-Length: 398 { "id": "REGEX([0-9a-f]{24})", @@ -602,7 +596,6 @@ Content-Length: 416 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", @@ -640,7 +633,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 788 +Content-Length: 752 [ { @@ -650,7 +643,6 @@ Content-Length: 788 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -680,7 +672,6 @@ Content-Length: 788 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test index 2eaf4e7c1e..fcbbb8f196 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test @@ -300,7 +300,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 465 +Content-Length: 447 { "id": "REGEX([0-9a-f]{24})", @@ -309,7 +309,6 @@ Content-Length: 465 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "StructuredValue", @@ -364,7 +363,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -373,7 +372,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -412,7 +410,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 466 +Content-Length: 448 { "id": "REGEX([0-9a-f]{24})", @@ -421,7 +419,6 @@ Content-Length: 466 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "StructuredValue", @@ -480,7 +477,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 462 +Content-Length: 444 { "id": "REGEX([0-9a-f]{24})", @@ -489,7 +486,6 @@ Content-Length: 462 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "C": { "type": "StructuredValue", @@ -544,7 +540,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -553,7 +549,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -594,7 +589,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -603,7 +598,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -642,7 +636,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 468 +Content-Length: 450 { "id": "REGEX([0-9a-f]{24})", @@ -651,7 +645,6 @@ Content-Length: 468 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "StructuredValue", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test index 0ea36db1d8..4ed1f9ce81 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test @@ -266,7 +266,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 404 +Content-Length: 386 { "id": "REGEX([0-9a-f]{24})", @@ -275,7 +275,6 @@ Content-Length: 404 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -320,7 +319,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -329,7 +328,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -368,7 +366,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 394 +Content-Length: 376 { "id": "REGEX([0-9a-f]{24})", @@ -377,7 +375,6 @@ Content-Length: 394 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -421,7 +418,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 384 +Content-Length: 366 { "id": "REGEX([0-9a-f]{24})", @@ -430,7 +427,6 @@ Content-Length: 384 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "id": "E3", "type": "T3" @@ -472,7 +468,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -481,7 +477,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -522,7 +517,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -531,7 +526,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -570,7 +564,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 406 +Content-Length: 388 { "id": "REGEX([0-9a-f]{24})", @@ -579,7 +573,6 @@ Content-Length: 406 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test b/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test index 4054d27904..9aff1e0a20 100644 --- a/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test +++ b/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test @@ -262,7 +262,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 358 +Content-Length: 340 { "id": "REGEX([0-9a-f]{24})", @@ -271,7 +271,6 @@ Content-Length: 358 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -301,7 +300,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 360 +Content-Length: 342 [ { @@ -311,7 +310,6 @@ Content-Length: 360 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -350,7 +348,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 359 +Content-Length: 341 { "id": "REGEX([0-9a-f]{24})", @@ -359,7 +357,6 @@ Content-Length: 359 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -389,7 +386,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 343 [ { @@ -399,7 +396,6 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -438,7 +434,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 359 +Content-Length: 341 { "id": "REGEX([0-9a-f]{24})", @@ -447,7 +443,6 @@ Content-Length: 359 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -477,7 +472,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 343 [ { @@ -487,7 +482,6 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -526,7 +520,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 358 +Content-Length: 340 { "id": "REGEX([0-9a-f]{24})", @@ -535,7 +529,6 @@ Content-Length: 358 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -565,7 +558,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 360 +Content-Length: 342 [ { @@ -575,7 +568,6 @@ Content-Length: 360 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -620,7 +612,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 358 +Content-Length: 340 { "id": "REGEX([0-9a-f]{24})", @@ -629,7 +621,6 @@ Content-Length: 358 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -659,7 +650,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 360 +Content-Length: 342 [ { @@ -669,7 +660,6 @@ Content-Length: 360 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", diff --git a/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test b/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test index 191958e178..8352a77b7b 100644 --- a/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test +++ b/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test @@ -146,7 +146,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 542 +Content-Length: 524 { "description": "DateTime test", @@ -159,7 +159,6 @@ Content-Length: 542 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "dateobservedto": { "type": "DateTime", @@ -206,7 +205,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 546 +Content-Length: 528 { "description": "DateTime test", @@ -219,7 +218,6 @@ Content-Length: 546 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "dateobservedto2": { "type": "DateTime", From 48814b71055bc883c8cce85e0701c2a277f7da60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 10:52:33 +0200 Subject: [PATCH 066/104] FIX legacy term changed to basic --- doc/manuals/admin/statistics.md | 8 +++--- src/lib/common/macroSubstitute.cpp | 8 +++--- src/lib/common/statistics.cpp | 16 +++++------ src/lib/common/statistics.h | 42 ++++++++++++++--------------- src/lib/expressions/ExprContext.cpp | 34 +++++++++++------------ src/lib/expressions/ExprContext.h | 10 +++---- src/lib/expressions/ExprManager.cpp | 10 +++---- src/lib/ngsiNotify/Notifier.cpp | 18 ++++++------- src/lib/rest/rest.cpp | 4 +-- 9 files changed, 75 insertions(+), 75 deletions(-) diff --git a/doc/manuals/admin/statistics.md b/doc/manuals/admin/statistics.md index a2760412f2..7a0eb4f487 100644 --- a/doc/manuals/admin/statistics.md +++ b/doc/manuals/admin/statistics.md @@ -133,8 +133,8 @@ Provides timing information, i.e. the time that CB passes executing in different "mongoReadWait": 4656.924425628, "mongoWriteWait": 259.347915990, "mongoCommandWait": 0.514811318, - "exprLegacyCtxBld": FIXME PR, - "exprLegacyEval": FIXME PR, + "exprBasicCtxBld": FIXME PR, + "exprBasicEval": FIXME PR, "exprJexlCtxBld": FIXME PR, "exprJexlEval": FIXME PR, "render": 108.162782114, @@ -144,8 +144,8 @@ Provides timing information, i.e. the time that CB passes executing in different "mongoBackend": 0.014752309, "mongoReadWait": 0.012018445, "mongoWriteWait": 0.000574611, - "exprLegacyCtxBld": FIXME PR, - "exprLegacyEval": FIXME PR, + "exprBasicCtxBld": FIXME PR, + "exprBasicEval": FIXME PR, "exprJexlCtxBld": FIXME PR, "exprJexlEval": FIXME PR, "render": 0.000019136, diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index cc994b70cd..28dd9c156a 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -61,8 +61,8 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e } else { - // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. - if (exprContextObjectP->isLegacy()) + // in basic mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. + if (exprContextObjectP->isBasic()) { result = removeQuotes(r.toString()); } @@ -114,8 +114,8 @@ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, c { std::string s = r.toString(); - // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. - if (exprContextObjectP->isLegacy()) + // in basic mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. + if (exprContextObjectP->isBasic()) { s = removeQuotes(s); } diff --git a/src/lib/common/statistics.cpp b/src/lib/common/statistics.cpp index 1eb5dde069..10dd8b1631 100644 --- a/src/lib/common/statistics.cpp +++ b/src/lib/common/statistics.cpp @@ -195,8 +195,8 @@ std::string renderTimingStatistics(void) bool accMongoReadWaitTime = (accTimeStat.mongoReadWaitTime.tv_sec != 0) || (accTimeStat.mongoReadWaitTime.tv_nsec != 0); bool accMongoWriteWaitTime = (accTimeStat.mongoWriteWaitTime.tv_sec != 0) || (accTimeStat.mongoWriteWaitTime.tv_nsec != 0); bool accMongoCommandWaitTime = (accTimeStat.mongoCommandWaitTime.tv_sec != 0) || (accTimeStat.mongoCommandWaitTime.tv_nsec != 0); - bool accExprLegacyCtxBldTime = (accTimeStat.exprLegacyCtxBldTime.tv_sec != 0) || (accTimeStat.exprLegacyCtxBldTime.tv_nsec != 0); - bool accExprLegacyEvalTime = (accTimeStat.exprLegacyEvalTime.tv_sec != 0) || (accTimeStat.exprLegacyEvalTime.tv_nsec != 0); + bool accExprBasicCtxBldTime = (accTimeStat.exprBasicCtxBldTime.tv_sec != 0) || (accTimeStat.exprBasicCtxBldTime.tv_nsec != 0); + bool accExprBasicEvalTime = (accTimeStat.exprBasicEvalTime.tv_sec != 0) || (accTimeStat.exprBasicEvalTime.tv_nsec != 0); bool accExprJexlCtxBldTime = (accTimeStat.exprJexlCtxBldTime.tv_sec != 0) || (accTimeStat.exprJexlCtxBldTime.tv_nsec != 0); bool accExprJexlEvalTime = (accTimeStat.exprJexlEvalTime.tv_sec != 0) || (accTimeStat.exprJexlEvalTime.tv_nsec != 0); bool accRenderTime = (accTimeStat.renderTime.tv_sec != 0) || (accTimeStat.renderTime.tv_nsec != 0); @@ -208,8 +208,8 @@ std::string renderTimingStatistics(void) bool lastMongoReadWaitTime = (lastTimeStat.mongoReadWaitTime.tv_sec != 0) || (lastTimeStat.mongoReadWaitTime.tv_nsec != 0); bool lastMongoWriteWaitTime = (lastTimeStat.mongoWriteWaitTime.tv_sec != 0) || (lastTimeStat.mongoWriteWaitTime.tv_nsec != 0); bool lastMongoCommandWaitTime = (lastTimeStat.mongoCommandWaitTime.tv_sec != 0) || (lastTimeStat.mongoCommandWaitTime.tv_nsec != 0); - bool lastExprLegacyCtxBldTime = (lastTimeStat.exprLegacyCtxBldTime.tv_sec != 0) || (lastTimeStat.exprLegacyCtxBldTime.tv_nsec != 0); - bool lastExprLegacyEvalTime = (lastTimeStat.exprLegacyEvalTime.tv_sec != 0) || (lastTimeStat.exprLegacyEvalTime.tv_nsec != 0); + bool lastExprBasicCtxBldTime = (lastTimeStat.exprBasicCtxBldTime.tv_sec != 0) || (lastTimeStat.exprBasicCtxBldTime.tv_nsec != 0); + bool lastExprBasicEvalTime = (lastTimeStat.exprBasicEvalTime.tv_sec != 0) || (lastTimeStat.exprBasicEvalTime.tv_nsec != 0); bool lastExprJexlCtxBldTime = (lastTimeStat.exprJexlCtxBldTime.tv_sec != 0) || (lastTimeStat.exprJexlCtxBldTime.tv_nsec != 0); bool lastExprJexlEvalTime = (lastTimeStat.exprJexlEvalTime.tv_sec != 0) || (lastTimeStat.exprJexlEvalTime.tv_nsec != 0); bool lastRenderTime = (lastTimeStat.renderTime.tv_sec != 0) || (lastTimeStat.renderTime.tv_nsec != 0); @@ -236,8 +236,8 @@ std::string renderTimingStatistics(void) if (accMongoReadWaitTime) accJh.addNumber("mongoReadWait", timeSpecToFloat(accTimeStat.mongoReadWaitTime)); if (accMongoWriteWaitTime) accJh.addNumber("mongoWriteWait", timeSpecToFloat(accTimeStat.mongoWriteWaitTime)); if (accMongoCommandWaitTime) accJh.addNumber("mongoCommandWait", timeSpecToFloat(accTimeStat.mongoCommandWaitTime)); - if (accExprLegacyCtxBldTime) accJh.addNumber("exprLegacyCtxBld", timeSpecToFloat(accTimeStat.exprLegacyCtxBldTime)); - if (accExprLegacyEvalTime) accJh.addNumber("exprLegacyEval", timeSpecToFloat(accTimeStat.exprLegacyEvalTime)); + if (accExprBasicCtxBldTime) accJh.addNumber("exprBasicCtxBld", timeSpecToFloat(accTimeStat.exprBasicCtxBldTime)); + if (accExprBasicEvalTime) accJh.addNumber("exprBasicEval", timeSpecToFloat(accTimeStat.exprBasicEvalTime)); if (accExprJexlCtxBldTime) accJh.addNumber("exprJexlCtxBld", timeSpecToFloat(accTimeStat.exprJexlCtxBldTime)); if (accExprJexlEvalTime) accJh.addNumber("exprJexlEval", timeSpecToFloat(accTimeStat.exprJexlEvalTime)); if (accRenderTime) accJh.addNumber("render", timeSpecToFloat(accTimeStat.renderTime)); @@ -255,8 +255,8 @@ std::string renderTimingStatistics(void) if (lastMongoReadWaitTime) lastJh.addNumber("mongoReadWait", timeSpecToFloat(lastTimeStat.mongoReadWaitTime)); if (lastMongoWriteWaitTime) lastJh.addNumber("mongoWriteWait", timeSpecToFloat(lastTimeStat.mongoWriteWaitTime)); if (lastMongoCommandWaitTime) lastJh.addNumber("mongoCommandWait", timeSpecToFloat(lastTimeStat.mongoCommandWaitTime)); - if (lastExprLegacyCtxBldTime) lastJh.addNumber("exprLegacyCtxBld", timeSpecToFloat(lastTimeStat.exprLegacyCtxBldTime)); - if (lastExprLegacyEvalTime) lastJh.addNumber("exprLegacyEval", timeSpecToFloat(lastTimeStat.exprLegacyEvalTime)); + if (lastExprBasicCtxBldTime) lastJh.addNumber("exprBasicCtxBld", timeSpecToFloat(lastTimeStat.exprBasicCtxBldTime)); + if (lastExprBasicEvalTime) lastJh.addNumber("exprBasicEval", timeSpecToFloat(lastTimeStat.exprBasicEvalTime)); if (lastExprJexlCtxBldTime) lastJh.addNumber("exprJexlCtxBld", timeSpecToFloat(lastTimeStat.exprJexlCtxBldTime)); if (lastExprJexlEvalTime) lastJh.addNumber("exprJexlEval", timeSpecToFloat(lastTimeStat.exprJexlEvalTime)); if (lastRenderTime) lastJh.addNumber("render", timeSpecToFloat(lastTimeStat.renderTime)); diff --git a/src/lib/common/statistics.h b/src/lib/common/statistics.h index 623e074dcf..a5e1195781 100644 --- a/src/lib/common/statistics.h +++ b/src/lib/common/statistics.h @@ -233,9 +233,9 @@ struct timespec diff; \ clock_gettime(CLOCK_REALTIME, &exprCtxBldEnd); \ clock_difftime(&exprCtxBldEnd, &exprCtxBldStart, &diff); \ - if (legacy) \ + if (basic) \ { \ - clock_addtime(&threadLastTimeStat.exprLegacyCtxBldTime, &diff); \ + clock_addtime(&threadLastTimeStat.exprBasicCtxBldTime, &diff); \ } \ else \ { \ @@ -247,30 +247,30 @@ /* **************************************************************************** * -* TIME_EXPR_LEGACY_EVAL_START - +* TIME_EXPR_BASIC_EVAL_START - */ -#define TIME_EXPR_LEGACY_EVAL_START() \ - struct timespec exprLegacyEvalStart; \ - struct timespec exprLegacyEvalEnd; \ - \ - if (timingStatistics) \ - { \ - clock_gettime(CLOCK_REALTIME, &exprLegacyEvalStart); \ +#define TIME_EXPR_BASIC_EVAL_START() \ + struct timespec exprBasicEvalStart; \ + struct timespec exprBasicEvalEnd; \ + \ + if (timingStatistics) \ + { \ + clock_gettime(CLOCK_REALTIME, &exprBasicEvalStart); \ } /* **************************************************************************** * -* TIME_EXPR_LEGACY_EVAL_STOP - +* TIME_EXPR_BASIC_EVAL_STOP - */ -#define TIME_EXPR_LEGACY_EVAL_STOP() \ - if (timingStatistics) \ - { \ - struct timespec diff; \ - clock_gettime(CLOCK_REALTIME, &exprLegacyEvalEnd); \ - clock_difftime(&exprLegacyEvalEnd, &exprLegacyEvalStart, &diff); \ - clock_addtime(&threadLastTimeStat.exprLegacyEvalTime, &diff); \ +#define TIME_EXPR_BASIC_EVAL_STOP() \ + if (timingStatistics) \ + { \ + struct timespec diff; \ + clock_gettime(CLOCK_REALTIME, &exprBasicEvalEnd); \ + clock_difftime(&exprBasicEvalEnd, &exprBasicEvalStart, &diff); \ + clock_addtime(&threadLastTimeStat.exprBasicEvalTime, &diff); \ } @@ -299,7 +299,7 @@ { \ struct timespec diff; \ clock_gettime(CLOCK_REALTIME, &exprJexlEvalEnd); \ - clock_difftime(&exprJexlEvalEnd, &exprJexlEvalStart, &diff); \ + clock_difftime(&exprJexlEvalEnd, &exprJexlEvalStart, &diff); \ clock_addtime(&threadLastTimeStat.exprJexlEvalTime, &diff); \ } @@ -317,8 +317,8 @@ typedef struct TimeStat struct timespec mongoReadWaitTime; struct timespec mongoWriteWaitTime; struct timespec mongoCommandWaitTime; - struct timespec exprLegacyCtxBldTime; - struct timespec exprLegacyEvalTime; + struct timespec exprBasicCtxBldTime; + struct timespec exprBasicEvalTime; struct timespec exprJexlCtxBldTime; struct timespec exprJexlEvalTime; struct timespec renderTime; diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 6f8421ca1e..927809b693 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -34,9 +34,9 @@ * * ExprContextObject::ExprContextObject - */ -ExprContextObject::ExprContextObject(bool _legacy) +ExprContextObject::ExprContextObject(bool _basic) { - legacy = _legacy; + basic = _basic; } @@ -69,7 +69,7 @@ std::map* ExprContextObject::getMap(void) */ void ExprContextObject::add(const std::string &key, const std::string &_value, bool raw) { - if (legacy) + if (basic) { std::string value = _value; if (!raw) @@ -77,7 +77,7 @@ void ExprContextObject::add(const std::string &key, const std::string &_value, b // This is the case of regular string. The raw case is for JSON generated from compound values value = '"' + _value + '"'; } - LM_T(LmtExpr, ("adding to legacy expression context object (string): %s=%s", key.c_str(), value.c_str())); + LM_T(LmtExpr, ("adding to basic expression context object (string): %s=%s", key.c_str(), value.c_str())); repl.insert(std::pair(key, value)); } else @@ -95,9 +95,9 @@ void ExprContextObject::add(const std::string &key, const std::string &_value, b */ void ExprContextObject::add(const std::string &key, double _value) { - if (legacy) + if (basic) { - LM_T(LmtExpr, ("adding to legacy expression context object (double): %s=%f", key.c_str(), _value)); + LM_T(LmtExpr, ("adding to basic expression context object (double): %s=%f", key.c_str(), _value)); repl.insert(std::pair(key, double2string(_value))); } else @@ -115,9 +115,9 @@ void ExprContextObject::add(const std::string &key, double _value) */ void ExprContextObject::add(const std::string &key, bool _value) { - if (legacy) + if (basic) { - LM_T(LmtExpr, ("adding to legacy expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); + LM_T(LmtExpr, ("adding to basic expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); repl.insert(std::pair(key, _value? "true": "false")); } else @@ -135,9 +135,9 @@ void ExprContextObject::add(const std::string &key, bool _value) */ void ExprContextObject::add(const std::string &key) { - if (legacy) + if (basic) { - LM_T(LmtExpr, ("adding to legacy expression context object (none): %s", key.c_str())); + LM_T(LmtExpr, ("adding to basic expression context object (none): %s", key.c_str())); repl.insert(std::pair(key, "null")); } else @@ -155,9 +155,9 @@ void ExprContextObject::add(const std::string &key) */ void ExprContextObject::add(const std::string &key, ExprContextObject exprContextObject) { - if (legacy) + if (basic) { - LM_E(("Runtime Error (this method must not be invoked in legacy mode)")); + LM_E(("Runtime Error (this method must not be invoked in basic mode)")); } else { @@ -175,9 +175,9 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex */ void ExprContextObject::add(const std::string &key, ExprContextList exprContextList) { - if (legacy) + if (basic) { - LM_E(("Runtime Error (this method must not be invoked in legacy mode)")); + LM_E(("Runtime Error (this method must not be invoked in basic mode)")); } else { @@ -191,11 +191,11 @@ void ExprContextObject::add(const std::string &key, ExprContextList exprContextL /* **************************************************************************** * -* ExprContextObject::isLegacy - +* ExprContextObject::isBasic - */ -bool ExprContextObject::isLegacy(void) +bool ExprContextObject::isBasic(void) { - return legacy; + return basic; } diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index aa92e3094a..0e994be3b2 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -40,12 +40,12 @@ class ExprContextList; // forward declaration class ExprContextObject { private: - bool legacy; - JsonObjectHelper jh; // used in regular (i.e. not legacy) mode - std::map repl; // used in legacy mode + bool basic; + JsonObjectHelper jh; // used in regular (i.e. not basic) mode + std::map repl; // used in basic mode public: - ExprContextObject(bool legacy = false); + ExprContextObject(bool basic = false); std::string getJexlContext(void); std::map* getMap(void); @@ -57,7 +57,7 @@ class ExprContextObject void add(const std::string& key, ExprContextObject exprContextObject); void add(const std::string& key, ExprContextList exprContextList); - bool isLegacy(void); + bool isBasic(void); }; class ExprContextList diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index f54f91ef7c..d06bfb6d06 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -65,12 +65,12 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st ExprResult r; r.valueType = orion::ValueTypeNull; - if (exprContextObjectP->isLegacy()) + if (exprContextObjectP->isBasic()) { // std::map based evaluation. Only pure replacement is supported - TIME_EXPR_LEGACY_EVAL_START(); - LM_T(LmtExpr, ("evaluating legacy expression: <%s>", _expression.c_str())); + TIME_EXPR_BASIC_EVAL_START(); + LM_T(LmtExpr, ("evaluating basic expression: <%s>", _expression.c_str())); std::map* replacementsP = exprContextObjectP->getMap(); @@ -79,9 +79,9 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st { r.valueType = orion::ValueTypeString; r.stringValue = iter->second; - LM_T(LmtExpr, ("legacy evaluation result: <%s>", r.stringValue.c_str())); + LM_T(LmtExpr, ("basic evaluation result: <%s>", r.stringValue.c_str())); } - TIME_EXPR_LEGACY_EVAL_STOP(); + TIME_EXPR_BASIC_EVAL_STOP(); } else { diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index bd8c66738d..0542e7aa56 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -316,19 +316,19 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - // FIXME PR: unhardwire legacy == false - bool legacy = false; - ExprContextObject exprContext(legacy); + // FIXME PR: unhardwire basic == false + bool basic = false; + ExprContextObject exprContext(basic); - // FIXME PR: no-legacy context is now based in JsonHelper, which may mess with key repetition. Check this - // It seems that add() semantics are different in legacy and jexl mode. In jexl mode, if the key already exists, it is - // updated. In legacy model, if the key already exists, the operation is ignored (so previous value is preserved). Taking + // FIXME PR: jexl context (i.e. no basic) is now based in JsonHelper, which may mess with key repetition. Check this + // It seems that add() semantics are different in basic and jexl mode. In jexl mode, if the key already exists, it is + // updated. In basic model, if the key already exists, the operation is ignored (so previous value is preserved). Taking // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence // over macros comming from macros of the same name we conditionally add them depending the case TIME_EXPR_CTXBLD_START(); exprContext.add("id", en.id); exprContext.add("type", en.type); - if (!legacy) + if (!basic) { exprContext.add("service", tenant); exprContext.add("servicePath", en.servicePath); @@ -336,9 +336,9 @@ static SenderThreadParams* buildSenderParamsCustom } for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - en.attributeVector[ix]->addToContext(&exprContext, legacy); + en.attributeVector[ix]->addToContext(&exprContext, basic); } - if (legacy) + if (basic) { exprContext.add("service", tenant); exprContext.add("servicePath", en.servicePath); diff --git a/src/lib/rest/rest.cpp b/src/lib/rest/rest.cpp index 3ddd0a85e7..f09fd37529 100644 --- a/src/lib/rest/rest.cpp +++ b/src/lib/rest/rest.cpp @@ -618,8 +618,8 @@ static void requestCompleted clock_addtime(&accTimeStat.mongoWriteWaitTime, &threadLastTimeStat.mongoWriteWaitTime); clock_addtime(&accTimeStat.mongoReadWaitTime, &threadLastTimeStat.mongoReadWaitTime); clock_addtime(&accTimeStat.mongoCommandWaitTime, &threadLastTimeStat.mongoCommandWaitTime); - clock_addtime(&accTimeStat.exprLegacyCtxBldTime, &threadLastTimeStat.exprLegacyCtxBldTime); - clock_addtime(&accTimeStat.exprLegacyEvalTime, &threadLastTimeStat.exprLegacyEvalTime); + clock_addtime(&accTimeStat.exprBasicCtxBldTime, &threadLastTimeStat.exprBasicCtxBldTime); + clock_addtime(&accTimeStat.exprBasicEvalTime, &threadLastTimeStat.exprBasicEvalTime); clock_addtime(&accTimeStat.exprJexlCtxBldTime, &threadLastTimeStat.exprJexlCtxBldTime); clock_addtime(&accTimeStat.exprJexlEvalTime, &threadLastTimeStat.exprJexlEvalTime); clock_addtime(&accTimeStat.renderTime, &threadLastTimeStat.renderTime); From 81e98c6669815e9a6edb36563e04ccbbb40118f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 12:22:59 +0200 Subject: [PATCH 067/104] ADD get_cjex.sh --- get_cjexl.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 get_cjexl.sh diff --git a/get_cjexl.sh b/get_cjexl.sh new file mode 100644 index 0000000000..c906af50b2 --- /dev/null +++ b/get_cjexl.sh @@ -0,0 +1,33 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +CJEXL_VERSION=$1 +TOKEN=$2 + +res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION) +if [ "$res_code" -eq 200 ]; then + echo "get_cjexl: downloading cjexl lib" + ASSET_ID=$(curl -s -S -H "Authorization: token $TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') + curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID + MD5SUM=$(md5sum /usr/local/lib/libcjexl.a | awk -F ' ' '{print$1}') + echo "get_cjexl: cjexl lib md5sum is $MD5SUM" +else + echo "get_cjexl: error $res_code accessing cjexl release. Maybe token is invalid?" +fi \ No newline at end of file From 8eefa7eef6b11215550be99413213c4316d3c313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 14:59:40 +0200 Subject: [PATCH 068/104] FIX Dockerfile --- ci/deb/build.sh | 1 + docker/Dockerfile | 4 ++++ docker/Dockerfile.alpine | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index 107cb11f30..fcdd48fa83 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -173,6 +173,7 @@ echo "Builder: create temp folders" rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} # FIXME PR: unhardwire release number +# FIXME PR: use get_cjexl script CJEXL_VERSION="0.0.0" if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" diff --git a/docker/Dockerfile b/docker/Dockerfile index b5b80157d4..13033a9343 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -29,11 +29,13 @@ ARG GITHUB_REPOSITORY=fiware-orion ARG GIT_NAME ARG GIT_REV_ORION ARG CLEAN_DEV_TOOLS +ARG REPO_ACCESS_TOKEN ENV ORION_USER ${ORION_USER:-orion} ENV GIT_NAME ${GIT_NAME:-telefonicaid} ENV GIT_REV_ORION ${GIT_REV_ORION:-master} ENV CLEAN_DEV_TOOLS ${CLEAN_DEV_TOOLS:-1} +ENV REPO_ACCESS_TOKEN ${REPO_ACCESS_TOKEN:-""} SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -102,6 +104,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ + bash get_cjexl.sh 0.0.0 ${REPO_ACCESS_TOKEN} && \ make && \ make install && \ # reduce size of installed binaries @@ -125,6 +128,7 @@ RUN \ /opt/mongo-c-driver-1.24.3 \ /usr/local/include/mongo \ /usr/local/lib/libmongoclient.a \ + /usr/local/lib/libcjexl.a \ /opt/rapidjson-1.1.0 \ /opt/v1.1.0.tar.gz \ /usr/local/include/rapidjson \ diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 524335f677..2c4ed4a02e 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -32,11 +32,13 @@ ARG GITHUB_REPOSITORY=fiware-orion ARG GIT_NAME ARG GIT_REV_ORION ARG CLEAN_DEV_TOOLS +ARG REPO_ACCESS_TOKEN ENV ORION_USER ${ORION_USER:-orion} ENV GIT_NAME ${GIT_NAME:-telefonicaid} ENV GIT_REV_ORION ${GIT_REV_ORION:-master} ENV CLEAN_DEV_TOOLS ${CLEAN_DEV_TOOLS:-1} +ENV REPO_ACCESS_TOKEN ${REPO_ACCESS_TOKEN:-""} SHELL ["/bin/ash", "-o", "pipefail", "-c"] @@ -108,6 +110,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ + bash get_cjexl.sh 0.0.0 ${REPO_ACCESS_TOKEN} && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ @@ -133,6 +136,7 @@ RUN \ /opt/mongo-c-driver-1.24.3 \ /usr/local/include/mongo \ /usr/local/lib/libmongoclient.a \ + /usr/local/lib/libcjexl.a \ /opt/rapidjson-1.1.0 \ /opt/v1.1.0.tar.gz \ /usr/local/include/rapidjson \ From 915188554156d8d6eaab58db27ed646a504b4f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 15:36:44 +0200 Subject: [PATCH 069/104] FIX add new jexl test cases --- .../jexl_several_expressions.test | 232 ++++++++++++++++++ .../jexl_syntax_errors.test | 178 ++++++++++++++ .../jexl_transformation_unknown.test | 144 +++++++++++ 3 files changed, 554 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test new file mode 100644 index 0000000000..82ffb1ccaa --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test @@ -0,0 +1,232 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (several expressions in same subscription) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression sum: A+B, mul: A*B +# 02. Create entity E1 with A=2 and B=3 +# 03. Update entity E1 with A=2.1 and B=-3.8 +# 04. Dump accumulator and see two notifications ({sum: 5, mul: 6}, {sum: -1.7, mul: -7.98}) +# + + +echo "01. Create custom sub with custom expression sum: A+B, mul: A*B" +echo "===============================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "sum": { + "value": "${A+B}", + "type": "Calculated" + }, + "mul": { + "value": "${A*B}", + "type": "Calculated" + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=2 and B=3" +echo "=====================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 2, + "type": "Number" + }, + "B": { + "value": 3, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A=2.1 and B=-3.8" +echo "==========================================" +payload='{ + "A": { + "value": 2.1, + "type": "Number" + }, + "B": { + "value": -3.8, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "Dump accumulator and see two notifications ({sum: 5, mul: 6}, {sum: -1.7, mul: -7.98})" +echo "======================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: A+B, mul: A*B +=============================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=2 and B=3 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A=2.1 and B=-3.8 +========================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +Dump accumulator and see two notifications ({sum: 5, mul: 6}, {sum: -1.7, mul: -7.98}) +====================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 273 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 3 + }, + "id": "E1", + "mul": { + "metadata": {}, + "type": "Calculated", + "value": 6 + }, + "sum": { + "metadata": {}, + "type": "Calculated", + "value": 5 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 285 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2.1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": -3.8 + }, + "id": "E1", + "mul": { + "metadata": {}, + "type": "Calculated", + "value": -7.98 + }, + "sum": { + "metadata": {}, + "type": "Calculated", + "value": -1.7 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test new file mode 100644 index 0000000000..4359b235bc --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test @@ -0,0 +1,178 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification with syntax errors + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression with several JEXL expressions with syntax errors +# 02. Create entity E1 with A=[ foobar ] +# 03. Dump accumulator and see notification with expressions resolved to null (except e0 = TRUE) +# + + +echo "01. Create custom sub with custom expression with several JEXL expressions with syntax errors" +echo "=============================================================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "e0": { + "value": "${A[0]|includes('\''foo'\'')|toString|uppercase}", + "type": "Calculated" + }, + "e1": { + "value": "${A[0|includes('\''foo'\'')|toString|uppercase}", + "type": "Calculated" + }, + "e2": { + "value": "${A[0]|includes('\''foo)|toString|uppercase}", + "type": "Calculated" + }, + "e3": { + "value": "${A[0]|includes('\''foo'\''|toString|uppercase}", + "type": "Calculated" + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=[ foobar ]" +echo "======================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": [ "foobar" ], + "type": "StructuredValue" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Dump accumulator and see notification with expressions resolved to null (except e0 = TRUE)" +echo "==============================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression with several JEXL expressions with syntax errors +============================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=[ foobar ] +====================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Dump accumulator and see notification with expressions resolved to null (except e0 = TRUE) +============================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 359 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "StructuredValue", + "value": [ + "foobar" + ] + }, + "e0": { + "metadata": {}, + "type": "Calculated", + "value": "TRUE" + }, + "e1": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "e2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "e3": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test new file mode 100644 index 0000000000..3d536559b4 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test @@ -0,0 +1,144 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using unknown transformation) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression low: A|patatacase +# 02. Create entity E1 with A=IwantAllInLOWERCase +# 03. Dump accumulator and see notifications (low: null) +# + + +echo "01. Create custom sub with custom expression low: A|patatacase" +echo "==============================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "low": { + "value": "${A|patatacase}", + "type": "Calculated" + } + } + }, + "attrs": [ "low" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=IwantAllInLOWERCase" +echo "===============================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": "IwantAllInLOWERCase", + "type": "Text" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Dump accumulator and see notifications (low: null)" +echo "======================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression low: A|patatacase +============================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=IwantAllInLOWERCase +=============================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Dump accumulator and see notifications (low: null) +====================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "low": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 76234f49836c353caeb679d138368a44b1508023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 17:40:56 +0200 Subject: [PATCH 070/104] FIX some FIXME PR marks --- ci/deb/build.sh | 14 +------------- src/lib/ngsiNotify/Notifier.cpp | 10 +++++----- .../jexl_basic_attrs_in_update.test | 10 +++++----- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index fcdd48fa83..5a5393f877 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -172,22 +172,10 @@ echo "===================================== PREPARE ============================ echo "Builder: create temp folders" rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} -# FIXME PR: unhardwire release number -# FIXME PR: use get_cjexl script -CJEXL_VERSION="0.0.0" if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION) - if [ "$res_code" -eq 200 ]; then - echo "Builder: downloading cjexl lib" - ASSET_ID=$(curl -s -S -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') - curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $REPO_ACCESS_TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID - MD5SUM=$(md5sum /usr/local/lib/libcjexl.a | awk -F ' ' '{print$1}') - echo "Builder: cjexl lib md5sum is $MD5SUM" - else - echo "Builder: error $res_code accessing cjexl release. Maybe token is invalid?" - fi + bash /opt/fiware-orion/get_cjexl.sh 0.0.0 $REPO_ACCESS_TOKEN fi if [ -n "${branch}" ]; then diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 0542e7aa56..a04d7c124b 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -316,15 +316,15 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - // FIXME PR: unhardwire basic == false + // FIXME PR: unhardwire basic == false depending on conditional compiling bool basic = false; ExprContextObject exprContext(basic); - // FIXME PR: jexl context (i.e. no basic) is now based in JsonHelper, which may mess with key repetition. Check this // It seems that add() semantics are different in basic and jexl mode. In jexl mode, if the key already exists, it is - // updated. In basic model, if the key already exists, the operation is ignored (so previous value is preserved). Taking - // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence - // over macros comming from macros of the same name we conditionally add them depending the case + // updated (in other words, the last added keys is the one that takes precedence). In basic model, if the key already + // exists, the operation is ignored (in other words, the first added key is the one that takes precedence). Taking + // into account that in the case of an attribute with name "service", "servicePath" or "authToken", it must have precedence + // over the ones comming from headers of the same name, we conditionally add them depending the case TIME_EXPR_CTXBLD_START(); exprContext.add("id", en.id); exprContext.add("type", en.type); diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test index 601f69d25a..6e31d9b36e 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test @@ -35,7 +35,7 @@ accumulatorStart --pretty-print # 02. Create entity E1 with A=1 and B=2 # 03. Update entity E1 with A=foo and B=bar # 04. Update entity E1 with A=2.1 and B=-3.8 -# 05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7) +# 05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7) # @@ -120,8 +120,8 @@ echo echo -echo "05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7)" -echo "===============================================================================" +echo "05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7)" +echo "=================================================================================" accumulatorDump echo echo @@ -164,8 +164,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7) -=============================================================================== +05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7) +================================================================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 221 From 4bf61333d920361a6fcbd59433d9cdb3db2a4838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 18:36:06 +0200 Subject: [PATCH 071/104] FIX conditional compilinig depending on cjexl availability --- CMakeLists.txt | 11 ++++++++++- src/lib/expressions/ExprManager.cpp | 17 +++++++++++++++++ src/lib/ngsiNotify/Notifier.cpp | 8 ++++++-- src/lib/serviceRoutines/versionTreat.cpp | 4 ++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b871e47ad8..0118dbef19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -230,11 +230,20 @@ SET (BOOST_MT # SET for common static libs. We use 1.24.3 as reference version. find_package (mongoc-1.0 1.24.3 EXACT) +# Is cjexl lib available? +find_library (HAVE_CJEXL cjexl PATHS /usr/lib /usr/lib64 /usr/local/lib64 /usr/local/lib) +if (HAVE_CJEXL) + message("Using cjexl") +else (HAVE_CJEXL) + message("Not using cjexl") + add_definitions(-DEXPR_BASIC) +endif (HAVE_CJEXL) + # Static libs common to contextBroker and unitTest binaries SET (COMMON_STATIC_LIBS microhttpd.a mosquitto.a - cjexl.a + ${HAVE_CJEXL} mongo::mongoc_static ) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index d06bfb6d06..47e492f223 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -30,6 +30,22 @@ #include "orionTypes/OrionValueType.h" #include "common/statistics.h" +#ifdef EXPR_BASIC +// Never called, but need to be defined to avoid compilation errors +static void* cjexl_new_engine() +{ + return NULL; +} + +static void cjexl_free_engine(void* ptr) +{ +} + +static const char* cjexl_eval(void* ptr, const char* script_ptr, const char* context_ptr) +{ + return ""; +} +#else // Interface to use libcjexl.a extern "C" { void* cjexl_new_engine(); @@ -42,6 +58,7 @@ extern "C" { extern "C" { const char* cjexl_eval(void* ptr, const char* script_ptr, const char* context_ptr); } +#endif /* **************************************************************************** diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index a04d7c124b..df16cbb610 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -315,9 +315,13 @@ static SenderThreadParams* buildSenderParamsCustom std::map headers; Entity& en = notifyCerP->entity; - // Used by several macroSubstitute() calls along this function - // FIXME PR: unhardwire basic == false depending on conditional compiling +#ifdef EXPR_BASIC + bool basic = true; +#else bool basic = false; +#endif + + // Used by several macroSubstitute() calls along this function ExprContextObject exprContext(basic); // It seems that add() semantics are different in basic and jexl mode. In jexl mode, if the key already exists, it is diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index eec99efedb..8a0fa24038 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -47,10 +47,12 @@ #include #include +#ifndef EXPR_BASIC // Interface to use libcjexl.a extern "C" { const char* cjexl_version(); } +#endif /* **************************************************************************** * @@ -94,7 +96,9 @@ std::string libVersions(void) total += curl + "\"" + curlVersion + "\"" + ",\n"; total += mosq + "\"" + mosqVersion + "\"" + ",\n"; total += mhd + "\"" + MHD_get_version() + "\"" + ",\n"; +#ifndef EXPR_BASIC total += cjexl + "\"" + cjexl_version() + "\"" + ",\n"; +#endif #ifdef OLD_SSL_VERSION_FORMAT // Needed by openssl 1.1.1n in Debian 11 and before total += ssl + "\"" + SHLIB_VERSION_NUMBER "\"" + ",\n"; From 25b463cf910f36a5f8edb7014ce32bac6e5da293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 13 May 2024 14:03:00 +0200 Subject: [PATCH 072/104] ADD jexl_missing_attribute_as_null.test test --- .../jexl_missing_attribute_as_null.test | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test new file mode 100644 index 0000000000..7cdc57b975 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test @@ -0,0 +1,247 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (missing attribute replaced as null) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression B: (A==null)?'is null':A +# 02. Create entity E1 with A=1 +# 03. Create entity E2 with A=null +# 04. Create entity E3 without A +# 05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +# + + +echo "01. Create custom sub with custom expression B: (A==null)?'is null':A" +echo "=====================================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B": { + "value": "${(A==null)?'\''is null'\'':A}", + "type": "Calculated" + } + } + }, + "attrs": [ "B" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Create entity E2 with A=null" +echo "================================" +payload='{ + "id": "E2", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "04. Create entity E3 without A" +echo "==============================" +payload='{ + "id": "E3", + "type": "T", + "C": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null)" +echo "===============================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression B: (A==null)?'is null':A +===================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Create entity E2 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +04. Create entity E3 without A +============================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E3?type=T +Content-Length: 0 + + + +05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +=============================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 127 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 135 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "id": "E2", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 135 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "id": "E3", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From d5108567bba54a189783b06cf61bffd99cd0a9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 14 May 2024 11:45:21 +0200 Subject: [PATCH 073/104] FIX simplify attribute_as_null test case and add extra one --- .../jexl_missing_attribute_as_null.test | 12 +- .../jexl_missing_attribute_as_null2.test | 247 ++++++++++++++++++ 2 files changed, 253 insertions(+), 6 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test index 7cdc57b975..a7af691e9c 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test @@ -31,7 +31,7 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with custom expression B: (A==null)?'is null':A +# 01. Create custom sub with custom expression B: A||'is null' # 02. Create entity E1 with A=1 # 03. Create entity E2 with A=null # 04. Create entity E3 without A @@ -39,8 +39,8 @@ accumulatorStart --pretty-print # -echo "01. Create custom sub with custom expression B: (A==null)?'is null':A" -echo "=====================================================================" +echo "01. Create custom sub with custom expression B: A||'is null'" +echo "============================================================" # NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) payload='{ "subject": { @@ -56,7 +56,7 @@ payload='{ "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", "ngsi": { "B": { - "value": "${(A==null)?'\''is null'\'':A}", + "value": "${A||'\''is null'\''}", "type": "Calculated" } } @@ -122,8 +122,8 @@ echo --REGEXPECT-- -01. Create custom sub with custom expression B: (A==null)?'is null':A -===================================================================== +01. Create custom sub with custom expression B: A||'is null' +============================================================ HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test new file mode 100644 index 0000000000..a7af691e9c --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test @@ -0,0 +1,247 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (missing attribute replaced as null) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression B: A||'is null' +# 02. Create entity E1 with A=1 +# 03. Create entity E2 with A=null +# 04. Create entity E3 without A +# 05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +# + + +echo "01. Create custom sub with custom expression B: A||'is null'" +echo "============================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B": { + "value": "${A||'\''is null'\''}", + "type": "Calculated" + } + } + }, + "attrs": [ "B" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Create entity E2 with A=null" +echo "================================" +payload='{ + "id": "E2", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "04. Create entity E3 without A" +echo "==============================" +payload='{ + "id": "E3", + "type": "T", + "C": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null)" +echo "===============================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression B: A||'is null' +============================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Create entity E2 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +04. Create entity E3 without A +============================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E3?type=T +Content-Length: 0 + + + +05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +=============================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 127 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 135 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "id": "E2", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 135 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "id": "E3", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 264a9a6203a787e1c8af5c923878637cf00c994c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 12:38:50 +0200 Subject: [PATCH 074/104] FIX improve missing attribute as null jexl tests --- .../jexl_missing_attribute_as_null.test | 131 +++++++++++++++--- .../jexl_missing_attribute_as_null2.test | 83 +++++------ 2 files changed, 156 insertions(+), 58 deletions(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test index a7af691e9c..9d711fc6a7 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test @@ -31,16 +31,16 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with custom expression B: A||'is null' +# 01. Create custom sub with custom expression B0: (A==null)?-1:A, B1: A||'is null', B2: A||-1, B3: A||0-1||, B4: A||('-1'|parseInt), B5: A||0 # 02. Create entity E1 with A=1 # 03. Create entity E2 with A=null # 04. Create entity E3 without A -# 05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +# 05. Dump accumulator and see three notifications (<1, 1, null, 1, 1, 1>, <'is null', 'is null', null, -1, -1, 0>, ) # -echo "01. Create custom sub with custom expression B: A||'is null'" -echo "============================================================" +echo "01. Create custom sub with custom expression B0: (A==null)?-1:A, B1: A||'is null', B2: A||-1, B3: A||0-1||, B4: A||'-1'||parseInt, B5: A||0" +echo "===========================================================================================================================================" # NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) payload='{ "subject": { @@ -55,13 +55,33 @@ payload='{ "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", "ngsi": { - "B": { + "B0": { + "value": "${(A==null)?'\''is null'\'':A}", + "type": "Calculated" + }, + "B1": { "value": "${A||'\''is null'\''}", "type": "Calculated" + }, + "B2": { + "value": "${A||-1}", + "type": "Calculated" + }, + "B3": { + "value": "${A||0-1}", + "type": "Calculated" + }, + "B4": { + "value": "${A||('\''-1'\''|parseInt)}", + "type": "Calculated" + }, + "B5": { + "value": "${A||0}", + "type": "Calculated" } } }, - "attrs": [ "B" ] + "attrs": [ "B0", "B1", "B2", "B3", "B4", "B5" ] } }' orionCurl --url /v2/subscriptions --payload "$payload" @@ -114,16 +134,16 @@ echo echo -echo "05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null)" -echo "===============================================================================" +echo "05. Dump accumulator and see three notifications (<1, 1, null, 1, 1, 1>, <'is null', 'is null', null, -1, -1, 0>, )" +echo "=====================================================================================================================================================" accumulatorDump echo echo --REGEXPECT-- -01. Create custom sub with custom expression B: A||'is null' -============================================================ +01. Create custom sub with custom expression B0: (A==null)?-1:A, B1: A||'is null', B2: A||-1, B3: A||0-1||, B4: A||'-1'||parseInt, B5: A||0 +=========================================================================================================================================== HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -162,11 +182,11 @@ Content-Length: 0 -05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) -=============================================================================== +05. Dump accumulator and see three notifications (<1, 1, null, 1, 1, 1>, <'is null', 'is null', null, -1, -1, 0>, ) +===================================================================================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 127 +Content-Length: 386 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -177,7 +197,32 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "B0": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "B1": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "B2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "B3": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "B4": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "B5": { "metadata": {}, "type": "Calculated", "value": 1 @@ -191,7 +236,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 135 +Content-Length: 404 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -202,11 +247,36 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "B0": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "B1": { "metadata": {}, "type": "Calculated", "value": "is null" }, + "B2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "B3": { + "metadata": {}, + "type": "Calculated", + "value": -1 + }, + "B4": { + "metadata": {}, + "type": "Calculated", + "value": -1 + }, + "B5": { + "metadata": {}, + "type": "Calculated", + "value": 0 + }, "id": "E2", "type": "T" } @@ -216,7 +286,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 135 +Content-Length: 399 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -227,11 +297,36 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "B0": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "B1": { "metadata": {}, "type": "Calculated", "value": "is null" }, + "B2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "B3": { + "metadata": {}, + "type": "Calculated", + "value": -1 + }, + "B4": { + "metadata": {}, + "type": "Calculated", + "value": -1 + }, + "B5": { + "metadata": {}, + "type": "Calculated", + "value": 0 + }, "id": "E3", "type": "T" } diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test index a7af691e9c..8c0fd939f4 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification (missing attribute replaced as null) +JEXL expression in custom notification (missing attribute replaced as null additional case) --SHELL-INIT-- dbInit CB @@ -31,17 +31,16 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with custom expression B: A||'is null' -# 02. Create entity E1 with A=1 -# 03. Create entity E2 with A=null -# 04. Create entity E3 without A -# 05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +# 01. Create custom sub with custom expression S: (A||0-1)+(B||0-2) +# 02. Create entity E1 with A=1, B=2 +# 03. Create entity E2 with B=1 +# 04. Create entity E3 without A or B +# 05. Dump accumulator and see three notifications (S: 3, S: 0, S: -3) # -echo "01. Create custom sub with custom expression B: A||'is null'" -echo "============================================================" -# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +echo "01. Create custom sub with custom expression S: (A||0-1)+(B||0-2)" +echo "=================================================================" payload='{ "subject": { "entities": [ @@ -55,13 +54,13 @@ payload='{ "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", "ngsi": { - "B": { - "value": "${A||'\''is null'\''}", + "S": { + "value": "${(A||0-1)+(B||0-2)}", "type": "Calculated" } } }, - "attrs": [ "B" ] + "attrs": [ "S" ] } }' orionCurl --url /v2/subscriptions --payload "$payload" @@ -69,14 +68,18 @@ echo echo -echo "02. Create entity E1 with A=1" -echo "=============================" +echo "02. Create entity E1 with A=1, B=2" +echo "==================================" payload='{ "id": "E1", "type": "T", "A": { "value": 1, "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" } }' orionCurl --url /v2/entities --payload "$payload" @@ -84,13 +87,13 @@ echo echo -echo "03. Create entity E2 with A=null" -echo "================================" +echo "03. Create entity E2 with B=1" +echo "=============================" payload='{ "id": "E2", "type": "T", - "A": { - "value": null, + "B": { + "value": 1, "type": "Number" } }' @@ -99,8 +102,8 @@ echo echo -echo "04. Create entity E3 without A" -echo "==============================" +echo "04. Create entity E3 without A or B" +echo "===================================" payload='{ "id": "E3", "type": "T", @@ -114,16 +117,16 @@ echo echo -echo "05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null)" -echo "===============================================================================" +echo "05. Dump accumulator and see three notifications (S: 3, S: 0, S: -3)" +echo "====================================================================" accumulatorDump echo echo --REGEXPECT-- -01. Create custom sub with custom expression B: A||'is null' -============================================================ +01. Create custom sub with custom expression S: (A||0-1)+(B||0-2) +================================================================= HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -132,8 +135,8 @@ Content-Length: 0 -02. Create entity E1 with A=1 -============================= +02. Create entity E1 with A=1, B=2 +================================== HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -142,8 +145,8 @@ Content-Length: 0 -03. Create entity E2 with A=null -================================ +03. Create entity E2 with B=1 +============================= HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -152,8 +155,8 @@ Content-Length: 0 -04. Create entity E3 without A -============================== +04. Create entity E3 without A or B +=================================== HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -162,8 +165,8 @@ Content-Length: 0 -05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) -=============================================================================== +05. Dump accumulator and see three notifications (S: 3, S: 0, S: -3) +==================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 127 @@ -177,10 +180,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "S": { "metadata": {}, "type": "Calculated", - "value": 1 + "value": 3 }, "id": "E1", "type": "T" @@ -191,7 +194,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 135 +Content-Length: 127 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -202,10 +205,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "S": { "metadata": {}, "type": "Calculated", - "value": "is null" + "value": 0 }, "id": "E2", "type": "T" @@ -216,7 +219,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 135 +Content-Length: 128 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -227,10 +230,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "S": { "metadata": {}, "type": "Calculated", - "value": "is null" + "value": -3 }, "id": "E3", "type": "T" From dc35a4a81050c827a8347c1182373134a24e9b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 13:34:21 +0200 Subject: [PATCH 075/104] ADD GitAction to publish docker in Dockerhub for branches --- .github/workflows/publishimage-master.yml | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/publishimage-master.yml diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml new file mode 100644 index 0000000000..c302600bad --- /dev/null +++ b/.github/workflows/publishimage-master.yml @@ -0,0 +1,35 @@ +name: Publish Docker image (master) + +# The workflow will push images on every tag in the format x.y.z +# FIXME PR: change label and branch before merging into master + +on: + push: + branches: + #- master + - feature/4004_jexl_expressions + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build Docker image + #run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + run: docker build -t telefonicaiot/fiware-orion:jexl --build-arg GIT_REV_ORION=feature/4004_jexl_expressions --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + + - name: Push Docker image + #run: docker push telefonicaiot/fiware-orion:latest + run: docker push telefonicaiot/fiware-orion:jexl From 2a75eb1fcc311b5891217934913ce425d06295b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 13:38:22 +0200 Subject: [PATCH 076/104] FIX GitAction login step --- .github/workflows/publishimage-master.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index c302600bad..07347c86c7 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -23,8 +23,8 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build Docker image #run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . From e789afcd7dbaae8fd4c281d3b00847f00754ca30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 14:15:16 +0200 Subject: [PATCH 077/104] FIX GitAction dockerhub credential fix --- .github/workflows/publishimage-master.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index 07347c86c7..a238eb15fb 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -23,8 +23,8 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_TEF_USERNAME }} + password: ${{ secrets.DOCKERHUB_TEF_TOKEN }} - name: Build Docker image #run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . From 7281e1f9abd3991a01d71388781153c4a9ff51fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 14:47:13 +0200 Subject: [PATCH 078/104] ADD GitAction for build docker for tags --- ...blishimage.yml => publishimage-fiware.yml} | 4 +-- .github/workflows/publishimage-master.yml | 12 +++---- .github/workflows/publishimage-tag.yml | 35 +++++++++++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) rename .github/workflows/{publishimage.yml => publishimage-fiware.yml} (89%) create mode 100644 .github/workflows/publishimage-tag.yml diff --git a/.github/workflows/publishimage.yml b/.github/workflows/publishimage-fiware.yml similarity index 89% rename from .github/workflows/publishimage.yml rename to .github/workflows/publishimage-fiware.yml index af56a99bae..07f10beb01 100644 --- a/.github/workflows/publishimage.yml +++ b/.github/workflows/publishimage-fiware.yml @@ -1,6 +1,6 @@ -name: Publish pre Image from master +name: Publish pre Image from master in FIWARE dockerhub -# The workflow will push PRE images from master on every merge. The images will be tagged with PRE and the next minor increase on the +# The workflow will push PRE images from master on every merge to the fiware dockerhub organization. The images will be tagged with PRE and the next minor increase on the # semver(based on the github releases) # It will NOT produce releases and release images or the 'latest' tag from master. Both (releases and 'latest' tag) rely on the # dockerhub autobuild feature diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index a238eb15fb..2811dea64f 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -1,13 +1,11 @@ name: Publish Docker image (master) -# The workflow will push images on every tag in the format x.y.z -# FIXME PR: change label and branch before merging into master +# The workflow will push images for master on every merge on: push: branches: - #- master - - feature/4004_jexl_expressions + - master jobs: build-and-push: @@ -27,9 +25,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TEF_TOKEN }} - name: Build Docker image - #run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . - run: docker build -t telefonicaiot/fiware-orion:jexl --build-arg GIT_REV_ORION=feature/4004_jexl_expressions --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . - name: Push Docker image - #run: docker push telefonicaiot/fiware-orion:latest - run: docker push telefonicaiot/fiware-orion:jexl + run: docker push telefonicaiot/fiware-orion:latest diff --git a/.github/workflows/publishimage-tag.yml b/.github/workflows/publishimage-tag.yml new file mode 100644 index 0000000000..6cd6b94b40 --- /dev/null +++ b/.github/workflows/publishimage-tag.yml @@ -0,0 +1,35 @@ +name: Publish Docker image (tag) + +# The workflow will push images on every tag in the format x.y.z + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_TEF_USERNAME }} + password: ${{ secrets.DOCKERHUB_TEF_TOKEN }} + + - name: Extract version from tag + id: extract_version + run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Build Docker image + run: docker build -t telefonicaiot/fiware-orion:${{ env.VERSION }} --build-arg GIT_REV_ORION=${{ env.VERSION }} --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + + - name: Push Docker image + run: docker push telefonicaiot/fiware-orion:${{ env.VERSION }} From 8e92831ea026fbdc4545fcf93fd0d889fd71d5ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 20 May 2024 17:35:44 +0200 Subject: [PATCH 079/104] Update test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test --- .../cases/4004_jexl_expressions_in_subs/jexl_null_values.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test index 35bfbe4567..2027736e1a 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test @@ -86,7 +86,7 @@ echo "03. Update entity E1 with A=null again (with forcedUpdate)" echo "==========================================================" payload='{ "A": { - "value": "null", + "value": null, "type": "Text" } }' From 12c4574b842146f60792914205e34c4e3f827a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 May 2024 09:38:18 +0200 Subject: [PATCH 080/104] ADD jexl_transformation_full.test --- .../jexl_transformation_full.test | 365 ++++++++++++++++++ ...n.test => jexl_transformation_simple.test} | 0 2 files changed, 365 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test rename test/functionalTest/cases/4004_jexl_expressions_in_subs/{jexl_transformation.test => jexl_transformation_simple.test} (100%) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test new file mode 100644 index 0000000000..7508139a98 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -0,0 +1,365 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using all transformations) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub using all transformations +# 02. Create entity E1 with A to P attributes +# 03. Dump accumulator and see expected notification +# + + +echo "01. Create custom sub using all transformations" +echo "===============================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + }, + "notification": { + "attrs": [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P" + ], + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "A": { + "type": "Text", + "value": "${A+(3|toString)}" + }, + "B": { + "type": "Text", + "value": "${B|replaceStr('\''A'\'','\''B'\'')}" + }, + "C": { + "type": "Number", + "value": "${C+1}" + }, + "D": { + "type": "Text", + "value": "${D|uppercase}" + }, + "E": { + "type": "TextUnrestricted", + "value": "${E|lowercase}" + }, + "F": { + "type": "Text", + "value": "${F[1]|trim}" + }, + "G": { + "type": "Number", + "value": "${G|toFixed(1)}" + }, + "H": { + "type": "Text", + "value": "${H|includes('\''N'\'')}" + }, + "I": { + "type": "Text", + "value": "${I|indexOf('\''test'\'')}" + }, + "J": { + "type": "Text", + "value": "${(J|isNaN)}" + }, + "K": { + "type": "Text", + "value": "${K|typeOf}" + }, + "L": { + "type": "TextUnrestricted", + "value": "${(L|split('\'' '\''))}" + }, + "M": { + "type": "Number", + "value": "${M|round(2)}" + }, + "N": { + "type": "Text", + "value": "${N|substring(1,2)}" + }, + "O": { + "type": "Text", + "value": "${Z||'\''Is null'\''}" + }, + "P": { + "type": "Number", + "value": "${P|len}" + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A to P attributes" +echo "===========================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "type": "Text", + "value": "NA" + }, + "B": { + "type": "Text", + "value": "NA" + }, + "C": { + "type": "Number", + "value": 0.17 + }, + "D": { + "type": "Text", + "value": "rice" + }, + "E": { + "type": "TextUnrestricted", + "value": "NA" + }, + "F": { + "type": "Json", + "value": [ + "Dolot sequitud", + " trimable" + ], + "metadata": {} + }, + "G": { + "type": "Number", + "value": 0.85 + }, + "H": { + "type": "Text", + "value": "NA" + }, + "I": { + "type": "Text", + "value": "Ipsum test", + "metadata": {} + }, + "J": { + "type": "Text", + "value": "NA" + }, + "K": { + "type": "Text", + "value": "NA" + }, + "L": { + "type": "Json", + "value": "Lorem limoso" + }, + "M": { + "type": "Number", + "value": 0.3 + }, + "N": { + "type": "Text", + "value": "NA" + }, + "O": { + "type": "Text", + "value": null + }, + "P": { + "type": "Text", + "value": "NA" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Dump accumulator and see expected notification" +echo "==================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub using all transformations +=============================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A to P attributes +=========================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Dump accumulator and see expected notification +================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 882 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Text", + "value": "NA3" + }, + "B": { + "metadata": {}, + "type": "Text", + "value": "NB" + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 1.17 + }, + "D": { + "metadata": {}, + "type": "Text", + "value": "RICE" + }, + "E": { + "metadata": {}, + "type": "TextUnrestricted", + "value": "na" + }, + "F": { + "metadata": {}, + "type": "Text", + "value": "trimable" + }, + "G": { + "metadata": {}, + "type": "Number", + "value": 0.8 + }, + "H": { + "metadata": {}, + "type": "Text", + "value": true + }, + "I": { + "metadata": {}, + "type": "Text", + "value": 6 + }, + "J": { + "metadata": {}, + "type": "Text", + "value": true + }, + "K": { + "metadata": {}, + "type": "Text", + "value": "String" + }, + "L": { + "metadata": {}, + "type": "TextUnrestricted", + "value": [ + "Lorem", + "limoso" + ] + }, + "M": { + "metadata": {}, + "type": "Number", + "value": 0 + }, + "N": { + "metadata": {}, + "type": "Text", + "value": "A" + }, + "O": { + "metadata": {}, + "type": "Text", + "value": "Is null" + }, + "P": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test similarity index 100% rename from test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test rename to test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test From 62d1a22ede43c3eb015ebafcfd39bc7b8fffc6eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 May 2024 16:15:15 +0200 Subject: [PATCH 081/104] FIX remove FIXME PR mark --- src/lib/expressions/ExprResult.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 5f34023565..eec2e415d6 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -273,8 +273,6 @@ std::string ExprResult::toString(void) } else if (valueType == orion::ValueTypeString) { - // FIXME PR: does this break the no legacy - //return "\"" + toJsonString(stringValue) + "\""; return "\"" + stringValue + "\""; } else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) From 21d8f2868c183346cd515f9d6e888c647611c204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 11:38:02 +0200 Subject: [PATCH 082/104] FIX CMakeList.txt --- src/lib/expressions/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index 4f938e3634..27346743a0 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES ExprManager.cpp From d5a14947f18893c5e25e25944e8b4661057d2ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 14:07:45 +0200 Subject: [PATCH 083/104] FIX conditional testing depending on expr flavour --- src/app/contextBroker/contextBroker.cpp | 6 +++ .../cases/0000_cli/version.test | 2 +- .../jexl_array_filtering.test | 1 + .../jexl_basic_attrs_in_update.test | 1 + .../jexl_basic_attrs_not_in_update.test | 1 + .../jexl_expr_attrs_weird_syntax.test | 1 + .../jexl_json_navigation.test | 1 + .../jexl_missing_attribute_as_null.test | 1 + .../jexl_missing_attribute_as_null2.test | 1 + .../jexl_not_defined_attrs.test | 1 + .../jexl_null_values.test | 1 + .../jexl_several_expressions.test | 1 + .../jexl_syntax_errors.test | 1 + .../jexl_transformation_full.test | 1 + .../jexl_transformation_in_id_and_type.test | 1 + .../jexl_transformation_simple.test | 1 + .../jexl_transformation_unknown.test | 1 + .../jexl_transformation_with_arguments.test | 1 + test/functionalTest/testHarness.sh | 50 +++++++++++++++++++ 19 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index ec994937fa..413c63a85b 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -1045,6 +1045,12 @@ int main(int argC, char* argV[]) std::string versionString = std::string(ORION_VERSION) + " (git version: " + GIT_HASH + ")"; + #ifdef EXPR_BASIC + versionString += " flavours: basic-expr"; + #else + versionString += " flavours: jexl-expr"; + #endif + paConfig("man synopsis", (void*) "[options]"); paConfig("man shortdescription", (void*) "Options:"); paConfig("man description", (void*) description); diff --git a/test/functionalTest/cases/0000_cli/version.test b/test/functionalTest/cases/0000_cli/version.test index bac1621329..8d034ea9f2 100644 --- a/test/functionalTest/cases/0000_cli/version.test +++ b/test/functionalTest/cases/0000_cli/version.test @@ -30,7 +30,7 @@ broker version contextBroker --version --REGEXPECT-- -REGEX(\d+\.\d+\.\d+.* \(git version: .*\)) +REGEX(\d+\.\d+\.\d+.* \(git version: .*\) flavours:.*) Copyright 2013-2024 Telefonica Investigacion y Desarrollo, S.A.U Orion Context Broker is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test index d6428e2074..7762f53a9a 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (array filtering) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test index 6e31d9b36e..3bc2121392 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL basic expression in custom notification (source attributes in udpate) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test index f6c634a077..7a284c23aa 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL basic expression in custom notification (source attributes not in udpate) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test index 79ecfd9eeb..862f671a54 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression using attributes with weird syntax diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test index 05e8a0b061..f286e7f2c8 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (JSON navigation in objects and arrays) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test index 9d711fc6a7..d90db1f687 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (missing attribute replaced as null) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test index 8c0fd939f4..2d8d503914 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (missing attribute replaced as null additional case) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test index 4fe7de3997..3af9b1bdef 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (not defined attributes) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test index 2027736e1a..070ab89ce8 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (null values) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test index 82ffb1ccaa..b1b06e070a 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (several expressions in same subscription) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test index 4359b235bc..0804b22c8d 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification with syntax errors diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index 7508139a98..decc289255 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using all transformations) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test index fc128a4e4d..d1a0fca669 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using transformation in id and type) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test index 693818d970..05ffeb6471 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using transformation) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test index 3d536559b4..c4ad0adf67 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using unknown transformation) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test index 5616763fde..2bc3422ef1 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using transformation with arguments) diff --git a/test/functionalTest/testHarness.sh b/test/functionalTest/testHarness.sh index c309c4e8c2..9b30488480 100755 --- a/test/functionalTest/testHarness.sh +++ b/test/functionalTest/testHarness.sh @@ -96,6 +96,8 @@ declare -A skipV typeset -i skips declare -A disabledTestV typeset -i disabledTests +declare -A notInFlavourTestV +typeset -i notInFlavourTests export DIFF=$SCRIPT_HOME/testDiff.py testError=0 @@ -104,6 +106,7 @@ okOnThird=0 okOnPlus3=0 skips=0 disabledTests=0 +notInFlavourTests=0 # ----------------------------------------------------------------------------- @@ -937,6 +940,30 @@ function testDisabled +# ----------------------------------------------------------------------------- +# +# testMatchExprFlavour +# +function testMatchExprFlavour +{ + testcase=$1 + + if grep -q JEXL_EXPR_FLAVOUR $testcase + then + if $(contextBroker --version | grep -q jexl-expr) + then + echo NOT Disabled + else + echo "Disabled" + echo "Disabled" > /tmp/valgrind.out + fi + else + echo NOT Disabled + fi +} + + + # ------------------------------------------------------------------------------ # # Main loop @@ -973,6 +1000,17 @@ do continue fi + # + # Should the test be skipped due to it doesn't mach in the contextBroker flavour? + # + notInFlavour=$(testMatchExprFlavour $testFile) + if [ "$notInFlavour" == "Disabled" ] + then + notInFlavourTestV[$notInFlavourTests]=$testNo': '$testFile + notInFlavourTests=$notInFlavourTests+1 + continue + fi + if [ "$ixList" != "" ] then hit=$(echo ' '$ixList' ' | grep ' '$testNo' ') @@ -1190,4 +1228,16 @@ then done fi +if [ $notInFlavourTests != 0 ] +then + echo + echo WARNING: $notInFlavourTests test cases were not executed due to contexBroker not matching flavour: + ix=0 + while [ $ix -lt $notInFlavourTests ] + do + echo " o " ${notInFlavourTestV[$ix]} + ix=$ix+1 + done +fi + exit $exitCode From 68425b99f648bcb61ab5c52fd3100ba71e7ba102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 15:06:42 +0200 Subject: [PATCH 084/104] FIX to make basic behaviour the same than jexl regarding null --- src/lib/expressions/ExprManager.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 47e492f223..c56587cdf3 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -94,8 +94,18 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st std::map::iterator iter = replacementsP->find(_expression); if (iter != replacementsP->end()) { - r.valueType = orion::ValueTypeString; r.stringValue = iter->second; + + // To have the same behaviour than in JEXL case + if (r.stringValue == "null") + { + r.valueType = orion::ValueTypeNull; + } + else + { + r.valueType = orion::ValueTypeString; + } + LM_T(LmtExpr, ("basic evaluation result: <%s>", r.stringValue.c_str())); } TIME_EXPR_BASIC_EVAL_STOP(); From 42c3a9ac377e7753995cdbd87eec76ce5647c2e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 May 2024 17:49:16 +0200 Subject: [PATCH 085/104] ADD more test for null values cases --- .../jexl_null_values_all_cases.test | 269 ++++++++++++++ ...exl_null_values_all_cases_covered_sub.test | 276 ++++++++++++++ .../null_values_all_cases.test | 340 +++++++++++++++++ .../null_values_all_cases_covered_sub.test | 347 ++++++++++++++++++ 4 files changed, 1232 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test new file mode 100644 index 0000000000..f40c7e825a --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test @@ -0,0 +1,269 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +Expression in custom notification (null values in all cases with JEXL expression fail) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create custom sub using payload: "${A|patata}" +# 02. Create custom sub using payload: "X:${A|patata}:Y" +# 03. Create custom sub using json: "${A|patata}" and "X:${A|patata}:Y" +# 04. Create custom sub using ngsi: "${A|patata}" and "X:${A|patata}:Y" +# 05. Create entity E1 with A=1 +# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# + + +echo '01. Create custom sub using payload: "${A|patata}"' +echo "==================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "${A|patata}" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '02. Create custom sub using payload: "X:${A|patata}:Y"' +echo "======================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "X:${A|patata}:Y" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Create custom sub using json: "${A|patata}" and "X:${A|patata}:Y"' +echo "=====================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "json": { + "B1": "${A|patata}", + "B2": "X:${A|patata}:Y" + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '04. Create custom sub with using ngsi: "${A|patata}" and "X:${A|patata}:Y"' +echo "==========================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B1": { + "value": "${A|patata}", + "type": "Calculated" + }, + "B2": { + "value": "X:${A|patata}:Y", + "type": "Calculated" + } + } + }, + "attrs": [ "B1", "B2" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" +echo "===========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create custom sub using payload: "${A|patata}" +================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create custom sub using payload: "X:${A|patata}:Y" +====================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom sub using json: "${A|patata}" and "X:${A|patata}:Y" +===================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create custom sub with using ngsi: "${A|patata}" and "X:${A|patata}:Y" +========================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test new file mode 100644 index 0000000000..0205830e13 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test @@ -0,0 +1,276 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +Expression in custom notification (null values in all cases with JEXL expression fail with covered subs) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create custom sub with covered using payload: "${A|patata}" +# 02. Create custom sub with covered using payload: "X:${A|patata}:Y" +# 03. Create custom sub with covered using json: "${A|patata}" and "X:${A|patata}:Y" +# 04. Create custom sub with covered using ngsi: "${A|patata}" and "X:${A|patata}:Y" +# 05. Create entity E1 with A=1 +# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# + + +echo '01. Create custom sub with covered using payload: "${A|patata}"' +echo "===============================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "${A|patata}" + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '02. Create custom sub with covered using payload: "X:${A|patata}:Y"' +echo "===================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "X:${A|patata}:Y" + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Create custom sub with covered using json: "${A|patata}" and "X:${A|patata}:Y"' +echo "==================================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "json": { + "B1": "${A|patata}", + "B2": "X:${A|patata}:Y" + } + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '04. Create custom sub with covered using ngsi: "${A|patata}" and "X:${A|patata}:Y"' +echo "==================================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B1": { + "value": "${A|patata}", + "type": "Calculated" + }, + "B2": { + "value": "X:${A|patata}:Y", + "type": "Calculated" + } + } + }, + "attrs": [ "B1", "B2" ], + "covered": true + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" +echo "===========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create custom sub with covered using payload: "${A|patata}" +=============================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create custom sub with covered using payload: "X:${A|patata}:Y" +=================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom sub with covered using json: "${A|patata}" and "X:${A|patata}:Y" +================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create custom sub with covered using ngsi: "${A|patata}" and "X:${A|patata}:Y" +================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test new file mode 100644 index 0000000000..85a6674b15 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test @@ -0,0 +1,340 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Expression in custom notification (null values in all cases) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create custom sub using payload: "${A}" +# 02. Create custom sub using payload: "X:${A}:Y" +# 03. Create custom sub using json: "${A}" and "X:${A}:Y" +# 04. Create custom sub using ngsi: "${A}" and "X:${A}:Y" +# 05. Create entity E1 with A=null +# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 07. Create entity E2 without A +# 08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# + + +echo '01. Create custom sub using payload: "${A}"' +echo "===========================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "${A}" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '02. Create custom sub using payload: "X:${A}:Y"' +echo "===============================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "X:${A}:Y" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Create custom sub using json: "${A}" and "X:${A}:Y"' +echo "=======================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "json": { + "B1": "${A}", + "B2": "X:${A}:Y" + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '04. Create custom sub with using ngsi: "${A}" and "X:${A}:Y"' +echo "============================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B1": { + "value": "${A}", + "type": "Calculated" + }, + "B2": { + "value": "X:${A}:Y", + "type": "Calculated" + } + } + }, + "attrs": [ "B1", "B2" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create entity E1 with A=null" +echo "================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" +echo "===========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +echo "07. Create entity E2 without A" +echo "==============================" +payload='{ + "id": "E2", + "type": "T", + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" +echo "===========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create custom sub using payload: "${A}" +=========================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create custom sub using payload: "X:${A}:Y" +=============================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom sub using json: "${A}" and "X:${A}:Y" +======================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create custom sub with using ngsi: "${A}" and "X:${A}:Y" +============================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create entity E1 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +07. Create entity E2 without A +============================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E2","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test new file mode 100644 index 0000000000..97d28912a4 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test @@ -0,0 +1,347 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Expression in custom notification (null values in all cases with covered subs) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create custom sub with covered using payload: "${A}" +# 02. Create custom sub with covered using payload: "X:${A}:Y" +# 03. Create custom sub with covered using json: "${A}" and "X:${A}:Y" +# 04. Create custom sub with covered using ngsi: "${A}" and "X:${A}:Y" +# 05. Create entity E1 with A=null +# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 07. Create entity E2 without A +# 08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# + + +echo '01. Create custom sub with covered using payload: "${A}"' +echo "========================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "${A}" + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '02. Create custom sub with covered using payload: "X:${A}:Y"' +echo "============================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "X:${A}:Y" + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Create custom sub with covered using json: "${A}" and "X:${A}:Y"' +echo "====================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "json": { + "B1": "${A}", + "B2": "X:${A}:Y" + } + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '04. Create custom sub with covered using json: "${A}" and "X:${A}:Y"' +echo "====================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B1": { + "value": "${A}", + "type": "Calculated" + }, + "B2": { + "value": "X:${A}:Y", + "type": "Calculated" + } + } + }, + "attrs": [ "B1", "B2" ], + "covered": true + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create entity E1 with A=null" +echo "================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset accumulator and see notifications" +echo "==================================================" +accumulatorDump +accumulatorReset +echo +echo + + +echo "07. Create entity E2 without A" +echo "==============================" +payload='{ + "id": "E2", + "type": "T", + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "08. Dump & reset accumulator and see notifications" +echo "==================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create custom sub with covered using payload: "${A}" +======================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create custom sub with covered using payload: "X:${A}:Y" +============================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom sub with covered using json: "${A}" and "X:${A}:Y" +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create custom sub with covered using json: "${A}" and "X:${A}:Y" +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create entity E1 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +06. Dump & reset accumulator and see notifications +================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +07. Create entity E2 without A +============================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +08. Dump & reset accumulator and see notifications +================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E2","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From fcb9304217302e720a4a3b295c03b562fba22deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 09:45:28 +0200 Subject: [PATCH 086/104] FIX null behaviour in custom notification payload field --- src/lib/ngsiNotify/Notifier.cpp | 2 +- ...fication_templates_many_notifications.test | 20 ++++---- .../covered_custom_notification.test | 18 +++---- .../jexl_null_values_all_cases.test | 25 +++++++--- ...exl_null_values_all_cases_covered_sub.test | 25 +++++++--- .../null_values_all_cases.test | 50 ++++++++++++++----- .../null_values_all_cases_covered_sub.test | 50 ++++++++++++++----- 7 files changed, 134 insertions(+), 56 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index df16cbb610..e26fc31e37 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -170,7 +170,7 @@ static bool setPayload } else { - if (!macroSubstitute(payloadP, notifPayload, exprContextObjectP, "", true)) + if (!macroSubstitute(payloadP, notifPayload, exprContextObjectP, "null", true)) { return false; } diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test index 341c55b7b9..4ffa8c252e 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test @@ -280,7 +280,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 44 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -292,8 +292,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { "A1": "true", - "A2": "", - "A3": "" + "A2": "null", + "A3": "null" } #SORT_END #SORT_START @@ -351,7 +351,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) PUT http://127.0.0.1:REGEX(\d+)/notify?id=E1&type=T2 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 32 +Content-Length: 44 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -361,9 +361,9 @@ Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "", - "A2": "", - "A3": "" + "A1": "null", + "A2": "null", + "A3": "null" } #SORT_END #SORT_START @@ -422,7 +422,7 @@ PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 Fiware-Servicepath: / A3: 3 Entity-Id: E1 -Content-Length: 33 +Content-Length: 41 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -432,8 +432,8 @@ Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "", - "A2": "", + "A1": "null", + "A2": "null", "A3": "3" } ======================================= diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test index 6d40e8e28b..14fffa6e0f 100644 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test @@ -36,7 +36,7 @@ accumulatorStart # 03. Create E1 with attribute A1=1 # 04. Dump & reset, see notifications A1=1 and A2=null in custom payload # 05. Create E2 with attribute A1=1 -# 06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) +# 06. Dump & reset, see notifications A1=1 and A2=null in custom payload # echo "01. Create covered custom subscriptions for E1 covering attributes A1 and A2" @@ -131,8 +131,8 @@ echo echo -echo "06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON)" -echo "========================================================================================" +echo "06. Dump & reset, see notifications A1=1 and A2=null in custom payload" +echo "======================================================================" accumulatorDump accumulatorReset echo @@ -174,7 +174,7 @@ Content-Length: 0 ====================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 31 +Content-Length: 35 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -182,7 +182,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": }======================================= +{ "A1-value": 1, "A2-value": null }======================================= 05. Create E2 with attribute A1=1 @@ -195,11 +195,11 @@ Content-Length: 0 -06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) -======================================================================================== +06. Dump & reset, see notifications A1=1 and A2=null in custom payload +====================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 31 +Content-Length: 35 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -207,7 +207,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": }======================================= +{ "A1-value": 1, "A2-value": null }======================================= --TEARDOWN-- diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test index f40c7e825a..aba513b32e 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test @@ -37,7 +37,7 @@ accumulatorStart # 03. Create custom sub using json: "${A|patata}" and "X:${A|patata}:Y" # 04. Create custom sub using ngsi: "${A|patata}" and "X:${A|patata}:Y" # 05. Create entity E1 with A=1 -# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 06. Dump & reset accumulator and see 4 notifications # @@ -167,8 +167,8 @@ echo echo -echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" -echo "===========================================================================================" +echo "06. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -226,8 +226,9 @@ Content-Length: 0 -06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) -=========================================================================================== +06. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -238,7 +239,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -261,6 +273,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END --TEARDOWN-- diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test index 0205830e13..6cf375ceae 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test @@ -37,7 +37,7 @@ accumulatorStart # 03. Create custom sub with covered using json: "${A|patata}" and "X:${A|patata}:Y" # 04. Create custom sub with covered using ngsi: "${A|patata}" and "X:${A|patata}:Y" # 05. Create entity E1 with A=1 -# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 06. Dump & reset accumulator and see 4 notifications # @@ -174,8 +174,8 @@ echo echo -echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" -echo "===========================================================================================" +echo "06. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -233,8 +233,9 @@ Content-Length: 0 -06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) -=========================================================================================== +06. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -245,7 +246,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -268,6 +280,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END --TEARDOWN-- diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test index 85a6674b15..3c7fe0ad03 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test @@ -36,9 +36,9 @@ accumulatorStart # 03. Create custom sub using json: "${A}" and "X:${A}:Y" # 04. Create custom sub using ngsi: "${A}" and "X:${A}:Y" # 05. Create entity E1 with A=null -# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 06. Dump & reset accumulator and see 4 notifications # 07. Create entity E2 without A -# 08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 08. Dump & reset accumulator and see 4 notifications # @@ -168,8 +168,8 @@ echo echo -echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" -echo "===========================================================================================" +echo "06. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -191,8 +191,8 @@ echo echo -echo "08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" -echo "===========================================================================================" +echo "08. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -250,8 +250,9 @@ Content-Length: 0 -06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) -=========================================================================================== +06. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -262,7 +263,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -285,6 +297,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END 07. Create entity E2 without A @@ -297,8 +310,9 @@ Content-Length: 0 -08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) -=========================================================================================== +08. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -309,7 +323,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -332,6 +357,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E2","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END --TEARDOWN-- diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test index 97d28912a4..562fb86085 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test @@ -36,9 +36,9 @@ accumulatorStart # 03. Create custom sub with covered using json: "${A}" and "X:${A}:Y" # 04. Create custom sub with covered using ngsi: "${A}" and "X:${A}:Y" # 05. Create entity E1 with A=null -# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 06. Dump & reset accumulator and see 4 notifications # 07. Create entity E2 without A -# 08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 08. Dump & reset accumulator and see 4 notifications # @@ -175,8 +175,8 @@ echo echo -echo "06. Dump & reset accumulator and see notifications" -echo "==================================================" +echo "06. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -198,8 +198,8 @@ echo echo -echo "08. Dump & reset accumulator and see notifications" -echo "==================================================" +echo "08. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -257,8 +257,9 @@ Content-Length: 0 -06. Dump & reset accumulator and see notifications -================================================== +06. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -269,7 +270,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -292,6 +304,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END 07. Create entity E2 without A @@ -304,8 +317,9 @@ Content-Length: 0 -08. Dump & reset accumulator and see notifications -================================================== +08. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -316,7 +330,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -339,6 +364,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E2","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END --TEARDOWN-- From ac2f915253ac5be4ef9638aad7ce1914a0e3ae7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 10:14:25 +0200 Subject: [PATCH 087/104] FIX ftest --- .../2015_notification_templates/notification_templates.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates.test b/test/functionalTest/cases/2015_notification_templates/notification_templates.test index 23df76a2fe..705c99c8bc 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates.test @@ -539,7 +539,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&id=E1&step=07&type=Thing Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 40 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Step: 07 Ngsiv2-Attrsformat: custom @@ -554,7 +554,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) { "A1": "a1", "A2": "a2", - "A4": "" + "A4": "null" } ======================================= #SORT_END From d039ce1e7f7c5b0f9bffc33f81526274172f08bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 10:24:23 +0200 Subject: [PATCH 088/104] ADD log, log10, log2 and sqrt tests --- .../jexl_transformation_full.test | 60 ++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index decc289255..b69cf370ba 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -67,7 +67,11 @@ payload='{ "M", "N", "O", - "P" + "P", + "Q", + "R", + "S", + "T" ], "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", @@ -135,6 +139,22 @@ payload='{ "P": { "type": "Number", "value": "${P|len}" + }, + "Q": { + "type": "Number", + "value": "${Q|log}" + }, + "R": { + "type": "Number", + "value": "${R|log10}" + }, + "S": { + "type": "Number", + "value": "${S|log2}" + }, + "T": { + "type": "Number", + "value": "${T|sqrt}" } } } @@ -218,6 +238,22 @@ payload='{ "P": { "type": "Text", "value": "NA" + }, + "Q": { + "type": "Number", + "value": 2.80 + }, + "R": { + "type": "Number", + "value": 100 + }, + "S": { + "type": "Number", + "value": 32 + }, + "T": { + "type": "Number", + "value": 25 } }' orionCurl --url /v2/entities --payload "$payload" @@ -257,7 +293,7 @@ Content-Length: 0 ================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 882 +Content-Length: 1076 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -351,6 +387,26 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Number", "value": 2 }, + "Q": { + "metadata": {}, + "type": "Number", + "value": 1.029619417 + }, + "R": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "S": { + "metadata": {}, + "type": "Number", + "value": 5 + }, + "T": { + "metadata": {}, + "type": "Number", + "value": 5 + }, "id": "E1", "type": "T" } From 13e20349565208c1d36ef6347c85b09bfb3a3ca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 11:00:24 +0200 Subject: [PATCH 089/104] FIX use cjexl 0.1.0 --- ci/deb/build.sh | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.alpine | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index 5a5393f877..4dffcd865b 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -175,7 +175,7 @@ rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - bash /opt/fiware-orion/get_cjexl.sh 0.0.0 $REPO_ACCESS_TOKEN + bash /opt/fiware-orion/get_cjexl.sh 0.1.0 $REPO_ACCESS_TOKEN fi if [ -n "${branch}" ]; then diff --git a/docker/Dockerfile b/docker/Dockerfile index 13033a9343..04394da083 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.0.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.1.0 ${REPO_ACCESS_TOKEN} && \ make && \ make install && \ # reduce size of installed binaries diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 2c4ed4a02e..5c302db67d 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -110,7 +110,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.0.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.1.0 ${REPO_ACCESS_TOKEN} && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ From 9cdcb5262a20ccba788b2fbe59f533e0de81bf0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 15:26:18 +0200 Subject: [PATCH 090/104] FIX memory leak --- src/lib/expressions/ExprManager.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index c56587cdf3..4a92d91e34 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -123,6 +123,10 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st // The ExprResult::fill() method allocates dynamic memory. So, the callers to evaluate() are supposed to invoke ExprResult::release() // method to free it r.fill(result); + + // cjexl_eval() allocated memory for us. We have to release it in order to avoid a leak + free((char*)result); + TIME_EXPR_JEXL_EVAL_STOP(); } From da9c9b51805a542ab2e6e590408420b0b78ac226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 16:41:04 +0200 Subject: [PATCH 091/104] ADD documentation --- CHANGES_NEXT_RELEASE | 9 +- doc/manuals/admin/build_source.md | 2 + doc/manuals/admin/statistics.md | 45 ++- doc/manuals/devel/sourceCode.md | 6 + doc/manuals/orion-api.md | 510 +++++++++++++++++++++++++++++- 5 files changed, 527 insertions(+), 45 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index c44ac67e3f..21a630772a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,7 +1,6 @@ -<<<<<<< HEAD -- Add: expression context build and evaluation counter in timing section in GET /statistics (#4004) -- Fix: lighter operation to get databases list from MongoDB (#4517) -======= +- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.1.0) (#4004) +- Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) +- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviur more consistent - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) ->>>>>>> master + diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index 64064030e4..3292e71d32 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -4,6 +4,8 @@ Orion Context Broker reference distribution is Debian 12. This doesn't mean that You can also have a look to [3.1 Building in not official distributions](../../../docker/README.md#31-building-in-not-official-distributions) section in the Docker documentation to check how to build Docker containers images in distributions other than the official one. +*NOTE:* the build process described in this document does not include the cjexl library, as it is considered optional from the point of view of the basic building process. + ## Debian 12 (officially supported) The Orion Context Broker uses the following libraries as build dependencies: diff --git a/doc/manuals/admin/statistics.md b/doc/manuals/admin/statistics.md index 7a0eb4f487..a44bafcb74 100644 --- a/doc/manuals/admin/statistics.md +++ b/doc/manuals/admin/statistics.md @@ -129,28 +129,22 @@ Provides timing information, i.e. the time that CB passes executing in different "timing": { "accumulated": { "jsonV1Parse": 7.860908311, - "mongoBackend": 416.796091597, - "mongoReadWait": 4656.924425628, - "mongoWriteWait": 259.347915990, - "mongoCommandWait": 0.514811318, - "exprBasicCtxBld": FIXME PR, - "exprBasicEval": FIXME PR, - "exprJexlCtxBld": FIXME PR, - "exprJexlEval": FIXME PR, - "render": 108.162782114, - "total": 6476.593504743 - }, + "jsonV2Parse": 120.680244446, + "mongoBackend": 12778.52734375, + "mongoReadWait": 7532.301757812, + "mongoWriteWait": 3619.282226562, + "mongoCommandWait": 0.120559767, + "exprJexlCtxBld": 27.092681885, + "exprJexlEval": 124.217208862, + "render": 44.540554047, + "total": 25051.384765625 + }, "last": { - "mongoBackend": 0.014752309, - "mongoReadWait": 0.012018445, - "mongoWriteWait": 0.000574611, - "exprBasicCtxBld": FIXME PR, - "exprBasicEval": FIXME PR, - "exprJexlCtxBld": FIXME PR, - "exprJexlEval": FIXME PR, - "render": 0.000019136, - "total": 0.015148915 - } + "mongoBackend": 0.003775352, + "mongoReadWait": 0.0013743, + "render": 0.000286864, + "total": 0.00440685 + } } ... } @@ -164,8 +158,6 @@ The block includes two main sections: The particular counters are as follows: -FIXME PR: explain expr* fields - * `total`: processing time for the whole request, excluding the time that the HTTP library takes for request/response dispatching (pseudo end-to-end time) * `jsonV1Parse`: time passed in NGSIv1 JSON parsing module (pseudo self-time) @@ -177,10 +169,15 @@ FIXME PR: explain expr* fields `last` includes the accumulation for all of them. In the case of mongoReadWait, only the time used to get the results cursor is taken into account, but not the time to process cursors results (which is time that belongs to mongoBackend counters). +* `exprJexlCtxBld`: time passed building context for custom notification expression evaluation (see [macro substitution](../orion-api.md#macro-substitution) and [JEXL support](../orion-api.md#jexl-support)) +* `exprJexlEval`: time passed evaluating custom notification expressions (see [macro substitution](../orion-api.md#macro-substitution) and [JEXL support](../orion-api.md#jexl-support)) + +*NOTE*: if Orion binary is build without using cjexl and only basic replacement is available, then `exprBasicCtxtBld` and `exprBasicEval` +fields appear instead of `exprJexlCtxBld` and `exprJexlEval`. Times are measured from the point in time in which a particular thread request starts using a module until it finishes using it. Thus, if the thread is stopped for some reason (e.g. the kernel decides to give priority to another thread based on its -scheculing policy) the time that the thread was sleeping, waiting to execute again is included in the measurement and thus, the measurement is not accurate. That is why we say *pseudo* selt/end-to-end time. However, +scheculing policy) the time that the thread was sleeping, waiting to execute again is included in the measurement and thus, the measurement is not accurate. That is why we say *pseudo* self/end-to-end time. However, under low load conditions this situation is not expected to have a significant impact. ### NotifQueue block diff --git a/doc/manuals/devel/sourceCode.md b/doc/manuals/devel/sourceCode.md index c9227740b0..951e8777f8 100644 --- a/doc/manuals/devel/sourceCode.md +++ b/doc/manuals/devel/sourceCode.md @@ -24,6 +24,7 @@ * [src/lib/cache/](#srclibcache) (Subscription cache implementation) * [src/lib/logSummary/](#srcliblogsummary) (Log Summary implementation) * [src/lib/metricsMgr/](#srclibmetricsmgr) (Metrics Manager implementation) +* [src/lib/expressions/](#seribexpressions) (Custom notification expressions support) ## src/app/contextBroker/ The main program is found in `contextBroker.cpp` and its purpose it to: @@ -551,3 +552,8 @@ This Metrics Manager resides in the library **metricsMgr**. For information about the metrics, please refer to [this document](../admin/metrics_api.md). [Top](#top) + +## src/lib/expressions/ +Provides support to the [macro substition logic used by custom notifications](../orion-api.md#macro-substitution). This library provides an abstraction for expression evaluation, providing two implementations: JEXL based and basic replacement based (the implementation to use is choosen at building time, based on the availability of the cjex library). + +[Top](#top) \ No newline at end of file diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 5e8e6e9bef..ea23b5f764 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -73,6 +73,32 @@ - [NGSI payload patching](#ngsi-payload-patching) - [Omitting payload](#omitting-payload) - [Additional considerations](#additional-considerations) + - [JEXL Support](#jexl-support) + - [Available Transformations](#available-transformations) + - [`uppercase`](#uppercase) + - [`lowercase`](#lowercase) + - [`split`](#split) + - [`indexOf`](#indexOf) + - [`len`](#len) + - [`trim`](#trim) + - [`substring`](#substring) + - [`includes`](#includes) + - [`isNaN`](#isNaN) + - [`parseInt`](#parseInt) + - [`parseFloat`](#parseFloat) + - [`typeOf`](#typeOf) + - [`toString`](#toString) + - [`floor`](#floor) + - [`ceil`](#ceil) + - [`round`](#round) + - [`toFixed`](#toFixed) + - [`log`](#log) + - [`log10`](#log10) + - [`log2`](#log2) + - [`sqrt`](#sqrt) + - [`replaceStr`](#replaceStr) + - [Failsafe cases](#failsafe-cases) + - [Known limitations](#known-limitations) - [Oneshot Subscriptions](#oneshot-subscriptions) - [Covered Subscriptions](#covered-subscriptions) - [Subscriptions based in alteration type](#subscriptions-based-in-alteration-type) @@ -2022,23 +2048,22 @@ In case of `mqttCustom`: * `payload`, `json` and `ngsi` (all them payload related fields) * `topic` -Macro substitution for templates is based on the syntax `${..}`. In particular: +Macro substitution for templates is based on the syntax `${}`. The support to JEXL +is explained in [JEXL Support](#jexl-support) section. The following identifiers are included in +the context evaluated by the JEXL expression: -* `${id}` is replaced by the `id` of the entity -* `${type}` is replaced by the `type` of the entity -* `${service}` is replaced by the service (i.e. `fiware-service` header value) in the +* `id`: for the `id` of the entity +* `type`: for the `type` of the entity +* `service`: for the service (i.e. `fiware-service` header value) in the update request triggering the subscription. -* `${servicePath}` is replaced by the service path (i.e. `fiware-servicepath` header value) in the +* `servicePath`: for the service path (i.e. `fiware-servicepath` header value) in the update request triggering the subscription. -* `${authToken}` is replaced by the authorization token (i.e. `x-auth-token` header value) in the +* `authToken: for the authorization token (i.e. `x-auth-token` header value) in the update request triggering the subscription. -* Any other `${token}` is replaced by the value of the attribute whose name is `token` or with - an empty string if the attribute is not included in the notification. If the value is a number, - a bool or null then its string representation is used. If the value is a JSON array or object - then its JSON representation as string is used. +* All the attributes in the entity triggering the notification (included in the update triggering the notification or not) -In the rare case an attribute was named in the same way of the `${service}`, `${servicePath}` or -`${authToken}` (e.g. an attribute which name is `service`) then the attribute value takes precedence. +In the rare case an attribute was named in the same way of the `service`, `servicePath` or +`authToken` (e.g. an attribute which name is `service`) then the attribute value takes precedence. Example: @@ -2296,6 +2321,463 @@ Some considerations to take into account when using custom notifications: (i.e. `ngsi` field) then `Ngsiv2-AttrsFormat: normalized` is used, as in a regular notification (given that the notification format is actually the same). +## JEXL Support + +Orion Context Broker supports [JEXL expressions](https://github.com/TomFrost/Jexl) in custom notification [macro replacement](#macro-substitution). Thus, subscriptions like this can be defined: + +``` +"httpCustom": { + ... + "ngsi": { + "relativeHumidity": { + "value": "${humidity/100}", + "type": "Calculated" + } + } +} +``` + +So, if a given update sets entity `humidity` attribute to `84.4` then the notification will include a `relativeHumidity` attribute with value `0.844`. + +A particular case of expressions are the ones in which the expression is a given context identifier, without an actual expression using it. For instance: + +``` +"httpCustom": { + ... + "ngsi": { + "originalHumidity": { + "value": "${humidity}", + "type": "Calculated" + } + } +} +``` + +We also refers to this case as *basic replacement*. + +An useful resource to test JEXL expressions is the [JEXL playground](https://czosel.github.io/jexl-playground). However, take into account the differences between the original JEXL implementation in JavaScript and the one included in Orion, described in the [known limitations](#known-limitations) section. + +Orion relies on cjexl library to provide this functionality. If Orion binary is build without using cjexl, then only basic replacement functionality is available. + +### Available Transformations + +#### `uppercase` + +Convert a string into uppercase. + +Extra arguments: none + +Example (being context `{"c": "fooBAR"}`): + +``` +c|uppercase +``` + +results in + +``` +"FOOBAR" +``` + +#### lowercase + +Convert a string into lowercase. + +Extra arguments: none + +Example (being context `{"c": "fooBAR"}`): + +``` +c|lowercase +``` + +results in + +``` +"foobar" +``` + +#### split + +Split the input string into array items. + +Extra arguments: delimiter to use for the split. + +Example (being context `{"c": "foo,bar,zzz"}`): + +``` +c|split(',') +``` + +results in + +``` +[ "foo", "bar", "zzz" ] +``` + +#### indexOf + +Provides the position of a given string within the input string. In the string is not found, returns `null`. + +Extra arguments: the input string to search. + +Note this function doesn't work if the input is an array (it only works for strings). + +Example (being context `{"c": "fooxybar"}`): + +``` +c|indexOf('xy') +``` + +results in + +``` +3 +``` + +#### len + +Provides the length of a string. + +Extra arguments: none. + +Note this function doesn't work if the input is an array (it only works for strings). + +Example (being context `{"c": "foobar"}`): + +``` +c|len +``` + +results in + +``` +6 +``` + +#### trim + +Removes starting and trailing whitespaces. + +Extra arguments: none. + +Example (being context `{"c": " foo bar "}`): + +``` +c|trim +``` + +results in + +``` +foo bar +``` + +#### substring + +Returns a substring between two positions. + +Extra arguments: +* Initial position +* Final position + +Example (being context `{"c": "foobar"}`): + +``` +c|substring(3,5) +``` + +results in + +``` +ba +``` + +#### includes + +Returns `true` if a given string is contained in the input string, `false` otherwise. + +Extra arguments: the input string to search. + +Example (being context `{"c": "foobar"}`): + +``` +c|includes('ba') +``` + +results in + +``` +true +``` + +#### isNaN + +Returns `true` if the input is not a number, `false` otherwise. + +Extra arguments: none. + +Example (being context `{"c": "foobar"}`): + +``` +c|isNaN +``` + +results in + +``` +true +``` + +#### parseInt + +Parses a string and return the corresponding integer number. + +Extra arguments: none. + +Example (being context `{"c": "25"}`): + +``` +c|parseInt +``` + +results in + +``` +25 +``` + +#### parseFloat + +Parses a string and return the corresponding float number + +Extra arguments: none. + +Example (being context `{"c": "25.33"}`): + +``` +c|parseFloat +``` + +results in + +``` +25.33 +``` + +#### typeOf + +Return a string with the type of the input. + +Extra arguments: none. + +Example (being context `{"c": 23}`): + +``` +c|typeOf +``` + +results in + +``` +"Number" +``` + +#### toString + +Return a string representation of the input. + +Extra arguments: none. + +Example (being context `{"c": 23}`): + +``` +c|toString +``` + +results in + +``` +"23" +``` + +#### floor + +Return the closest lower integer of a given number. + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|floor +``` + +results in + +``` +3 +``` + +#### ceil + +Return the closest upper integer of a given number. + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|ceil +``` + +results in + +``` +4 +``` + +#### round + +Return the closest integer (either lower or upper) of a given number. + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|round +``` + +results in + +``` +3 +``` + +#### toFixed + +Rounds a number to a number of decimal digits. + +Extra arguments: number of decimal digits. + +Example (being context `{"c": 3.18}`): + +``` +c|toFixed(1) +``` + +results in + +``` +3.2 +``` + +#### log + +Return the natural logarithm of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|log +``` + +results in + +``` +1.144222799920162 +``` + +#### log10 + +Return the base 10 logarithm of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|log10 +``` + +results in + +``` +0.49692964807321494 +``` + +#### log2 + +Return the base 2 logarithm of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|log2 +``` + +results in + +``` +1.6507645591169025 +``` + +#### sqrt + +Return the square root of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|sqrt +``` + +results in + +``` +1.772004514666935 +``` + +#### replaceStr + +Replace occurrences of a string with another in the input string. + +Extra arguments: +* Source string to replace +* Destination string to replace + +Example (being context `{"c": "foobar"}`): + +``` +c|replaceStr('o','u') +``` + +results in + +``` +"fuubar" +``` + +### Failsafe cases + +As failsafe behaviour, evaluation returns `null` in the following cases: + +* Some of the transformation used in the expression is unknown +* Operations with identifiers that are not defined in the context are used. For instance, `(A==null)?-1:A` will result in `null` (and not `-1`) if `A` is not in the context, due to `==` is an operation that cannot be done on undefined identifiers. However, `A||-1` will work (i.e. `-1` will result if `A` is not in the context), as `||` is not considered an operation on `A`. +* Syntax error in the JEXL expression (e.g. `A[0|uppercase`) + +### Known limitations + +- The unitary minus operator is not working properly, e.g. the following expression doesn't work (it failsafes to `null`): `A||-1`. However, the following alternatives are working: `A||0-1` and `A||'-1'|parseInt)` +- Negation operator `!` (supported in original JavaScript JEXL) is not supported + ## Oneshot Subscriptions Oneshot subscription provides an option to subscribe an entity only for one time notification. When consumer creates a subscription @@ -2395,10 +2877,6 @@ in which case all attributes are included in the notification, no matter if they entity. For these attributes that don't exist (`brightness` in this example) the `null` value (of type `"None"`) is used. -In the case of custom notifications, if `covered` is set to `true` then `null` will be used to replace `${...}` -for non existing attributes (the default behavior when `covered` is not set to `true` is to replace by the -empty string the non existing attributes). - We use the term "covered" in the sense the notification "covers" completely all the attributes in the `notification.attrs` field. It can be useful for those notification endpoints that are not flexible enough for a variable set of attributes and needs always the same set of incoming attributes From 826ffdd4bcb917de04d14e06eb89d9cc9c4042f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 16:47:30 +0200 Subject: [PATCH 092/104] FIX basic replacement logic --- src/lib/expressions/ExprManager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 4a92d91e34..cd1a3d5182 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -124,8 +124,11 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st // method to free it r.fill(result); +#ifndef EXPR_BASIC // cjexl_eval() allocated memory for us. We have to release it in order to avoid a leak + // (disbled with EXPR_BASIC because in that case result is static memory) free((char*)result); +#endif TIME_EXPR_JEXL_EVAL_STOP(); } From 9e04d8c5a658daed8530ae45ee9794e87cd36d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 17:03:44 +0200 Subject: [PATCH 093/104] FIX warning about special chars --- doc/manuals/orion-api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index ea23b5f764..985db83d48 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -455,6 +455,8 @@ In addition, the [General syntax restrictions](#general-syntax-restrictions) als In case a client attempts to use a field that is invalid from a syntax point of view, the client gets a "Bad Request" error response, explaining the cause. +Note that although `:` and `-` are allowed in identifiers, they are strongly discouraged, as they collide with the [JEXL syntax](#jexl-support). In particular, `-` is used for subtraction operation (e.g. `${A-B}`) and `:` is used in the ternary operator (eg. `A?'A is true':'A is false`). Thus, an attribute name `lower-temperature` in an expression `${lower-temperature}` would be interpreted as the value of `lower` attribute minus `temperature` attribute (and not as the value of an attribute named `lower-temperature`). + ## Error Responses If present, the error payload is a JSON object including the following fields: From 8587216a1f129a473a6318253b6653582fe32008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 17:06:19 +0200 Subject: [PATCH 094/104] FIX enhance CNR --- CHANGES_NEXT_RELEASE | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 21a630772a..0f5194da98 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,6 +1,7 @@ - Add: JEXL expression support in custom notification macro replacement (using cjexl 0.1.0) (#4004) - Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) -- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviur more consistent +- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004) +- Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) From 90ca88eb934ad0d8c7c9233d36dbb638fc76fe63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 May 2024 12:20:31 +0200 Subject: [PATCH 095/104] ADD mapper and thMapper transformations tests --- doc/manuals/orion-api.md | 46 +++++++++++++++++++ .../jexl_transformation_full.test | 38 +++++++++++++-- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 985db83d48..57e3396481 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -97,6 +97,8 @@ - [`log2`](#log2) - [`sqrt`](#sqrt) - [`replaceStr`](#replaceStr) + - [`mapper`](#mapper) + - [`thMapper`](#thmapper) - [Failsafe cases](#failsafe-cases) - [Known limitations](#known-limitations) - [Oneshot Subscriptions](#oneshot-subscriptions) @@ -2767,6 +2769,50 @@ results in "fuubar" ``` +#### mapper + +Returns a value among several choices based in one to one mapping. This function is based in an array of *values* and an array of *choices* (which length is exactly the same). Thus, if the input value is equal to the *i*-th item of *values*, then *i*-th item of *choices* is returned. + +This transformation returns `null` if some problem with the arguments is found (i.e. input is not found among the values, choices length is not exacly the same as values, the input is not an string, etc.) + +Extra arguments: +* values array +* choices array + +Example (being context `{"c": "fr", "values": ["es", "fr", "de"], "choices": ["Spain", "France", "Germany"]}`): + +``` +c|mapper(values,choices) +``` + +results in + +``` +"France" +``` + +#### thMapper + +Returns a value among several choices based in threshold values. This function is based in an array of *values* and an array of *choices* (which length is exactly the same as values plus one). Thus, if the input value is between the *i*-th and the *i+1*-th item of *values*, then *i*+1-th item of *choices* is returned. + +This transformation returns `null` if some problem with the arguments is found (i.e. choices length is not exacly the same as values plus one, some of the items in the values array is not a number, etc.) + +Extra arguments: +* values array +* choices array + +Example (being context `{"c": 0.5, "values": [-1, 1], "choices": ["low", "medium", "high"]}`): + +``` +c|thMapper(values,choices) +``` + +results in + +``` +"medium" +``` + ### Failsafe cases As failsafe behaviour, evaluation returns `null` in the following cases: diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index b69cf370ba..d4b83278e1 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -33,7 +33,7 @@ accumulatorStart --pretty-print # # 01. Create custom sub using all transformations -# 02. Create entity E1 with A to P attributes +# 02. Create entity E1 with A to V attributes # 03. Dump accumulator and see expected notification # @@ -71,7 +71,9 @@ payload='{ "Q", "R", "S", - "T" + "T", + "U", + "V" ], "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", @@ -155,6 +157,14 @@ payload='{ "T": { "type": "Number", "value": "${T|sqrt}" + }, + "U": { + "type": "Text", + "value": "${U|mapper(['\''es'\'','\''fr'\''],['\''Spain'\'','\''France'\''])}" + }, + "V": { + "type": "Text", + "value": "${V|thMapper([1,2],['\''low'\'','\''medium'\'','\''high'\''])}" } } } @@ -165,7 +175,7 @@ echo echo -echo "02. Create entity E1 with A to P attributes" +echo "02. Create entity E1 with A to V attributes" echo "===========================================" payload='{ "id": "E1", @@ -254,6 +264,14 @@ payload='{ "T": { "type": "Number", "value": 25 + }, + "U": { + "type": "Text", + "value": "fr" + }, + "V": { + "type": "Number", + "value": 1.5 } }' orionCurl --url /v2/entities --payload "$payload" @@ -279,7 +297,7 @@ Content-Length: 0 -02. Create entity E1 with A to P attributes +02. Create entity E1 with A to V attributes =========================================== HTTP/1.1 201 Created Date: REGEX(.*) @@ -293,7 +311,7 @@ Content-Length: 0 ================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 1076 +Content-Length: 1178 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -407,6 +425,16 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Number", "value": 5 }, + "U": { + "metadata": {}, + "type": "Text", + "value": "France" + }, + "V": { + "metadata": {}, + "type": "Text", + "value": "medium" + }, "id": "E1", "type": "T" } From e3a3f4584dd548727006c3b771fc788795ac0375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 May 2024 12:36:18 +0200 Subject: [PATCH 096/104] FIB use cjexl 0.2.0 --- CHANGES_NEXT_RELEASE | 2 +- ci/deb/build.sh | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.alpine | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0f5194da98..ae44980113 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.1.0) (#4004) +- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.2.0) (#4004) - Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) - Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004) - Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index 4dffcd865b..b0e5027b8a 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -175,7 +175,7 @@ rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - bash /opt/fiware-orion/get_cjexl.sh 0.1.0 $REPO_ACCESS_TOKEN + bash /opt/fiware-orion/get_cjexl.sh 0.2.0 $REPO_ACCESS_TOKEN fi if [ -n "${branch}" ]; then diff --git a/docker/Dockerfile b/docker/Dockerfile index 04394da083..e6c7846b56 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.1.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.2.0 ${REPO_ACCESS_TOKEN} && \ make && \ make install && \ # reduce size of installed binaries diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 5c302db67d..9e35eb3aa9 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -110,7 +110,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.1.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.2.0 ${REPO_ACCESS_TOKEN} && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ From ba0826f3add3e57c6104717357dd6086df84e3b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 May 2024 13:01:31 +0200 Subject: [PATCH 097/104] FIX improve get_cjexl.sh --- get_cjexl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get_cjexl.sh b/get_cjexl.sh index c906af50b2..efd360756e 100644 --- a/get_cjexl.sh +++ b/get_cjexl.sh @@ -23,7 +23,7 @@ TOKEN=$2 res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION) if [ "$res_code" -eq 200 ]; then - echo "get_cjexl: downloading cjexl lib" + echo "get_cjexl: downloading cjexl lib $CJEXL_VERSION" ASSET_ID=$(curl -s -S -H "Authorization: token $TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID MD5SUM=$(md5sum /usr/local/lib/libcjexl.a | awk -F ' ' '{print$1}') From 41f046fd5fac94947c738fbe309bf0c78f8365cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 May 2024 18:17:06 +0200 Subject: [PATCH 098/104] ADD additional jexl expression test --- .../jexl_transformation_multiple.test | 294 ++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test new file mode 100644 index 0000000000..ec3f7f0fff --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test @@ -0,0 +1,294 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +JEXL basic expression in custom notification (multiple transformations) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with several attributes with transformations +# 02. Create entity E1 +# 03. Update entity E1 +# 05. Dump accumulator and see two expected transformations +# + + +echo "01. Create custom sub with several attributes with transformations" +echo "==================================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "speed": { + "value": "${(speed|split('\'' '\''))[0]|parseInt}", + "type": "Calculated" + }, + "ratio": { + "value": "${count.sum/count.count}", + "type": "Calculated" + }, + "code": { + "value": "${code||'\''invalid'\''}", + "type": "Calculated" + }, + "alert": { + "value": "${(value>max)?'\''nok'\'':'\''ok'\''}", + "type": "Calculated" + }, + "count": { + "value": "${{count:count.count+1, sum:count.sum+((speed|split('\'' '\''))[0]|parseInt)}}", + "type": "Calculated" + } + } + }, + "attrs": [ "speed", "ratio", "code", "alert", "count" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + +#"value": "${{count:count.count+1,sum:count.sum+speed|split('\'' '\'')[0]|parseInt}}", + +echo "02. Create entity E1" +echo "====================" +payload='{ + "id": "E1", + "type": "T", + "speed": { + "value": "10 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 100 + }, + "type": "StructuredValue" + }, + "code": { + "value": null, + "type": "Number" + }, + "value": { + "value": 14, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1" +echo "====================" +payload='{ + "speed": { + "value": "30 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 500 + }, + "type": "StructuredValue" + }, + "code": { + "value": 456, + "type": "Number" + }, + "value": { + "value": 75, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see two expected transformations" +echo "=========================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with several attributes with transformations +================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 +==================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 +==================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see two expected transformations +========================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 379 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "ok" + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": "invalid" + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 110 + } + }, + "id": "E1", + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 20 + }, + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 10 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 375 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "nok" + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": 456 + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 530 + } + }, + "id": "E1", + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 100 + }, + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 30 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 486e7df47353eb1489be2d153acadec1bd9e49ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 08:09:45 +0200 Subject: [PATCH 099/104] ADD JEXL usage example to doc --- doc/manuals/orion-api.md | 198 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 184 insertions(+), 14 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 57e3396481..01040ca9ab 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -74,6 +74,7 @@ - [Omitting payload](#omitting-payload) - [Additional considerations](#additional-considerations) - [JEXL Support](#jexl-support) + - [JEXL usage example](#jexl-usage-example) - [Available Transformations](#available-transformations) - [`uppercase`](#uppercase) - [`lowercase`](#lowercase) @@ -2331,13 +2332,13 @@ Orion Context Broker supports [JEXL expressions](https://github.com/TomFrost/Jex ``` "httpCustom": { - ... - "ngsi": { - "relativeHumidity": { - "value": "${humidity/100}", - "type": "Calculated" - } - } + ... + "ngsi": { + "relativeHumidity": { + "value": "${humidity/100}", + "type": "Calculated" + } + } } ``` @@ -2347,13 +2348,13 @@ A particular case of expressions are the ones in which the expression is a given ``` "httpCustom": { - ... - "ngsi": { - "originalHumidity": { - "value": "${humidity}", - "type": "Calculated" - } - } + ... + "ngsi": { + "originalHumidity": { + "value": "${humidity}", + "type": "Calculated" + } + } } ``` @@ -2363,6 +2364,175 @@ An useful resource to test JEXL expressions is the [JEXL playground](https://czo Orion relies on cjexl library to provide this functionality. If Orion binary is build without using cjexl, then only basic replacement functionality is available. +### JEXL usage example + +As example, let's consider a subscription like this: + +``` +"httpCustom": { + ... + "ngsi": { + "speed": { + "value": "${(speed|split('\'' '\''))[0]|parseInt}", + "type": "Calculated" + }, + "ratio": { + "value": "${count.sum/count.count}", + "type": "Calculated" + }, + "code": { + "value": "${code||'\''invalid'\''}", + "type": "Calculated" + }, + "alert": { + "value": "${(value>max)?'\''nok'\'':'\''ok'\''}", + "type": "Calculated" + }, + "count": { + "value": "${{count:count.count+1, sum:count.sum+((speed|split('\'' '\''))[0]|parseInt)}}", + "type": "Calculated" + } +} +``` + +A entity update like this: + +``` +{ + ... + "speed": { + "value": "10 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 100 + }, + "type": "StructuredValue" + }, + "code": { + "value": null, + "type": "Number" + }, + "value": { + "value": 14, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +} +``` + +will trigger a notification like this: + +``` +"data": [ + { + ... + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 10 + }, + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 20 + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": "invalid" + }, + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "ok" + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 110 + } + } + } +] +``` + +A new entity update like this: + +``` +{ + ... + "speed": { + "value": "30 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 500 + }, + "type": "StructuredValue" + }, + "code": { + "value": 456, + "type": "Number" + }, + "value": { + "value": 75, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +} +``` + +will trigger a notification like this: + +``` +"data": [ + { + ... + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 30 + }, + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 100 + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": 456 + }, + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "nok" + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 530 + } + } + } +] +``` + ### Available Transformations #### `uppercase` From b0df97eac75918eec50678a32b80f04e18dd9468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 08:23:25 +0200 Subject: [PATCH 100/104] FIX doc --- doc/manuals/orion-api.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 01040ca9ab..3d2ac6db18 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2373,7 +2373,7 @@ As example, let's consider a subscription like this: ... "ngsi": { "speed": { - "value": "${(speed|split('\'' '\''))[0]|parseInt}", + "value": "${(speed|split(' '))[0]|parseInt}", "type": "Calculated" }, "ratio": { @@ -2381,15 +2381,15 @@ As example, let's consider a subscription like this: "type": "Calculated" }, "code": { - "value": "${code||'\''invalid'\''}", + "value": "${code||'invalid'}", "type": "Calculated" }, "alert": { - "value": "${(value>max)?'\''nok'\'':'\''ok'\''}", + "value": "${(value>max)?'nok':'ok'}", "type": "Calculated" }, "count": { - "value": "${{count:count.count+1, sum:count.sum+((speed|split('\'' '\''))[0]|parseInt)}}", + "value": "${{count:count.count+1, sum:count.sum+((speed|split(' '))[0]|parseInt)}}", "type": "Calculated" } } From 047a63b32d50def2c821b4c397c5b963e1cb7db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 08:35:37 +0200 Subject: [PATCH 101/104] Apply suggestions from code review --- .github/workflows/publishimage-master.yml | 1 + .github/workflows/publishimage-tag.yml | 1 + CHANGES_NEXT_RELEASE | 1 - doc/manuals/devel/sourceCode.md | 2 +- doc/manuals/orion-api.md | 2 +- src/lib/expressions/ExprContext.cpp | 2 -- src/lib/expressions/ExprManager.cpp | 3 +-- src/lib/rest/rest.cpp | 4 ++-- src/lib/serviceRoutines/versionTreat.cpp | 2 +- 9 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index 2811dea64f..64ceb47bcf 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -1,6 +1,7 @@ name: Publish Docker image (master) # The workflow will push images for master on every merge +# Ideally, this should be done at dockerhub, but it doesn't support secrets (see https://stackoverflow.com/questions/78446824/use-environment-variables-with-sensible-information-in-docker-hub-autobuild) on: push: diff --git a/.github/workflows/publishimage-tag.yml b/.github/workflows/publishimage-tag.yml index 6cd6b94b40..5dde6e8d27 100644 --- a/.github/workflows/publishimage-tag.yml +++ b/.github/workflows/publishimage-tag.yml @@ -1,6 +1,7 @@ name: Publish Docker image (tag) # The workflow will push images on every tag in the format x.y.z +# Ideally, this should be done at dockerhub, but it doesn't support secrets (see https://stackoverflow.com/questions/78446824/use-environment-variables-with-sensible-information-in-docker-hub-autobuild) on: push: diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index ae44980113..c3acdc183d 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -4,4 +4,3 @@ - Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) - diff --git a/doc/manuals/devel/sourceCode.md b/doc/manuals/devel/sourceCode.md index 951e8777f8..a829aa1f41 100644 --- a/doc/manuals/devel/sourceCode.md +++ b/doc/manuals/devel/sourceCode.md @@ -24,7 +24,7 @@ * [src/lib/cache/](#srclibcache) (Subscription cache implementation) * [src/lib/logSummary/](#srcliblogsummary) (Log Summary implementation) * [src/lib/metricsMgr/](#srclibmetricsmgr) (Metrics Manager implementation) -* [src/lib/expressions/](#seribexpressions) (Custom notification expressions support) +* [src/lib/expressions/](#srclibexpressions) (Custom notification expressions support) ## src/app/contextBroker/ The main program is found in `contextBroker.cpp` and its purpose it to: diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 3d2ac6db18..a294ce2d46 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -432,7 +432,7 @@ There are some exception cases in which the above restrictions do not apply. In * URL parameter `q` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `mq` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `georel` and `coords` allow `;` -* Within `ngsi` (i.e. `id`, `type` and attribute values) in [NGSI Payload patching](#ngsi-payload-patching) (to support characters used in the JEXL expression syntax) +* Within `ngsi` (i.e. `id`, `type` and attribute values) in [NGSI Payload patching](#ngsi-payload-patching) (to support characters used in the [JEXL expression syntax](#jexl-support)) * Whichever attribute value which uses `TextUnrestricted` as attribute type (see [Special Attribute Types](#special-attribute-types) section) ## Identifiers syntax restrictions diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 927809b693..80148399b3 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -217,7 +217,6 @@ std::string ExprContextList::get(void) */ void ExprContextList::add(const std::string &_value) { - LM_T(LmtExpr, ("adding to JEXL expression context list (string): %s", _value.c_str())); jh.addString(_value); } @@ -279,7 +278,6 @@ void ExprContextList::add(ExprContextObject exprContextObject) */ void ExprContextList::add(ExprContextList exprContextList) { - std::string s = exprContextList.get(); LM_T(LmtExpr, ("adding to JEXL expression context list (list): %s", s.c_str())); jh.addRaw(s); diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index cd1a3d5182..c9bbf0299e 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -46,7 +46,7 @@ static const char* cjexl_eval(void* ptr, const char* script_ptr, const char* con return ""; } #else -// Interface to use libcjexl.a +// Interface to use libcjexl extern "C" { void* cjexl_new_engine(); } @@ -72,7 +72,6 @@ void ExprManager::init(void) - /* **************************************************************************** * * ExprManager::evaluate - diff --git a/src/lib/rest/rest.cpp b/src/lib/rest/rest.cpp index f09fd37529..f0ce3fc4c0 100644 --- a/src/lib/rest/rest.cpp +++ b/src/lib/rest/rest.cpp @@ -618,8 +618,8 @@ static void requestCompleted clock_addtime(&accTimeStat.mongoWriteWaitTime, &threadLastTimeStat.mongoWriteWaitTime); clock_addtime(&accTimeStat.mongoReadWaitTime, &threadLastTimeStat.mongoReadWaitTime); clock_addtime(&accTimeStat.mongoCommandWaitTime, &threadLastTimeStat.mongoCommandWaitTime); - clock_addtime(&accTimeStat.exprBasicCtxBldTime, &threadLastTimeStat.exprBasicCtxBldTime); - clock_addtime(&accTimeStat.exprBasicEvalTime, &threadLastTimeStat.exprBasicEvalTime); + clock_addtime(&accTimeStat.exprBasicCtxBldTime, &threadLastTimeStat.exprBasicCtxBldTime); + clock_addtime(&accTimeStat.exprBasicEvalTime, &threadLastTimeStat.exprBasicEvalTime); clock_addtime(&accTimeStat.exprJexlCtxBldTime, &threadLastTimeStat.exprJexlCtxBldTime); clock_addtime(&accTimeStat.exprJexlEvalTime, &threadLastTimeStat.exprJexlEvalTime); clock_addtime(&accTimeStat.renderTime, &threadLastTimeStat.renderTime); diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index ecd3740e1f..1b71e28125 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -48,7 +48,7 @@ #include #ifndef EXPR_BASIC -// Interface to use libcjexl.a +// Interface to use libcjexl extern "C" { const char* cjexl_version(); } From fbde34e73bc50a01e63a11b9473a295a7723262a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 09:17:17 +0200 Subject: [PATCH 102/104] Update doc/manuals/orion-api.md --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index a294ce2d46..a75523cf91 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2988,7 +2988,7 @@ results in As failsafe behaviour, evaluation returns `null` in the following cases: * Some of the transformation used in the expression is unknown -* Operations with identifiers that are not defined in the context are used. For instance, `(A==null)?-1:A` will result in `null` (and not `-1`) if `A` is not in the context, due to `==` is an operation that cannot be done on undefined identifiers. However, `A||-1` will work (i.e. `-1` will result if `A` is not in the context), as `||` is not considered an operation on `A`. +* Operations with identifiers that are not defined in the context are used. For instance, `(A==null)?0:A` will result in `null` (and not `0`) if `A` is not in the context, due to `==` is an operation that cannot be done on undefined identifiers. However, `A||0` will work (i.e. `0` will result if `A` is not in the context), as `||` is not considered an operation on `A`. * Syntax error in the JEXL expression (e.g. `A[0|uppercase`) ### Known limitations From cfa543a5408363e6be32895bd27fe4b5d21a8d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 10:35:27 +0200 Subject: [PATCH 103/104] Update test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test --- .../4004_jexl_expressions_in_subs/jexl_transformation_full.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index d4b83278e1..977301f192 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -128,7 +128,7 @@ payload='{ }, "M": { "type": "Number", - "value": "${M|round(2)}" + "value": "${M|round}" }, "N": { "type": "Text", From 06cbe4e3506c56a67c81207a86cd29555dd58fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 30 May 2024 11:31:52 +0200 Subject: [PATCH 104/104] Update doc/manuals/orion-api.md Co-authored-by: mapedraza <40356341+mapedraza@users.noreply.github.com> --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index a75523cf91..0d81391493 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2987,7 +2987,7 @@ results in As failsafe behaviour, evaluation returns `null` in the following cases: -* Some of the transformation used in the expression is unknown +* Some of the transformation used in the expression is unknown (e.g. `A|undefinedExpression`) * Operations with identifiers that are not defined in the context are used. For instance, `(A==null)?0:A` will result in `null` (and not `0`) if `A` is not in the context, due to `==` is an operation that cannot be done on undefined identifiers. However, `A||0` will work (i.e. `0` will result if `A` is not in the context), as `||` is not considered an operation on `A`. * Syntax error in the JEXL expression (e.g. `A[0|uppercase`)