From f856ede45eb95cc490442751c25bdf89512907e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 28 Jan 2022 15:43:39 +0100 Subject: [PATCH 001/390] ADD check_legacy_location_metadata.py script --- .../check_legacy_location_metadata.py | 219 ++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 scripts/managedb/upgrade-3.5.0/check_legacy_location_metadata.py diff --git a/scripts/managedb/upgrade-3.5.0/check_legacy_location_metadata.py b/scripts/managedb/upgrade-3.5.0/check_legacy_location_metadata.py new file mode 100644 index 0000000000..4c66409db3 --- /dev/null +++ b/scripts/managedb/upgrade-3.5.0/check_legacy_location_metadata.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2022 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# Hint: use 'PYTHONIOENCODING=utf8 python check_legacy_location_metadata.py' +# if you are going to redirect the output of this script to a file + + +__author__ = 'fermin' + +from pymongo import MongoClient +import sys + + +def flatten(_id): + """ + The way in which Python manage dictionaries doesn't make easy to be sure + of field ordering, which is important for MongoDB in the case of using an + embedded document for _id. This function helps. + + :param _id: JSON document containing id, type and servicePath + :return: a "flatten" version of the _id + """ + + r = {'_id.id': _id['id']} + + if 'type' in _id: + r['_id.type'] = _id['type'] + else: + r['_id.type'] = {'$exists': False} + + if 'servicePath' in _id: + r['_id.servicePath'] = _id['servicePath'] + else: + r['_id.servicePath'] = {'$exists': False} + + return r + + +def entity_id(doc): + """ + Extracts entity identification keys and returns an object with them + + :param doc: entity document, as taken from DB + :return: {id, type, servicePath} object + """ + id = doc['_id']['id'] + type = doc['_id']['type'] + sp = doc['_id']['servicePath'] + + return {'id': id, 'type': type, 'servicePath': sp} + + +########################## +# Main program starts here + +autofix = False + +if len(sys.argv) != 2: + print "invalid number of arguments, please check https://fiware-orion.readthedocs.io/en/master/admin/upgrading_crossing_3-5-0/index.html" + sys.exit() + +DB = sys.argv[1] + +# Warn user +if autofix: + print "WARNING!!!! This script modifies your '%s' database. It is STRONGLY RECOMMENDED that you" % DB + print "do a backup of your database before using it as described in https://fiware-orion.readthedocs.io/en/master/admin/database_admin/index.html#backup. Use this script at your own risk." + print "If you are sure you want to continue type 'yes' and press Enter" + + confirm = raw_input() + + if confirm != 'yes': + sys.exit() + +uri = 'mongodb://localhost:27017' +client = MongoClient(uri) +db = client[DB] + +need_fix = False +corrupted = False +verbose = True + +# Counters +processed = 0 +counter_location_ngsiv2 = 0 +counter_location_md_wgs84 = 0 +counter_location_md_not_wgs84 = 0 +counter_location_more_than_one = 0 +counter_untouched = 0 +counter_changed = 0 + +corrupted_entities = [] +affected_entities = [] + +total = db['entities'].count() + +print "- processing entities collection (%d entities) looking for attributes with ID metadata, this may take a while... " % total + +# The sort() is a way of ensuring that a modified document doesn't enter again at the end of the cursor (we have +# observed that this may happen with large collections, e.g ~50,000 entities). In addition, we have to use +# batch_size so the cursor doesn't expire at server (see http://stackoverflow.com/questions/10298354/mongodb-cursor-id-not-valid-error). +# The used batch_size value is a heuristic +for entity in db['entities'].find().sort([('_id.id', 1), ('_id.type', -1), ('_id.servicePath', 1)]).batch_size(100): + + processed += 1 + + sys.stdout.write('- processing entity: %d/%d \r' % (processed, total)) + sys.stdout.flush() + + # It may happen that entity doesn't have any attribute. We early detect that situation and skip in that case + if len(entity['attrs'].keys()) == 0: + # print '- %d: entity without attributes %s. Skipping' % (processed, json.dumps(entity['_id'])) + continue # entities loop + + wgs84_attr = None + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr] and 'location' in entity['attrs'][attr]['md']: + if entity['attrs'][attr]['md']['location']['value'] == 'WGS84' \ + or entity['attrs'][attr]['md']['location']['value'] == 'WSG84': + if wgs84_attr is not None: + # more than one location metadata is not allowed due to the checks done by CB + # in processLocationAtEntityCreation() function. However, we check in + # any case as CB could be buggy. Note in this case we don't append to + # affected_entities as that was done first time the location metadata was detected + counter_location_more_than_one += 1 + corrupted = True + corrupted_entities.append(entity_id(entity)) + continue # entities loop + else: + # Note that location metadata doesn't have any semantic in NGSIv2, so we can + # have an entity created with NGSIv2 with location metadata but not location geo-index. + # We need to detect that situation + if "location" in entity: + counter_location_md_wgs84 += 1 + affected_entities.append(entity_id(entity)) + wgs84_attr = attr + else: + counter_location_ngsiv2 += 1 + else: + # location metadata with a value different to WGS84 is not possible taking into + # account the checks done by CB in processLocationAtEntityCreation() function + # However, we check in any case as CB could be buggy + counter_location_md_not_wgs84 += 1 + corrupted = True + corrupted_entities.append(entity_id(entity)) + continue # entities loop + + if wgs84_attr is not None: + if autofix: + # Autofix consist on: + # 1) Remove location metadata (key in 'md' and item in 'mdNames') + # 2) Change attribute type by "geo:point" + + attr = entity['attrs'][wgs84_attr] + attr['mdNames'] = list(filter(lambda x: x != 'location', attr['mdNames'])) + + attr['md'].pop('location', None) + if len(attr['md'].keys()) == 0: + attr.pop('md', None) + + attr['type'] = 'geo:point' + + # it would be easier db['entities'].save(entity) but it seems it has problems when + # _id value is an object and may lead to duplicating entities instead of updating + # note we removed _id from update doc, to avoid possible problems reported by + # pymongo such as "the (immutable) field '_id' was found to have been altered" + query = flatten(entity['_id']) + entity.pop('_id', None) + db['entities'].update(query, entity) + counter_changed += 1 + else: + # Fix should be done by the user + counter_untouched += 1 + need_fix = True + else: + counter_untouched += 1 + +print '- processing entity: %d/%d' % (processed, total) +print '- documents analyzed: %d' % processed +print ' * entities w/ location md w/ WGS84/WSG84 value: %d' % counter_location_md_wgs84 +print ' * entities w/ location md w/o WGS84/WSG84 value (DB corruption!): %d' % counter_location_md_not_wgs84 +print ' * entities w/ more than one location md (DB corruption!): %d' % counter_location_more_than_one +print ' * entities w/ meaningless location md (created by NGSIv2) %d' % counter_location_ngsiv2 +print '- documents processed: %d' % processed +print ' * untouched: %d' % counter_untouched +print ' * changed: %d' % counter_changed + +if verbose: + if len(affected_entities) > 0: + print '- Affected entities:' + for entity in affected_entities: + print ' * ' + str(entity) + if len(corrupted_entities) > 0: + print '- Corrupted entities:' + for entity in corrupted_entities: + print ' * ' + str(entity) + +if need_fix or corrupted: + print "------------------------------------------------------" + print "WARNING: some problem was found during the process. Please check the documentation at https://fiware-orion.readthedocs.io/en/master/admin/upgrading_crossing_3-5-0/index.html" From f015be30281fcd598acc2ed054ad87d84dfbb2d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 2 Feb 2022 13:29:05 +0100 Subject: [PATCH 002/390] REMOVE NGSIv1 location metadata functionality --- CHANGES_NEXT_RELEASE | 1 + src/lib/common/globals.h | 3 - src/lib/mongoBackend/MongoCommonUpdate.cpp | 2 +- src/lib/mongoBackend/location.cpp | 40 +- src/lib/ngsi/ContextAttribute.cpp | 19 +- src/lib/ngsi/ContextAttribute.h | 2 +- src/lib/ngsi/ContextElementResponse.cpp | 30 +- src/lib/ngsi/Metadata.h | 1 - .../geoquery_circle_deprecated.test | 505 ------------ .../legacy_geolocalization_area_json.test | 219 ------ .../legacy_geoquery_bad_coords.test | 253 ------ .../legacy_geoquery_circle_json.test | 502 ------------ .../legacy_geoquery_polygon_json.test | 728 ------------------ ...cy_location_no_actual_location_change.test | 163 ---- .../legacy_wgs84.test | 503 ------------ .../wrong_inclusion_location_metadata_V1.test | 347 --------- ..._location_metadata_V1_V2_and_geoquery.test | 326 -------- .../create_location_attr_V1.test | 215 ------ .../create_location_attr_V2.test | 166 ---- .../get_location_attr_with_V1_creation.test | 176 ----- .../get_location_attr_with_V2_creation.test | 129 ---- .../get_location_attr_with_db_creation.test | 152 ---- 22 files changed, 15 insertions(+), 4467 deletions(-) delete mode 100644 test/functionalTest/cases/0000_deprecated_checkings/geoquery_circle_deprecated.test delete mode 100644 test/functionalTest/cases/0000_deprecated_checkings/legacy_geolocalization_area_json.test delete mode 100644 test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_bad_coords.test delete mode 100644 test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_circle_json.test delete mode 100644 test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_polygon_json.test delete mode 100644 test/functionalTest/cases/0000_deprecated_checkings/legacy_location_no_actual_location_change.test delete mode 100644 test/functionalTest/cases/0000_deprecated_checkings/legacy_wgs84.test delete mode 100644 test/functionalTest/cases/3045_wrong_inclusion_location_metadata_notification/wrong_inclusion_location_metadata_V1.test delete mode 100644 test/functionalTest/cases/3122_location_metadata_issues/create_and_get_location_metadata_V1_V2_and_geoquery.test delete mode 100644 test/functionalTest/cases/3122_location_metadata_issues/create_location_attr_V1.test delete mode 100644 test/functionalTest/cases/3122_location_metadata_issues/create_location_attr_V2.test delete mode 100644 test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_V1_creation.test delete mode 100644 test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_V2_creation.test delete mode 100644 test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_db_creation.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 70f6c9d210..51df6bbc55 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ - Add: ignoreType metadata so Orion ignores attribute type semantic (supported by geo-types at the present moment) (#4032) +- Remove: deprecated NGSIv1 location metadata to specify entity geo-location diff --git a/src/lib/common/globals.h b/src/lib/common/globals.h index 4caa07d716..8abc7b6751 100644 --- a/src/lib/common/globals.h +++ b/src/lib/common/globals.h @@ -44,9 +44,6 @@ #define EARTH_RADIUS_METERS 6371000 -#define LOCATION_WGS84 "WGS84" -#define LOCATION_WGS84_LEGACY "WSG84" // We fixed the right string at 0.17.0, but the old one needs to be mantained - /* **************************************************************************** diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 76bc31f6ce..3de9e1384c 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -2448,7 +2448,7 @@ static bool deleteContextAttributeItem *entityModified = true; /* Check aspects related with location */ - if (!targetAttr->getLocation(apiVersion).empty()) + if (targetAttr->getLocation(apiVersion)) { std::string details = std::string("action: DELETE") + " - entity: [" + entityDetail + "]" + diff --git a/src/lib/mongoBackend/location.cpp b/src/lib/mongoBackend/location.cpp index b874e33050..d1014f6390 100644 --- a/src/lib/mongoBackend/location.cpp +++ b/src/lib/mongoBackend/location.cpp @@ -340,9 +340,7 @@ bool processLocationAtEntityCreation { const ContextAttribute* caP = caV[ix]; - std::string location = caP->getLocation(apiVersion); - - if (location.empty()) + if (!caP->getLocation(apiVersion)) { continue; } @@ -354,13 +352,6 @@ bool processLocationAtEntityCreation return false; } - if ((location != LOCATION_WGS84) && (location != LOCATION_WGS84_LEGACY)) - { - *errDetail = "only WGS84 are supported, found: " + location; - oe->fill(SccBadRequest, *errDetail, ERROR_BAD_REQUEST); - return false; - } - if (!getGeoJson(caP, geoJson, errDetail, apiVersion)) { oe->fill(SccBadRequest, *errDetail, ERROR_BAD_REQUEST); @@ -390,22 +381,12 @@ bool processLocationAtUpdateAttribute ) { std::string subErr; - std::string locationString = targetAttr->getLocation(apiVersion); - - /* Check that location (if any) is using the correct coordinates string (it only - * makes sense for NGSIv1, this is legacy code that will be eventually removed) */ - if ((!locationString.empty()) && (locationString != LOCATION_WGS84) && (locationString != LOCATION_WGS84_LEGACY)) - { - *errDetail = "only WGS84 is supported for location, found: [" + targetAttr->getLocation() + "]"; - oe->fill(SccBadRequest, *errDetail, ERROR_BAD_REQUEST); - return false; - } // // Case 1: // update *to* location. There are 3 sub-cases // - if (!locationString.empty()) + if (targetAttr->getLocation(apiVersion)) { // // Case 1a: @@ -515,19 +496,10 @@ bool processLocationAtAppendAttribute ) { std::string subErr; - std::string locationString = targetAttr->getLocation(apiVersion); - - /* Check that location (if any) is using the correct coordinates string (it only - * makes sense for NGSIv1, this is legacy code that will be eventually removed) */ - if ((!locationString.empty()) && (locationString != LOCATION_WGS84) && (locationString != LOCATION_WGS84_LEGACY)) - { - *errDetail = "only WGS84 is supported for location, found: [" + targetAttr->getLocation() + "]"; - oe->fill(SccBadRequest, *errDetail, ERROR_BAD_REQUEST); - return false; - } + bool isALocation = targetAttr->getLocation(apiVersion); /* Case 1: append of new location attribute */ - if (actualAppend && (!locationString.empty())) + if (actualAppend && isALocation) { /* Case 1a: there is a previous location attribute -> error */ if (!currentLocAttrName->empty()) @@ -552,7 +524,7 @@ bool processLocationAtAppendAttribute } } /* Case 2: append-as-update changing attribute type from no-location -> location */ - else if (!actualAppend && (!locationString.empty())) + else if (!actualAppend && isALocation) { /* Case 2a: there is a previous (not empty and with different name) location attribute -> error */ if ((!currentLocAttrName->empty()) && (*currentLocAttrName != targetAttr->name)) @@ -588,7 +560,7 @@ bool processLocationAtAppendAttribute } /* Check 3: in the case of append-as-update, type changes from location -> no-location for the current location * attribute, then remove location attribute */ - else if (!actualAppend && (locationString.empty()) && (*currentLocAttrName == targetAttr->name)) + else if (!actualAppend && !isALocation && (*currentLocAttrName == targetAttr->name)) { *currentLocAttrName = ""; } diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 93aec55ef5..73a7110b95 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -550,25 +550,16 @@ ContextAttribute::ContextAttribute * * ContextAttribute::getLocation() - */ -std::string ContextAttribute::getLocation(ApiVersion apiVersion) const +bool ContextAttribute::getLocation(ApiVersion apiVersion) const { if (apiVersion == V1) { - // Deprecated way, but still supported - for (unsigned int ix = 0; ix < metadataVector.size(); ++ix) - { - if (metadataVector[ix]->name == NGSI_MD_LOCATION) - { - return metadataVector[ix]->stringValue; - } - } - // Current way of declaring location in NGSIv1, aligned with NGSIv2 (originally only only geo:point was supported // but doing so have problems so we need to support all them at the end, // see https://github.com/telefonicaid/fiware-orion/issues/3442 for details) if ((type == GEO_POINT) || (type == GEO_LINE) || (type == GEO_BOX) || (type == GEO_POLYGON) || (type == GEO_JSON)) { - return LOCATION_WGS84; + return true; } } else // v2 @@ -583,15 +574,15 @@ std::string ContextAttribute::getLocation(ApiVersion apiVersion) const { if ((metadataVector[ix]->valueType == orion::ValueTypeBoolean) && (metadataVector[ix]->boolValue == true)) { - return ""; + return false; } } } - return LOCATION_WGS84; + return true; } } - return ""; + return false; } diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index 4995a8b7b8..d490859419 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -92,7 +92,7 @@ typedef struct ContextAttribute ContextAttribute(const std::string& _name, const std::string& _type, orion::CompoundValueNode* _compoundValueP); /* Grabbers for metadata to which CB gives a special semantic */ - std::string getLocation(ApiVersion apiVersion = V1) const; + bool getLocation(ApiVersion apiVersion = V1) const; std::string toJsonV1(bool asJsonObject, RequestType request, diff --git a/src/lib/ngsi/ContextElementResponse.cpp b/src/lib/ngsi/ContextElementResponse.cpp index 1a140f5981..68c80aea06 100644 --- a/src/lib/ngsi/ContextElementResponse.cpp +++ b/src/lib/ngsi/ContextElementResponse.cpp @@ -136,7 +136,6 @@ ContextElementResponse::ContextElementResponse orion::BSONObj attr = getObjectFieldF(attrs, attrName); ContextAttribute* caP = NULL; ContextAttribute ca; - bool noLocationMetadata = true; // Name and type ca.name = dbDecode(attrName); @@ -215,7 +214,7 @@ ContextElementResponse::ContextElementResponse /* Setting custom metadata (if any) */ if (attr.hasField(ENT_ATTRS_MD)) { - orion::BSONObj mds = getObjectFieldF(attr, ENT_ATTRS_MD); + orion::BSONObj mds = getObjectFieldF(attr, ENT_ATTRS_MD); std::set mdsSet; mds.getFieldNames(&mdsSet); @@ -223,33 +222,6 @@ ContextElementResponse::ContextElementResponse { std::string currentMd = *i; Metadata* md = new Metadata(dbDecode(currentMd), getObjectFieldF(mds, currentMd)); - - /* The flag below indicates that a location metadata with WGS84 was found during iteration. - * It needs to the NGSIV1 check below, in order to add it if the flag is false - * In addition, adjust old wrong WSG84 metadata value with WGS84 */ - if (md->name == NGSI_MD_LOCATION) - { - noLocationMetadata = false; - - if (md->valueType == orion::ValueTypeString && md->stringValue == LOCATION_WGS84_LEGACY) - { - md->stringValue = LOCATION_WGS84; - } - } - - caP->metadataVector.push_back(md); - } - } - - if (apiVersion == V1) - { - /* Setting location metadata (if location attr found - * and the location metadata was not present or was present but with old wrong WSG84 value) */ - if ((locAttr == ca.name) && (ca.type != GEO_POINT) && noLocationMetadata) - { - /* Note that if attribute type is geo:point then the user is using the "new way" - * of locating entities in NGSIv1, thus location metadata is not rendered */ - Metadata* md = new Metadata(NGSI_MD_LOCATION, "string", LOCATION_WGS84); caP->metadataVector.push_back(md); } } diff --git a/src/lib/ngsi/Metadata.h b/src/lib/ngsi/Metadata.h index 1033020c08..d42a8de7f0 100644 --- a/src/lib/ngsi/Metadata.h +++ b/src/lib/ngsi/Metadata.h @@ -44,7 +44,6 @@ * * Metadata interpreted by Orion Context Broker, i.e. not custom metadata */ -#define NGSI_MD_LOCATION "location" // Deprecated (NGSIv1) #define NGSI_MD_IGNORE_TYPE "ignoreType" #define NGSI_MD_PREVIOUSVALUE "previousValue" // Special metadata #define NGSI_MD_ACTIONTYPE "actionType" // Special metadata diff --git a/test/functionalTest/cases/0000_deprecated_checkings/geoquery_circle_deprecated.test b/test/functionalTest/cases/0000_deprecated_checkings/geoquery_circle_deprecated.test deleted file mode 100644 index f69a093708..0000000000 --- a/test/functionalTest/cases/0000_deprecated_checkings/geoquery_circle_deprecated.test +++ /dev/null @@ -1,505 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Geo query circle - deprecated - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- -# The tests in this file use FIWARE_Location (instead of FIWARE::Location), which was deprecated -# in Orion 0.16.0 - -echo "1. Create entity corresponding to Madrid" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Madrid", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -echo -sleep 1.1 - -echo "2. Create entity corresponding to Alcobendas" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Alcobendas", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.533333, -3.633333", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -echo -sleep 1.1 - - -echo "3. Create entity corresponding to Leganes" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Leganes", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.316667, -3.75", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" - - -echo "4: ++++++++++ Madrid in 13.5 kms ++++++++++" -# Query inside 13.5 kms radius from Madrid center -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "City", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE_Location", - "value" : { - "circle": { - "centerLatitude": "40.418889", - "centerLongitude": "-3.691944", - "radius": "13500" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" - -echo "5: +++++++++++ Madrid in 15 kms +++++++++" -# Query inside 15 kms radius from Madrid center -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "City", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE_Location", - "value" : { - "circle": { - "centerLatitude": "40.418889", - "centerLongitude": "-3.691944", - "radius": "15000" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" - -echo "6: ++++++++++++ Madrid out 13.5 kms ++++++++" -# Query in 13.5 kms radius outside Madrid center -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "City", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE_Location", - "value" : { - "circle": { - "centerLatitude": "40.418889", - "centerLongitude": "-3.691944", - "radius": "13500", - "inverted": "true" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" - ---REGEXPECT-- -1. Create entity corresponding to Madrid -HTTP/1.1 200 OK -Content-Length: 267 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - -2. Create entity corresponding to Alcobendas -HTTP/1.1 200 OK -Content-Length: 271 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Alcobendas", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - -3. Create entity corresponding to Leganes -HTTP/1.1 200 OK -Content-Length: 268 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Leganes", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -4: ++++++++++ Madrid in 13.5 kms ++++++++++ -HTTP/1.1 200 OK -Content-Length: 549 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.316667, -3.75" - } - ], - "id": "Leganes", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -5: +++++++++++ Madrid in 15 kms +++++++++ -HTTP/1.1 200 OK -Content-Length: 818 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.533333, -3.633333" - } - ], - "id": "Alcobendas", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.316667, -3.75" - } - ], - "id": "Leganes", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -6: ++++++++++++ Madrid out 13.5 kms ++++++++ -HTTP/1.1 200 OK -Content-Length: 291 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.533333, -3.633333" - } - ], - "id": "Alcobendas", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/0000_deprecated_checkings/legacy_geolocalization_area_json.test b/test/functionalTest/cases/0000_deprecated_checkings/legacy_geolocalization_area_json.test deleted file mode 100644 index 97da9251a2..0000000000 --- a/test/functionalTest/cases/0000_deprecated_checkings/legacy_geolocalization_area_json.test +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Geo-localization JSON test (based on deprecated location metadata) - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- -echo "0: ++++++++++++++++++++" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "Room", - "isPattern": "false", - "id": "OfficeRoom", - "attributes": [ - { - "name": "location", - "type": "centigrade", - "value": "-3.5, 45.20", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" - -echo "1: ++++++++++++++++++++" -url="/v1/subscribeContext" -payload='{ - "entities": [ - { - "type": "Room", - "isPattern": "false", - "id": "OfficeRoom" - } - ], - "attributes": [ - "temperature", - "lightstatus" - ], - "reference": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "duration": "PT1H", - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "circle": { - "centerLatitude": "40", - "centerLongitude": "20", - "radius": "5" - } - } - } - ] - }, - "notifyConditions": [ - { - "type": "ONCHANGE", - "condValues": [ - "temperature", - "lightstatus" - ] - } - ] -}' -orionCurl --url "$url" --payload "$payload" - -echo "2: ++++++++++++++++++++" -url="/v1/subscribeContext" -payload='{ - "entities": [ - { - "type": "Room", - "isPattern": "false", - "id": "OfficeRoom" - } - ], - "attributes": [ - "temperature", - "lightstatus" - ], - "reference": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "duration": "PT1H", - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "polygon": { - "vertices": [ - { - "latitude": "40", - "longitude": "20" - }, - { - "latitude": "45", - "longitude": "45" - }, - { - "latitude": "35", - "longitude": "10" - } - ] - } - } - } - ] - }, - "notifyConditions": [ - { - "type": "ONCHANGE", - "condValues": [ - "temperature", - "lightstatus" - ] - } - ] -}' -orionCurl --url "$url" --payload "$payload" - ---REGEXPECT-- -0: ++++++++++++++++++++ -HTTP/1.1 200 OK -Content-Length: 275 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "centigrade", - "value": "" - } - ], - "id": "OfficeRoom", - "isPattern": "false", - "type": "Room" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -1: ++++++++++++++++++++ -HTTP/1.1 200 OK -Content-Length: 85 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "subscribeResponse": { - "duration": "PT1H", - "subscriptionId": "REGEX([0-9a-f]{24})" - } -} -2: ++++++++++++++++++++ -HTTP/1.1 200 OK -Content-Length: 85 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "subscribeResponse": { - "duration": "PT1H", - "subscriptionId": "REGEX([0-9a-f]{24})" - } -} ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_bad_coords.test b/test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_bad_coords.test deleted file mode 100644 index b0733cfdb6..0000000000 --- a/test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_bad_coords.test +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Geo query with invalid coordinates (based on deprecated location metadata) - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -echo "1: +++++++++ Update/Append +++++++++++" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Long200Lat100", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "200, 100", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -echo -echo - -echo "2: ++++++++++ OK Query ++++++++++" -# query according to issue #461 -url=/v1/queryContext -payload='{ - "entities": [ - { - "type": "Point", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "polygon": { - "vertices": [ - { "latitude" : 2.811371193331, "longitude" : 130.078061 }, - { "latitude" : 13.031027211328, "longitude" : 130.078061 }, - { "latitude" : 13.031027211328, "longitude" : 140.24472766667 }, - { "latitude" : 2.811371193331, "longitude" : 140.24472766667 } - ] - } - } - } - ] - } -}' - -orionCurl --url "$url" --payload "${payload}" -echo -echo - -echo "3: ++++++++++ Query with invalid latitude ++++++++++" -url=/v1/queryContext -payload='{ - "entities": [ - { - "type": "Point", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "polygon": { - "vertices": [ - { "latitude" : 92.811371193331, "longitude" : 130.078061 }, - { "latitude" : 13.031027211328, "longitude" : 130.078061 }, - { "latitude" : 13.031027211328, "longitude" : 140.24472766667 }, - { "latitude" : 2.811371193331, "longitude" : 140.24472766667 } - ] - } - } - } - ] - } -}' - -orionCurl --url "$url" --payload "${payload}" -echo -echo - - -echo "4: ++++++++++ Query with invalid longitude ++++++++++" -url=/v1/queryContext -payload='{ - "entities": [ - { - "type": "Point", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "polygon": { - "vertices": [ - { "latitude" : 12.811371193331, "longitude" : 130.078061 }, - { "latitude" : 13.031027211328, "longitude" : 130.078061 }, - { "latitude" : 13.031027211328, "longitude" : 240.24472766667 }, - { "latitude" : 2.811371193331, "longitude" : 140.24472766667 } - ] - } - } - } - ] - } -}' - -orionCurl --url "$url" --payload "${payload}" -echo -echo - - ---REGEXPECT-- -1: +++++++++ Update/Append +++++++++++ -HTTP/1.1 200 OK -Content-Length: 387 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Long200Lat100", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "472", - "details": "geo coordinates format error [see Orion user manual]: 200, 100", - "reasonPhrase": "request parameter is invalid/not allowed" - } - } - ] -} - - -2: ++++++++++ OK Query ++++++++++ -HTTP/1.1 200 OK -Content-Length: 70 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "errorCode": { - "code": "404", - "reasonPhrase": "No context element found" - } -} - - -3: ++++++++++ Query with invalid latitude ++++++++++ -HTTP/1.1 200 OK -Content-Length: 96 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "errorCode": { - "code": "400", - "details": "invalid value for latitude", - "reasonPhrase": "Bad Request" - } -} - - -4: ++++++++++ Query with invalid longitude ++++++++++ -HTTP/1.1 200 OK -Content-Length: 97 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "errorCode": { - "code": "400", - "details": "invalid value for longitude", - "reasonPhrase": "Bad Request" - } -} - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_circle_json.test b/test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_circle_json.test deleted file mode 100644 index 2c24c9dbe7..0000000000 --- a/test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_circle_json.test +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Geo query (circle) JSON test (based on deprecated location metadata) - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- -echo "1. Create entity corresponding to Madrid" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Madrid", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -echo -sleep 1.1 - -echo "2. Create entity corresponding to Alcobendas" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Alcobendas", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.533333, -3.633333", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -echo -sleep 1.1 - - -echo "3. Create entity corresponding to Leganes" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Leganes", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.316667, -3.75", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" - - -echo "4: ++++++++++ Madrid in 13.5 kms ++++++++++" -# Query inside 13.5 kms radius from Madrid center -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "City", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "circle": { - "centerLatitude": "40.418889", - "centerLongitude": "-3.691944", - "radius": "13500" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" - -echo "5: +++++++++++ Madrid in 15 kms +++++++++" -# Query inside 15 kms radius from Madrid center -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "City", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "circle": { - "centerLatitude": "40.418889", - "centerLongitude": "-3.691944", - "radius": "15000" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" - -echo "6: ++++++++++++ Madrid out 13.5 kms ++++++++" -# Query in 13.5 kms radius outside Madrid center -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "City", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "circle": { - "centerLatitude": "40.418889", - "centerLongitude": "-3.691944", - "radius": "13500", - "inverted": "true" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" - ---REGEXPECT-- -1. Create entity corresponding to Madrid -HTTP/1.1 200 OK -Content-Length: 267 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - -2. Create entity corresponding to Alcobendas -HTTP/1.1 200 OK -Content-Length: 271 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Alcobendas", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - -3. Create entity corresponding to Leganes -HTTP/1.1 200 OK -Content-Length: 268 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Leganes", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -4: ++++++++++ Madrid in 13.5 kms ++++++++++ -HTTP/1.1 200 OK -Content-Length: 549 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.316667, -3.75" - } - ], - "id": "Leganes", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -5: +++++++++++ Madrid in 15 kms +++++++++ -HTTP/1.1 200 OK -Content-Length: 818 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.533333, -3.633333" - } - ], - "id": "Alcobendas", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.316667, -3.75" - } - ], - "id": "Leganes", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -6: ++++++++++++ Madrid out 13.5 kms ++++++++ -HTTP/1.1 200 OK -Content-Length: 291 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.533333, -3.633333" - } - ], - "id": "Alcobendas", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_polygon_json.test b/test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_polygon_json.test deleted file mode 100644 index cd4c82d4a9..0000000000 --- a/test/functionalTest/cases/0000_deprecated_checkings/legacy_geoquery_polygon_json.test +++ /dev/null @@ -1,728 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Geo query test (polygon) (based on deprecated location metadata) - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- -echo "1: +++++++++ Point A +++++++++++" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "Point", - "isPattern": "false", - "id": "A", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "3, 2", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -sleep 1.1 -echo -echo - - -echo "2: +++++++++ Point B +++++++++++" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "Point", - "isPattern": "false", - "id": "B", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "5, 5", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -sleep 1.1 -echo -echo - - -echo "3: +++++++++ Point C +++++++++++" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "Point", - "isPattern": "false", - "id": "C", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "7, 4", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -sleep 1.1 -echo -echo - - -echo "4: ++++++++++ Rectangle in: A, B ++++++++++" -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "Point", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "polygon": { - "vertices": [ - { - "latitude": "0", - "longitude": "0" - }, - { - "latitude": "0", - "longitude": "6" - }, - { - "latitude": "6", - "longitude": "6" - }, - { - "latitude": "6", - "longitude": "0" - } - ] - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" -echo -echo - - -echo "5: ++++++++++ Rectangle in: B, C ++++++++++" -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "Point", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "polygon": { - "vertices": [ - { - "latitude": "3", - "longitude": "3" - }, - { - "latitude": "3", - "longitude": "8" - }, - { - "latitude": "11", - "longitude": "8" - }, - { - "latitude": "11", - "longitude": "3" - } - ] - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" -echo -echo - - -echo "6: ++++++++++ Triangle in: A ++++++++++" -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "Point", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "polygon": { - "vertices": [ - { - "latitude": "0", - "longitude": "0" - }, - { - "latitude": "0", - "longitude": "6" - }, - { - "latitude": "6", - "longitude": "0" - } - ] - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" -echo -echo - - -echo "7: ++++++++++ Rectangle out: A ++++++++++" -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "Point", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "polygon": { - "vertices": [ - { - "latitude": "3", - "longitude": "3" - }, - { - "latitude": "3", - "longitude": "8" - }, - { - "latitude": "11", - "longitude": "8" - }, - { - "latitude": "11", - "longitude": "3" - } - ], - "inverted": "true" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" -echo -echo - - -echo "8: ++++++++++ Triangle out: B, C ++++++++++" -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "Point", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "polygon": { - "vertices": [ - { - "latitude": "0", - "longitude": "0" - }, - { - "latitude": "0", - "longitude": "6" - }, - { - "latitude": "6", - "longitude": "0" - } - ], - "inverted": "true" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" -echo -echo - - ---REGEXPECT-- -1: +++++++++ Point A +++++++++++ -HTTP/1.1 200 OK -Content-Length: 263 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "A", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -2: +++++++++ Point B +++++++++++ -HTTP/1.1 200 OK -Content-Length: 263 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "B", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -3: +++++++++ Point C +++++++++++ -HTTP/1.1 200 OK -Content-Length: 263 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "C", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -4: ++++++++++ Rectangle in: A, B ++++++++++ -HTTP/1.1 200 OK -Content-Length: 512 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "3, 2" - } - ], - "id": "A", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "5, 5" - } - ], - "id": "B", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -5: ++++++++++ Rectangle in: B, C ++++++++++ -HTTP/1.1 200 OK -Content-Length: 512 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "5, 5" - } - ], - "id": "B", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "7, 4" - } - ], - "id": "C", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -6: ++++++++++ Triangle in: A ++++++++++ -HTTP/1.1 200 OK -Content-Length: 267 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "3, 2" - } - ], - "id": "A", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -7: ++++++++++ Rectangle out: A ++++++++++ -HTTP/1.1 200 OK -Content-Length: 267 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "3, 2" - } - ], - "id": "A", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -8: ++++++++++ Triangle out: B, C ++++++++++ -HTTP/1.1 200 OK -Content-Length: 512 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "5, 5" - } - ], - "id": "B", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "7, 4" - } - ], - "id": "C", - "isPattern": "false", - "type": "Point" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/0000_deprecated_checkings/legacy_location_no_actual_location_change.test b/test/functionalTest/cases/0000_deprecated_checkings/legacy_location_no_actual_location_change.test deleted file mode 100644 index 857588bd7a..0000000000 --- a/test/functionalTest/cases/0000_deprecated_checkings/legacy_location_no_actual_location_change.test +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Location - no actual location change (based on deprecated location metadata) - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- -echo "0: ++++++++++++++++++++" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "Room", - "isPattern": "false", - "id": "OfficeRoom", - "attributes": [ - { - "name": "location", - "type": "centigrade", - "value": "-3.5, 45.20", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" - -echo "1: ++++++++++++++++++++" - -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "Room", - "isPattern": "false", - "id": "OfficeRoom", - "attributes": [ - { - "name": "location", - "type": "centigrade", - "value": "-3.6, 48.20", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" - ---REGEXPECT-- -0: ++++++++++++++++++++ -HTTP/1.1 200 OK -Content-Length: 275 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "centigrade", - "value": "" - } - ], - "id": "OfficeRoom", - "isPattern": "false", - "type": "Room" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -1: ++++++++++++++++++++ -HTTP/1.1 200 OK -Content-Length: 275 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "centigrade", - "value": "" - } - ], - "id": "OfficeRoom", - "isPattern": "false", - "type": "Room" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/0000_deprecated_checkings/legacy_wgs84.test b/test/functionalTest/cases/0000_deprecated_checkings/legacy_wgs84.test deleted file mode 100644 index 4d2268ab34..0000000000 --- a/test/functionalTest/cases/0000_deprecated_checkings/legacy_wgs84.test +++ /dev/null @@ -1,503 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Geo query circle (old wrong WSG84 token) - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -echo "1. Create entity corresponding to Madrid" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Madrid", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WSG84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -echo -sleep 1.1 - -echo "2. Create entity corresponding to Alcobendas" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Alcobendas", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.533333, -3.633333", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WSG84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" -echo -sleep 1.1 - - -echo "3. Create entity corresponding to Leganes" -url="/v1/updateContext" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Leganes", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.316667, -3.75", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WSG84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url "$url" --payload "$payload" - - -echo "4: ++++++++++ Madrid in 13.5 kms ++++++++++" -# Query inside 13.5 kms radius from Madrid center -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "City", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "circle": { - "centerLatitude": "40.418889", - "centerLongitude": "-3.691944", - "radius": "13500" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" - -echo "5: +++++++++++ Madrid in 15 kms +++++++++" -# Query inside 15 kms radius from Madrid center -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "City", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "circle": { - "centerLatitude": "40.418889", - "centerLongitude": "-3.691944", - "radius": "15000" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" - -echo "6: ++++++++++++ Madrid out 13.5 kms ++++++++" -# Query in 13.5 kms radius outside Madrid center -url="/v1/queryContext" -payload='{ - "entities": [ - { - "type": "City", - "isPattern": "true", - "id": ".*" - } - ], - "restriction": { - "scopes": [ - { - "type" : "FIWARE::Location", - "value" : { - "circle": { - "centerLatitude": "40.418889", - "centerLongitude": "-3.691944", - "radius": "13500", - "inverted": "true" - } - } - } - ] - } -}' -orionCurl --url "$url" --payload "$payload" - ---REGEXPECT-- -1. Create entity corresponding to Madrid -HTTP/1.1 200 OK -Content-Length: 267 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WSG84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - -2. Create entity corresponding to Alcobendas -HTTP/1.1 200 OK -Content-Length: 271 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WSG84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Alcobendas", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - -3. Create entity corresponding to Leganes -HTTP/1.1 200 OK -Content-Length: 268 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WSG84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Leganes", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -4: ++++++++++ Madrid in 13.5 kms ++++++++++ -HTTP/1.1 200 OK -Content-Length: 549 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.316667, -3.75" - } - ], - "id": "Leganes", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -5: +++++++++++ Madrid in 15 kms +++++++++ -HTTP/1.1 200 OK -Content-Length: 818 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.533333, -3.633333" - } - ], - "id": "Alcobendas", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.316667, -3.75" - } - ], - "id": "Leganes", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -6: ++++++++++++ Madrid out 13.5 kms ++++++++ -HTTP/1.1 200 OK -Content-Length: 291 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "location", - "type": "coords", - "value": "40.533333, -3.633333" - } - ], - "id": "Alcobendas", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/3045_wrong_inclusion_location_metadata_notification/wrong_inclusion_location_metadata_V1.test b/test/functionalTest/cases/3045_wrong_inclusion_location_metadata_notification/wrong_inclusion_location_metadata_V1.test deleted file mode 100644 index aa95e78fda..0000000000 --- a/test/functionalTest/cases/3045_wrong_inclusion_location_metadata_notification/wrong_inclusion_location_metadata_V1.test +++ /dev/null @@ -1,347 +0,0 @@ -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Check "location" metadata is included in NGSIv1 notifications -triggered by NGSIV1 subscription creation, update or attribute update - ---SHELL-INIT-- -dbInit CB -brokerStart CB -accumulatorStart --pretty-print - ---SHELL-- - -# -# 01. Create NGSIv1 subscription for E -# 02. Create E with type T -# 03. Trigger notification by updating value of "name" attribute (in NGSIv1) -# 04. See NGSIv1 notifications and make sure "location" metadata is included -# -# FIXME P2: it seems that in the first notification metadata is no included. This -# makes me think that some difference between create entity and update entity logic -# is causing not including it in the first case. However, given that NGSIv1 is -# deprecated, we are not going to invest effort in fixing this problem. -# - - -echo "01. Create NGSIv1 subscription for E" -echo "====================================" -payload='{ - "entities": [ - { - "type": "T", - "isPattern": "false", - "id": "E" - } - ], - "attributes": [], - "reference": "http://localhost:'$LISTENER_PORT'/notify", - "notifyConditions": [ - { - "type": "ONCHANGE", - "condValues": [] - } - ] -}' -orionCurl --url /v1/subscribeContext --payload "$payload" -echo -echo - -SUB_ID=$(echo "$_response" | grep subscriptionId | awk -F\" '{ print $4 }') - - -echo "02. Create E with type: T" -echo "=========================" -payload='{ - "id": "E", - "type": "T", - "name":{ - "value": "SD3", - "type": "Text" - }, - "perimeter": { - "type":"geo:json", - "value":{ - "type": "Polygon", - "coordinates": [ - [ - [-13.806454999999991,33.46727800000001,0], - [-13.806707999999981,33.46723400000001,0], - [-13.806863999999999,33.46720599999997,0], - [-13.807017000000008,33.46719100000001,0], - [-13.806454999999991,33.46727800000001,0] - ] - ] - } - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "03. Trigger notification by updating value of name attribute (in NGSIv1)" -echo "========================================================================" -payload='{ - "contextElements": [ - { - "type": "T", - "isPattern": "false", - "id": "E", - "attributes": [ - { - "name": "name", - "type": "string", - "value": "2" - } - ] - } - ], - "updateAction": "UPDATE" -}' -orionCurl --url /v1/updateContext --payload "$payload" -echo -echo - - -echo "04. See NGSIv1 notifications and make sure location metadata is included" -echo "========================================================================" -accumulatorDump -echo -echo - - ---REGEXPECT-- -01. Create NGSIv1 subscription for E -==================================== -HTTP/1.1 200 OK -Content-Length: 86 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "subscribeResponse": { - "duration": "PT24H", - "subscriptionId": "REGEX([0-9a-f]{24})" - } -} - - -02. Create E with type: T -========================= -HTTP/1.1 201 Created -Content-Length: 0 -Location: /v2/entities/E?type=T -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - - - -03. Trigger notification by updating value of name attribute (in NGSIv1) -======================================================================== -HTTP/1.1 200 OK -Content-Length: 189 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "name": "name", - "type": "string", - "value": "" - } - ], - "id": "E", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -04. See NGSIv1 notifications and make sure location metadata is included -======================================================================== -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 468 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "name": "name", - "type": "Text", - "value": "SD3" - }, - { - "name": "perimeter", - "type": "geo:json", - "value": { - "coordinates": [ - [ - [ - -13.806455, - 33.467278, - 0 - ], - [ - -13.806708, - 33.467234, - 0 - ], - [ - -13.806864, - 33.467206, - 0 - ], - [ - -13.807017, - 33.467191, - 0 - ], - [ - -13.806455, - 33.467278, - 0 - ] - ] - ], - "type": "Polygon" - } - } - ], - "id": "E", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 534 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "name": "name", - "type": "string", - "value": "2" - }, - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "perimeter", - "type": "geo:json", - "value": { - "coordinates": [ - [ - [ - -13.806455, - 33.467278, - 0 - ], - [ - -13.806708, - 33.467234, - 0 - ], - [ - -13.806864, - 33.467206, - 0 - ], - [ - -13.807017, - 33.467191, - 0 - ], - [ - -13.806455, - 33.467278, - 0 - ] - ] - ], - "type": "Polygon" - } - } - ], - "id": "E", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - ---TEARDOWN-- -brokerStop CB -dbDrop CB -accumulatorStop diff --git a/test/functionalTest/cases/3122_location_metadata_issues/create_and_get_location_metadata_V1_V2_and_geoquery.test b/test/functionalTest/cases/3122_location_metadata_issues/create_and_get_location_metadata_V1_V2_and_geoquery.test deleted file mode 100644 index 6449c0539b..0000000000 --- a/test/functionalTest/cases/3122_location_metadata_issues/create_and_get_location_metadata_V1_V2_and_geoquery.test +++ /dev/null @@ -1,326 +0,0 @@ -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Create 2 entities, both with an attribute containing a location metadata. -One with NGSIv1 POST (old way of defining location attribute), the other one with NGSIv2 POST. -See that getting entities both in NGSIV1 and NGSIV2, the attribute metadatas are correctly rendered. -In addition, see that Geoqueries returns only the entity (E2) created with the NGSIv1 way of defining location attribute -and then contained in the polygon and bounding box of the geo query. - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -# -# 01. Create E1 with NGSIV2 -# 02. Create E2 with NGSIV1 -# 03. Get E1 and E2 with NGSIV2 -# 04. Get E1 and E2 with NGSIV1 -# 05. Make a Geo query with a polygon and check that only E2 is returned -# 06. Make a Geo query with a bounding box and check that only E2 is returned - -echo "01. Create E1 with NGSIV2" -echo "=========================" -payload=' -{ - "id": "E1", - "type": "T", - "A": { - "value": "40, -3", - "type": "whatever", - "metadata": { - "location": { - "type": "string", - "value": "WGS84" - } - } - } -} -' -orionCurl --url '/v2/entities' --payload "$payload" -echo -echo - - -echo "02. Create E2 with NGSIV1" -echo "=========================" -payload=' -{ - "attributes": [ - { - "name": "A", - "type": "whatever", - "value": "39,-2", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] -} -' -orionCurl --url '/v1/contextEntities/type/T/id/E2' --payload "$payload" -echo -echo - - -echo "03. Get E1 and E2 with NGSIV2" -echo "=============================" -orionCurl --url /v2/entities -echo -echo - - -echo "04. Get E1 and E2 with NGSIV1" -echo "=============================" -orionCurl --url /v1/contextEntities -echo -echo - - -echo "05. Make a Geo query with a polygon and check that only E2 is returned" -echo "======================================================================" -orionCurl --url '/v2/entities?georel=coveredBy&coords=0,0;50,0;50,-50;0,-50;0,0&geometry=polygon' -echo -echo - - -echo "06. Make a Geo query with a bounding box and check that only E2 is returned" -echo "===========================================================================" -orionCurl --url '/v2/entities?georel=coveredBy&coords=0,0;50,-50&geometry=box' -echo -echo - - ---REGEXPECT-- -01. Create E1 with NGSIV2 -========================= -HTTP/1.1 201 Created -Content-Length: 0 -Location: /v2/entities/E1?type=T -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - - - -02. Create E2 with NGSIV1 -========================= -HTTP/1.1 200 OK -Content-Length: 236 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "A", - "type": "whatever", - "value": "" - } - ], - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "id": "E2", - "isPattern": "false", - "type": "T" -} - - -03. Get E1 and E2 with NGSIV2 -============================= -HTTP/1.1 200 OK -Content-Length: 244 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -[ - { - "A": { - "metadata": { - "location": { - "type": "string", - "value": "WGS84" - } - }, - "type": "whatever", - "value": "40, -3" - }, - "id": "E1", - "type": "T" - }, - { - "A": { - "metadata": { - "location": { - "type": "string", - "value": "WGS84" - } - }, - "type": "whatever", - "value": "39,-2" - }, - "id": "E2", - "type": "T" - } -] - - -04. Get E1 and E2 with NGSIV1 -============================= -HTTP/1.1 200 OK -Content-Length: 499 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "A", - "type": "whatever", - "value": "40, -3" - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }, - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "A", - "type": "whatever", - "value": "39,-2" - } - ], - "id": "E2", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -05. Make a Geo query with a polygon and check that only E2 is returned -====================================================================== -HTTP/1.1 200 OK -Content-Length: 122 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -[ - { - "A": { - "metadata": { - "location": { - "type": "string", - "value": "WGS84" - } - }, - "type": "whatever", - "value": "39,-2" - }, - "id": "E2", - "type": "T" - } -] - - -06. Make a Geo query with a bounding box and check that only E2 is returned -=========================================================================== -HTTP/1.1 200 OK -Content-Length: 122 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -[ - { - "A": { - "metadata": { - "location": { - "type": "string", - "value": "WGS84" - } - }, - "type": "whatever", - "value": "39,-2" - }, - "id": "E2", - "type": "T" - } -] - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/3122_location_metadata_issues/create_location_attr_V1.test b/test/functionalTest/cases/3122_location_metadata_issues/create_location_attr_V1.test deleted file mode 100644 index 73cd8842ed..0000000000 --- a/test/functionalTest/cases/3122_location_metadata_issues/create_location_attr_V1.test +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Create 2 entities with NGSIV1 POST. One with the location metadata (defining a location attribute -according to the "old way"), the other without a location metadata (and then without location attribute) -Then check both entities in DB. - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -# -# 01. Create E1 with NGSIV1 with location metadata -# 02. Create E2 with NGSIV1 without location metadata -# 03. Check E1 entity in DB has location metadata and location GeoJSON field -# 04. Check E2 entity in DB has no location metadata nor location GeoJSON field - -echo "01. Create E1 with NGSIV1 with location metadata" -echo "================================================" -payload=' -{ - "attributes": [ - { - "name": "A", - "type": "whatever", - "value": "40,-3", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] -} -' -orionCurl --url '/v1/contextEntities/type/T/id/E1' --payload "$payload" -echo -echo - - -echo "02. Create E2 with NGSIV1 without location metadata" -echo "===================================================" -payload=' -{ - "attributes": [ - { - "name": "A", - "type": "whatever", - "value": "39,-2" - } - ] -} -' -orionCurl --url '/v1/contextEntities/type/T/id/E2' --payload "$payload" -echo -echo - - -echo "03. Check E1 entity in DB has location metadata and location GeoJSON field" -echo "==========================================================================" -mongoCmd ${CB_DB_NAME} 'db.entities.find({ "_id" : { "id" : "E1", "type": "T", "servicePath" : "/" } }, {_id:0, attrs: 1, location: 1})' | python -mjson.tool -echo -echo - - -echo "04. Check E2 entity in DB has no location metadata nor location GeoJSON field" -echo "=============================================================================" -mongoCmd ${CB_DB_NAME} 'db.entities.find({ "_id" : { "id" : "E2", "type": "T", "servicePath" : "/" } }, {_id:0, attrs: 1, location: 1})' | python -mjson.tool -echo -echo - - ---REGEXPECT-- -01. Create E1 with NGSIV1 with location metadata -================================================ -HTTP/1.1 200 OK -Content-Length: 236 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "A", - "type": "whatever", - "value": "" - } - ], - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" -} - - -02. Create E2 with NGSIV1 without location metadata -=================================================== -HTTP/1.1 200 OK -Content-Length: 170 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "attributes": [ - { - "name": "A", - "type": "whatever", - "value": "" - } - ], - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "id": "E2", - "isPattern": "false", - "type": "T" -} - - -03. Check E1 entity in DB has location metadata and location GeoJSON field -========================================================================== -{ - "attrs": { - "A": { - "creDate": REGEX(.*), - "md": { - "location": { - "type": "string", - "value": "WGS84" - } - }, - "mdNames": [ - "location" - ], - "modDate": REGEX(.*), - "type": "whatever", - "value": "40,-3" - } - }, - "location": { - "attrName": "A", - "coords": { - "coordinates": [ - -3, - 40 - ], - "type": "Point" - } - } -} - - -04. Check E2 entity in DB has no location metadata nor location GeoJSON field -============================================================================= -{ - "attrs": { - "A": { - "creDate": REGEX(.*), - "mdNames": [], - "modDate": REGEX(.*), - "type": "whatever", - "value": "39,-2" - } - } -} - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/3122_location_metadata_issues/create_location_attr_V2.test b/test/functionalTest/cases/3122_location_metadata_issues/create_location_attr_V2.test deleted file mode 100644 index 2dcf1535a6..0000000000 --- a/test/functionalTest/cases/3122_location_metadata_issues/create_location_attr_V2.test +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Create 2 entities with NGSIV2 POST. -One with the location metadata, managed as a regular "custom" metadata as it is NGSIV2. -The other without a location metadata but with a geo:point attribute, then managed as -a location attribute for NGSIV2. - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -# -# 01. Create E1 with NGSIV2 with location metadata but not geo:xxx type -# 02. Create E2 with NGSIV2 without location metadata but with geo:xxx type -# 03. Check E1 entity in DB has location metadata but no location GeoJSON field -# 04. Check E2 entity in DB has no location metadata but location GeoJSON field - -echo "01. Create E1 with NGSIV2 with location metadata but not geo:xxx type" -echo "=====================================================================" -payload=' -{ - "id": "E1", - "type": "T", - "A": { - "value": "40, -3", - "type": "whatever", - "metadata": { - "location": { - "type": "string", - "value": "WGS84" - } - } - } -} -' -orionCurl --url '/v2/entities' --payload "$payload" -echo -echo - - -echo "02. Create E2 with NGSIV2 without location metadata but with geo:xxx type" -echo "=========================================================================" -payload=' -{ - "id": "E2", - "type": "T", - "A": { - "value": "39, -2", - "type": "geo:point" - } -} -' -orionCurl --url '/v2/entities' --payload "$payload" -echo -echo - - -echo "03. Check E1 entity in DB has location metadata but no location GeoJSON field" -echo "=============================================================================" -mongoCmd ${CB_DB_NAME} 'db.entities.find({ "_id" : { "id" : "E1", "type": "T", "servicePath" : "/" } }, {_id:0, attrs: 1, location: 1})' | python -mjson.tool -echo -echo - - -echo "04. Check E2 entity in DB has no location metadata but location GeoJSON field" -echo "=============================================================================" -mongoCmd ${CB_DB_NAME} 'db.entities.find({ "_id" : { "id" : "E2", "type": "T", "servicePath" : "/" } }, {_id:0, attrs: 1, location: 1})' | python -mjson.tool -echo -echo - - ---REGEXPECT-- -01. Create E1 with NGSIV2 with location metadata but not geo:xxx type -===================================================================== -HTTP/1.1 201 Created -Content-Length: 0 -Location: /v2/entities/E1?type=T -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - - - -02. Create E2 with NGSIV2 without location metadata but with geo:xxx type -========================================================================= -HTTP/1.1 201 Created -Content-Length: 0 -Location: /v2/entities/E2?type=T -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - - - -03. Check E1 entity in DB has location metadata but no location GeoJSON field -============================================================================= -{ - "attrs": { - "A": { - "creDate": REGEX(.*), - "md": { - "location": { - "type": "string", - "value": "WGS84" - } - }, - "mdNames": [ - "location" - ], - "modDate": REGEX(.*), - "type": "whatever", - "value": "40, -3" - } - } -} - - -04. Check E2 entity in DB has no location metadata but location GeoJSON field -============================================================================= -{ - "attrs": { - "A": { - "creDate": REGEX(.*), - "mdNames": [], - "modDate": REGEX(.*), - "type": "geo:point", - "value": "39, -2" - } - }, - "location": { - "attrName": "A", - "coords": { - "coordinates": [ - -2, - 39 - ], - "type": "Point" - } - } -} - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_V1_creation.test b/test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_V1_creation.test deleted file mode 100644 index fa3682da77..0000000000 --- a/test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_V1_creation.test +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Create an entity with a location metadata, old way of creating a location attribute, with NGSIV1 POST. -GET the resulting entity both with NGSIV1 and NGSV2. - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -# -# 01. Create E1 with NGSIV1 with location metadata -# 02. Get E1 with NGSIV1 and check location metadata is included -# 03. Get E1 with NGSIV2 and check location metadata is included - -echo "01. Create E1 with NGSIV1 with location metadata" -echo "================================================" -payload=' -{ - "attributes": [ - { - "name": "A", - "type": "whatever", - "value": "40,-3", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ] - } - ] -} -' -orionCurl --url '/v1/contextEntities/type/T/id/E1' --payload "$payload" -echo -echo - - -echo "02. Get E1 with NGSIV1 and check location metadata is included" -echo "==============================================================" -orionCurl --url '/v1/contextEntities/E1' -echo -echo - - -echo "03. Get E1 with NGSIV2 and check location metadata is included" -echo "==============================================================" -orionCurl --url '/v2/entities/E1' -echo -echo - - ---REGEXPECT-- -01. Create E1 with NGSIV1 with location metadata -================================================ -HTTP/1.1 200 OK -Content-Length: 236 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextResponses": [ - { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "A", - "type": "whatever", - "value": "" - } - ], - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" -} - - -02. Get E1 with NGSIV1 and check location metadata is included -============================================================== -HTTP/1.1 200 OK -Content-Length: 237 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "A", - "type": "whatever", - "value": "40,-3" - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } -} - - -03. Get E1 with NGSIV2 and check location metadata is included -============================================================== -HTTP/1.1 200 OK -Content-Length: 120 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "A": { - "metadata": { - "location": { - "type": "string", - "value": "WGS84" - } - }, - "type": "whatever", - "value": "40,-3" - }, - "id": "E1", - "type": "T" -} - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_V2_creation.test b/test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_V2_creation.test deleted file mode 100644 index 4ef0187afb..0000000000 --- a/test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_V2_creation.test +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Create an entity with a location attribute and no metadata with NGSIV2 POST. -GET the resulting entity both with NGSIV1 and NGSV2. - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -# -# 01. Create E1 with NGSIV2 -# 02. Get E1 with NGSIV1 -# 03. Get E1 with NGSIV2 - -echo "01. Create E1 with NGSIV2" -echo "=========================" -payload=' -{ - "id": "E1", - "type": "T", - "A": { - "value": "40, -3", - "type": "geo:point" - } -} -' -orionCurl --url '/v2/entities' --payload "$payload" -echo -echo - - -echo "02. Get E1 with NGSIV1" -echo "======================" -orionCurl --url '/v1/contextEntities/E1' -echo -echo - - -echo "03. Get E1 with NGSIV2" -echo "======================" -orionCurl --url '/v2/entities/E1' -echo -echo - - ---REGEXPECT-- -01. Create E1 with NGSIV2 -========================= -HTTP/1.1 201 Created -Content-Length: 0 -Location: /v2/entities/E1?type=T -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - - - -02. Get E1 with NGSIV1 -====================== -HTTP/1.1 200 OK -Content-Length: 173 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextElement": { - "attributes": [ - { - "name": "A", - "type": "geo:point", - "value": "40, -3" - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } -} - - -03. Get E1 with NGSIV2 -====================== -HTTP/1.1 200 OK -Content-Length: 78 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "A": { - "metadata": {}, - "type": "geo:point", - "value": "40, -3" - }, - "id": "E1", - "type": "T" -} - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_db_creation.test b/test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_db_creation.test deleted file mode 100644 index 9b55638961..0000000000 --- a/test/functionalTest/cases/3122_location_metadata_issues/get_location_attr_with_db_creation.test +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Create an inconsistent entity with a location attribute and no metadata directly in DB. -GET the resulting entity both with NGSIV1 and NGSV2. - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -# -# 01. Create E1 directly in DB with location GeoJSON field but without location metadata nor geo:xxx type -# 02. Get E1 with NGSIV1 with location metadata -# 03. Get E1 with NGSIV2 without location metadata - -echo "01. Create E1 directly in DB with location GeoJSON field but without location metadata nor geo:xxx type" -echo "=======================================================================================================" -mongoCmd ${CB_DB_NAME} 'db.entities.insert({ - "_id": { - "id": "E1", - "type": "T", - "servicePath": "/" - }, - "attrNames": ["A"], - "attrs": { - "A": { - "type": "whatever", - "creDate": 1462826512, - "modDate": 1462826512, - "value": "40,-3" - } - }, - "creDate": 1462826512, - "modDate": 1462826512, - "location" : { - "attrName" : "A", - "coords" : { - "type" : "Point", - "coordinates" : [ - -3.0, - 40.0 - ] - } - } -})' -echo -echo -echo -echo - - -echo "02. Get E1 with NGSIV1 with location metadata" -echo "=============================================" -orionCurl --url '/v1/contextEntities/E1' -echo -echo - - -echo "03. Get E1 with NGSIV2 without location metadata" -echo "================================================" -orionCurl --url '/v2/entities/E1' -echo -echo - - ---REGEXPECT-- -01. Create E1 directly in DB with location GeoJSON field but without location metadata nor geo:xxx type -======================================================================================================= -WriteResult({ "nInserted" : 1 }) - - - - -02. Get E1 with NGSIV1 with location metadata -============================================= -HTTP/1.1 200 OK -Content-Length: 237 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WGS84" - } - ], - "name": "A", - "type": "whatever", - "value": "40,-3" - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } -} - - -03. Get E1 with NGSIV2 without location metadata -================================================ -HTTP/1.1 200 OK -Content-Length: 76 -Content-Type: application/json -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Date: REGEX(.*) - -{ - "A": { - "metadata": {}, - "type": "whatever", - "value": "40,-3" - }, - "id": "E1", - "type": "T" -} - - ---TEARDOWN-- -brokerStop CB -dbDrop CB From 27d4e5ab7045933820b0b584c5d7e615546a2647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 3 Feb 2022 08:52:01 +0100 Subject: [PATCH 003/390] FIX simplify geoloc implementation --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 4 +-- src/lib/mongoBackend/location.cpp | 33 ++++------------------ src/lib/ngsi/ContextAttribute.cpp | 31 ++++++-------------- src/lib/ngsi/ContextAttribute.h | 2 +- 4 files changed, 16 insertions(+), 54 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 3de9e1384c..c29033e624 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -2438,7 +2438,6 @@ static bool deleteContextAttributeItem std::string* currentLocAttrName, bool* entityModified, orion::BSONDate* dateExpiration, - ApiVersion apiVersion, OrionError* oe ) { @@ -2448,7 +2447,7 @@ static bool deleteContextAttributeItem *entityModified = true; /* Check aspects related with location */ - if (targetAttr->getLocation(apiVersion)) + if (targetAttr->getLocation()) { std::string details = std::string("action: DELETE") + " - entity: [" + entityDetail + "]" + @@ -2618,7 +2617,6 @@ static bool processContextAttributeVector currentLocAttrName, &entityModified, dateExpiration, - apiVersion, oe)) { return false; diff --git a/src/lib/mongoBackend/location.cpp b/src/lib/mongoBackend/location.cpp index d1014f6390..2327e1f496 100644 --- a/src/lib/mongoBackend/location.cpp +++ b/src/lib/mongoBackend/location.cpp @@ -105,6 +105,8 @@ static bool stringArray2coords * ContextAttribute provided as parameter. * * It returns true, except in the case of error (in which in addition errDetail gets filled) +* +* FIXME PR: try to avoid apiVersion */ static bool getGeoJson ( @@ -118,31 +120,6 @@ static bool getGeoJson std::vector coordLong; orion::BSONArrayBuilder ba; - if ((apiVersion == V1) && (caP->type != GEO_POINT) && (caP->type != GEO_LINE) && (caP->type != GEO_BOX) && - (caP->type != GEO_POLYGON) && (caP->type != GEO_JSON)) - { - // This corresponds to the legacy way in NGSIv1 based in metadata - // The block is the same that for GEO_POINT but it is clearer if we keep it separated - - double aLat; - double aLong; - - if (!string2coords(caP->stringValue, aLat, aLong)) - { - *errDetail = "geo coordinates format error [see Orion user manual]: " + caP->stringValue; - return false; - } - - geoJson->append("type", "Point"); - - orion::BSONArrayBuilder ba; - ba.append(aLong); - ba.append(aLat); - geoJson->append("coordinates", ba.arr()); - - return true; - } - if (caP->type == GEO_POINT) { double aLat; @@ -340,7 +317,7 @@ bool processLocationAtEntityCreation { const ContextAttribute* caP = caV[ix]; - if (!caP->getLocation(apiVersion)) + if (!caP->getLocation()) { continue; } @@ -386,7 +363,7 @@ bool processLocationAtUpdateAttribute // Case 1: // update *to* location. There are 3 sub-cases // - if (targetAttr->getLocation(apiVersion)) + if (targetAttr->getLocation()) { // // Case 1a: @@ -496,7 +473,7 @@ bool processLocationAtAppendAttribute ) { std::string subErr; - bool isALocation = targetAttr->getLocation(apiVersion); + bool isALocation = targetAttr->getLocation(); /* Case 1: append of new location attribute */ if (actualAppend && isALocation) diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 73a7110b95..45f74fc5bb 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -550,36 +550,23 @@ ContextAttribute::ContextAttribute * * ContextAttribute::getLocation() - */ -bool ContextAttribute::getLocation(ApiVersion apiVersion) const +bool ContextAttribute::getLocation() const { - if (apiVersion == V1) + // null value is allowed but inhibits the attribute to be used as location (e.g. in geo-queries) + if ((valueType != orion::ValueTypeNull) && ((type == GEO_POINT) || (type == GEO_LINE) || (type == GEO_BOX) || (type == GEO_POLYGON) || (type == GEO_JSON))) { - // Current way of declaring location in NGSIv1, aligned with NGSIv2 (originally only only geo:point was supported - // but doing so have problems so we need to support all them at the end, - // see https://github.com/telefonicaid/fiware-orion/issues/3442 for details) - if ((type == GEO_POINT) || (type == GEO_LINE) || (type == GEO_BOX) || (type == GEO_POLYGON) || (type == GEO_JSON)) + for (unsigned int ix = 0; ix < metadataVector.size(); ++ix) { - return true; - } - } - else // v2 - { - // null value is allowed but inhibits the attribute to be used as location (e.g. in geo-queries) - if ((valueType != orion::ValueTypeNull) && ((type == GEO_POINT) || (type == GEO_LINE) || (type == GEO_BOX) || (type == GEO_POLYGON) || (type == GEO_JSON))) - { - for (unsigned int ix = 0; ix < metadataVector.size(); ++ix) + // the existence of the ignoreType metadata set to true also inhibits the attribute to be used as location + if (metadataVector[ix]->name == NGSI_MD_IGNORE_TYPE) { - // the existence of the ignoreType metadata set to true also inhibits the attribute to be used as location - if (metadataVector[ix]->name == NGSI_MD_IGNORE_TYPE) + if ((metadataVector[ix]->valueType == orion::ValueTypeBoolean) && (metadataVector[ix]->boolValue == true)) { - if ((metadataVector[ix]->valueType == orion::ValueTypeBoolean) && (metadataVector[ix]->boolValue == true)) - { - return false; - } + return false; } } - return true; } + return true; } return false; diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index d490859419..7919d941a2 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -92,7 +92,7 @@ typedef struct ContextAttribute ContextAttribute(const std::string& _name, const std::string& _type, orion::CompoundValueNode* _compoundValueP); /* Grabbers for metadata to which CB gives a special semantic */ - bool getLocation(ApiVersion apiVersion = V1) const; + bool getLocation(void) const; std::string toJsonV1(bool asJsonObject, RequestType request, From 707f86a051a3546d6a3dbdff07966cadcf72dda1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 9 Feb 2022 12:46:56 +0100 Subject: [PATCH 004/390] FIX remove location from metadata.md --- doc/manuals/user/metadata.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manuals/user/metadata.md b/doc/manuals/user/metadata.md index 5a49336ebc..eee87660e8 100644 --- a/doc/manuals/user/metadata.md +++ b/doc/manuals/user/metadata.md @@ -10,7 +10,6 @@ You can use any name for your custom metadata except for a few reserved names, u for special metadata that are interpreted by Orion: - [ID](#metadata-id-for-attributes) (deprecated, but still "blocked" as forbidden keyword) -- location, which is currently [deprecated](../deprecated.md), but still supported - The ones defined in "Builtin metadata" section in the NGSIv2 spec ## Custom attribute metadata From a7723973dc30dc83fc61bc7553c391ccd53583bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 Nov 2023 13:36:48 +0100 Subject: [PATCH 005/390] FIX improve problematic attribute reporting on updates --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 36 ++++++++++++++++------ src/lib/rest/OrionError.cpp | 11 +++++++ src/lib/rest/OrionError.h | 2 ++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index afdec2510f..33759a32bd 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -2448,7 +2448,7 @@ static bool updateContextAttributeItem " - offending attribute: " + targetAttr->getName(); cerP->statusCode.fill(SccInvalidParameter, details); - oe->fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ATTRIBUTE, ERROR_NOT_FOUND); + //oe->fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ATTRIBUTE, ERROR_NOT_FOUND); /* Although 'ca' has been already pushed into cerP, the pointer is still valid, of course */ ca->found = false; @@ -2613,7 +2613,7 @@ static bool deleteContextAttributeItem " - attribute not found"; cerP->statusCode.fill(SccInvalidParameter, details); - oe->fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ATTRIBUTE, ERROR_NOT_FOUND); + //oe->fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ATTRIBUTE, ERROR_NOT_FOUND); alarmMgr.badInput(clientIp, "attribute to be deleted is not found", targetAttr->getName()); ca->found = false; @@ -3645,7 +3645,7 @@ static unsigned int updateEntity *attributeAlreadyExistsList += " ]"; } - if ((apiVersion == V2) && (action == ActionTypeUpdate)) + if ((apiVersion == V2) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) { for (unsigned int ix = 0; ix < eP->attributeVector.size(); ++ix) { @@ -4451,16 +4451,34 @@ unsigned int processContextElement if (attributeAlreadyExistsError == true) { - std::string details = "one or more of the attributes in the request already exist: " + attributeAlreadyExistsList; - buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); - responseP->oe.fill(SccInvalidModification, details, ERROR_UNPROCESSABLE); + if (responseP->oe.code == SccInvalidModification) + { + // Another previous entity had problems. Use appendDetails() + responseP->oe.appendDetails(", " + eP->id + " - " + attributeAlreadyExistsList); + } + else + { + // First entity with this problem. Use fill() + std::string details = "one or more of the attributes in the request already exist: " + eP->id + " - " + attributeAlreadyExistsList; + buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); + responseP->oe.fill(SccInvalidModification, details, ERROR_UNPROCESSABLE); + } } if (attributeNotExistingError == true) { - std::string details = "one or more of the attributes in the request do not exist: " + attributeNotExistingList; - buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); - responseP->oe.fill(SccInvalidModification, details, ERROR_UNPROCESSABLE); + if (responseP->oe.code == SccInvalidModification) + { + // Another previous entity had problems. Use appendDetails() + responseP->oe.appendDetails(", " + eP->id + " - " + attributeNotExistingList); + } + else + { + // First entity with this problem. Use fill() + std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - " + attributeNotExistingList; + buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); + responseP->oe.fill(SccInvalidModification, details, ERROR_UNPROCESSABLE); + } } // Response in responseP diff --git a/src/lib/rest/OrionError.cpp b/src/lib/rest/OrionError.cpp index cbec3a9c9e..9b25f1014d 100644 --- a/src/lib/rest/OrionError.cpp +++ b/src/lib/rest/OrionError.cpp @@ -98,6 +98,17 @@ void OrionError::fill(const StatusCode& sc) +/* **************************************************************************** +* +* OrionError::appendDetails - +*/ +void OrionError::appendDetails(const std::string& _details) +{ + details += _details; +} + + + /* **************************************************************************** * * OrionError::smartRender - diff --git a/src/lib/rest/OrionError.h b/src/lib/rest/OrionError.h index 47833a0dac..318cb83790 100644 --- a/src/lib/rest/OrionError.h +++ b/src/lib/rest/OrionError.h @@ -55,6 +55,8 @@ typedef struct OrionError std::string toJsonV1(void); void fill(HttpStatusCode _code, const std::string& _details, const std::string& _reasonPhrase = ""); void fill(const StatusCode& sc); + void appendDetails(const std::string& _details); + private: void shrinkReasonPhrase(void); From f4eb0a2718766d7a0248082ca7c781d2e6ae5116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 20 Nov 2023 15:08:32 +0100 Subject: [PATCH 006/390] FIX ftest --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 4 +- .../patch_v2_entities_eid.test | 4 +- .../patch_entity_with_param_type.test | 4 +- .../append_and_append_only_combinations.test | 4 +- .../append_only_attribute_already_there.test | 4 +- .../append_only_error_if_exists.test | 4 +- .../982_post_entity_with_type_param.test | 4 +- .../delete_not_exist_attr.test | 8 +- .../v1_append_strict.test | 8 +- .../patch_unknown_attribute.test | 4 +- .../error_response_for_entity_not_found.test | 4 +- .../ngsiv2_entity_id_revisited.test | 4 +- .../batch_update_APPEND.test | 4 +- ...date_DELETE_non_existing_and_existing.test | 8 +- ...xisting_and_existing_several_entities.test | 342 ++++++++++++++++++ ...batch_update_DELETE_non_existing_attr.test | 8 +- .../batch_update_UPDATE.test | 4 +- .../patch_non_existing_attribute.test | 4 +- ...error_updating_non_existing_attribute.test | 4 +- .../PATCH_offending_attributes.test | 8 +- 20 files changed, 390 insertions(+), 48 deletions(-) create mode 100644 test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 33759a32bd..feb5a99bf2 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -3623,7 +3623,7 @@ static unsigned int updateEntity { for (unsigned int ix = 0; ix < eP->attributeVector.size(); ++ix) { - if (attrs.hasField(eP->attributeVector[ix]->name)) + if (attrs.hasField(dbEncode(eP->attributeVector[ix]->name))) { alarmMgr.badInput(clientIp, "attribute already exists", eP->attributeVector[ix]->name); *attributeAlreadyExistsError = true; @@ -3649,7 +3649,7 @@ static unsigned int updateEntity { for (unsigned int ix = 0; ix < eP->attributeVector.size(); ++ix) { - if (!attrs.hasField (eP->attributeVector[ix]->name)) + if (!attrs.hasField(dbEncode(eP->attributeVector[ix]->name))) { *attributeNotExistingError = true; diff --git a/test/functionalTest/cases/0979_patch_v2_entities_eid/patch_v2_entities_eid.test b/test/functionalTest/cases/0979_patch_v2_entities_eid/patch_v2_entities_eid.test index 283558281f..5848be90e5 100644 --- a/test/functionalTest/cases/0979_patch_v2_entities_eid/patch_v2_entities_eid.test +++ b/test/functionalTest/cases/0979_patch_v2_entities_eid/patch_v2_entities_eid.test @@ -171,10 +171,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 110 +Content-Length: 115 { - "description": "one or more of the attributes in the request do not exist: [ attr3 ]", + "description": "one or more of the attributes in the request do not exist: E1 - [ attr3 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/0980_patch_entity_v2_wihth_param_type/patch_entity_with_param_type.test b/test/functionalTest/cases/0980_patch_entity_v2_wihth_param_type/patch_entity_with_param_type.test index 56e7955391..41c90e2615 100644 --- a/test/functionalTest/cases/0980_patch_entity_v2_wihth_param_type/patch_entity_with_param_type.test +++ b/test/functionalTest/cases/0980_patch_entity_v2_wihth_param_type/patch_entity_with_param_type.test @@ -169,10 +169,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 107 +Content-Length: 112 { - "description": "one or more of the attributes in the request do not exist: [ A2 ]", + "description": "one or more of the attributes in the request do not exist: E1 - [ A2 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/0981_POST_v2_attributes/append_and_append_only_combinations.test b/test/functionalTest/cases/0981_POST_v2_attributes/append_and_append_only_combinations.test index e00bf7c539..de25c1bcc8 100644 --- a/test/functionalTest/cases/0981_POST_v2_attributes/append_and_append_only_combinations.test +++ b/test/functionalTest/cases/0981_POST_v2_attributes/append_and_append_only_combinations.test @@ -263,10 +263,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 111 +Content-Length: 116 { - "description": "one or more of the attributes in the request already exist: [ attr4 ]", + "description": "one or more of the attributes in the request already exist: E1 - [ attr4 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/0981_POST_v2_attributes/append_only_attribute_already_there.test b/test/functionalTest/cases/0981_POST_v2_attributes/append_only_attribute_already_there.test index b03a038c07..ffc8b54c54 100644 --- a/test/functionalTest/cases/0981_POST_v2_attributes/append_only_attribute_already_there.test +++ b/test/functionalTest/cases/0981_POST_v2_attributes/append_only_attribute_already_there.test @@ -75,10 +75,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 111 +Content-Length: 116 { - "description": "one or more of the attributes in the request already exist: [ attr1 ]", + "description": "one or more of the attributes in the request already exist: E1 - [ attr1 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/0981_POST_v2_attributes/append_only_error_if_exists.test b/test/functionalTest/cases/0981_POST_v2_attributes/append_only_error_if_exists.test index 3755d1e4b6..ecd48882a5 100644 --- a/test/functionalTest/cases/0981_POST_v2_attributes/append_only_error_if_exists.test +++ b/test/functionalTest/cases/0981_POST_v2_attributes/append_only_error_if_exists.test @@ -119,10 +119,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 111 +Content-Length: 116 { - "description": "one or more of the attributes in the request already exist: [ attr1 ]", + "description": "one or more of the attributes in the request already exist: E1 - [ attr1 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/0982_post_entity_with_type_param/982_post_entity_with_type_param.test b/test/functionalTest/cases/0982_post_entity_with_type_param/982_post_entity_with_type_param.test index 2f6c1dcee9..3208886ffb 100644 --- a/test/functionalTest/cases/0982_post_entity_with_type_param/982_post_entity_with_type_param.test +++ b/test/functionalTest/cases/0982_post_entity_with_type_param/982_post_entity_with_type_param.test @@ -238,10 +238,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 108 +Content-Length: 113 { - "description": "one or more of the attributes in the request already exist: [ A3 ]", + "description": "one or more of the attributes in the request already exist: E1 - [ A3 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test b/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test index 405123b429..ae337a1564 100644 --- a/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test +++ b/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test @@ -102,15 +102,15 @@ Content-Length: 121 03. DELETE /v2/entities/E1/attrs/none ===================================== -HTTP/1.1 404 Not Found +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 79 +Content-Length: 114 { - "description": "The entity does not have such an attribute", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E1 - [ none ]", + "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test b/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test index 1dd08bb2c6..6588f5b23e 100644 --- a/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test +++ b/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test @@ -239,7 +239,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 224 +Content-Length: 229 { "contextResponses": [ @@ -251,7 +251,7 @@ Content-Length: 224 }, "statusCode": { "code": "400", - "details": "one or more of the attributes in the request already exist: [ attr2 ]", + "details": "one or more of the attributes in the request already exist: E1 - [ attr2 ]", "reasonPhrase": "Bad Request" } } @@ -289,7 +289,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 388 +Content-Length: 393 { "contextResponses": [ @@ -319,7 +319,7 @@ Content-Length: 388 }, "statusCode": { "code": "400", - "details": "one or more of the attributes in the request already exist: [ attr2 ]", + "details": "one or more of the attributes in the request already exist: E1 - [ attr2 ]", "reasonPhrase": "Bad Request" } } diff --git a/test/functionalTest/cases/1278_patch_unknown_attribute/patch_unknown_attribute.test b/test/functionalTest/cases/1278_patch_unknown_attribute/patch_unknown_attribute.test index c4dc659d5c..555d8cefda 100644 --- a/test/functionalTest/cases/1278_patch_unknown_attribute/patch_unknown_attribute.test +++ b/test/functionalTest/cases/1278_patch_unknown_attribute/patch_unknown_attribute.test @@ -75,10 +75,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 107 +Content-Length: 112 { - "description": "one or more of the attributes in the request do not exist: [ A2 ]", + "description": "one or more of the attributes in the request do not exist: E1 - [ A2 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test b/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test index c4d4e2c97b..11cca7a827 100644 --- a/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test +++ b/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test @@ -97,10 +97,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 113 +Content-Length: 122 { - "description": "one or more of the attributes in the request do not exist: [ dfgdfgdf ]", + "description": "one or more of the attributes in the request do not exist: room_2 - [ dfgdfgdf ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1682_ngsiv2_entity_id/ngsiv2_entity_id_revisited.test b/test/functionalTest/cases/1682_ngsiv2_entity_id/ngsiv2_entity_id_revisited.test index 98be178f7c..3f08385cc0 100644 --- a/test/functionalTest/cases/1682_ngsiv2_entity_id/ngsiv2_entity_id_revisited.test +++ b/test/functionalTest/cases/1682_ngsiv2_entity_id/ngsiv2_entity_id_revisited.test @@ -148,10 +148,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 107 +Content-Length: 112 { - "description": "one or more of the attributes in the request already exist: [ A ]", + "description": "one or more of the attributes in the request already exist: E1 - [ A ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_APPEND.test b/test/functionalTest/cases/1715_batch_update/batch_update_APPEND.test index f281d869f0..b9e1c5da97 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_APPEND.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_APPEND.test @@ -202,10 +202,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 108 +Content-Length: 113 { - "description": "one or more of the attributes in the request already exist: [ A1 ]", + "description": "one or more of the attributes in the request already exist: E1 - [ A1 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test index 8cd808c80b..1a6828476f 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test @@ -229,15 +229,15 @@ Content-Length: 454 03. Delete E1-A1/A2=null, E2-A1/A2=null (A2 not existing) and E3-A1/A2=null using POST /v2/op/update with delete, get error =========================================================================================================================== -HTTP/1.1 404 Not Found +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 79 +Content-Length: 112 { - "description": "The entity does not have such an attribute", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E2 - [ A2 ]", + "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test new file mode 100644 index 0000000000..a8b308e3d9 --- /dev/null +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test @@ -0,0 +1,342 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +V2 batch update DELETE in non existing and existing attribute (fail in several entities) + +--SHELL-INIT-- +dbInit CB +brokerStart CB 0 + +--SHELL-- + +# +# 01. Create E1/T/A1-A2-A3, E2/T/A1-A3, E3/T/A1-A2-A3 and E4/A2-A3 using POST /v2/op/update with append +# 02. GET /v2/entities to see E1/T/A1-A2-A3, E2/T/A1-A3, E3/T/A1-A2-A3 and E4/A2-A3 +# 03. Delete E1-A1/A2=null, E2-A1/A2=null (A2 not existing), E3-A1/A2=null and E4-A1/A2 (A1 not existing) using POST /v2/op/update with delete, get error +# 04. GET /v2/entities to see E1/T/A3, E2/T/A1-A3, E3/T/A3 and E4/T/A2-A3 in entities +# + +echo "01. Create E1/T/A1-A2-A3, E2/T/A1-A3, E3/T/A1-A2-A3 and E4/A2-A3 using POST /v2/op/update with append" +echo "=====================================================================================================" +payload='{ + "actionType": "append", + "entities": [ + { + "id": "E1", + "type": "T", + "A1": { + "type": "Number", + "value": 11 + }, + "A2": { + "type": "Number", + "value": 12 + }, + "A3": { + "type": "Number", + "value": 13 + } + }, + { + "id": "E2", + "type": "T", + "A1": { + "type": "Number", + "value": 21 + }, + "A3": { + "type": "Number", + "value": 23 + } + }, + { + "id": "E3", + "type": "T", + "A1": { + "type": "Number", + "value": 31 + }, + "A2": { + "type": "Number", + "value": 32 + }, + "A3": { + "type": "Number", + "value": 33 + } + }, + { + "id": "E4", + "type": "T", + "A2": { + "type": "Number", + "value": 42 + }, + "A3": { + "type": "Number", + "value": 43 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. GET /v2/entities to see E1/T/A1-A2-A3, E2/T/A1-A3, E3/T/A1-A2-A3 and E4/A2-A3" +echo "=================================================================================" +orionCurl --url /v2/entities +echo +echo + + +echo "03. Delete E1-A1/A2=null, E2-A1/A2=null (A2 not existing), E3-A1/A2=null and E4-A1/A2 (A1 not existing) using POST /v2/op/update with delete, get error" +echo "=======================================================================================================================================================" +payload='{ + "actionType": "delete", + "entities": [ + { + "id": "E1", + "type": "T", + "A1": { + "type": "Number", + "value": null + }, + "A2": { + "type": "Number", + "value": null + } + }, + { + "id": "E2", + "type": "T", + "A1": { + "type": "Number", + "value": null + }, + "A2": { + "type": "Number", + "value": null + } + }, + { + "id": "E3", + "type": "T", + "A1": { + "type": "Number", + "value": null + }, + "A2": { + "type": "Number", + "value": null + } + }, + { + "id": "E4", + "type": "T", + "A1": { + "type": "Number", + "value": null + }, + "A2": { + "type": "Number", + "value": null + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "04. GET /v2/entities to see E1/T/A3, E2/T/A1-A3, E3/T/A3 and E4/T/A2-A3 in entities" +echo "===================================================================================" +orionCurl --url /v2/entities +echo +echo + + +--REGEXPECT-- +01. Create E1/T/A1-A2-A3, E2/T/A1-A3, E3/T/A1-A2-A3 and E4/A2-A3 using POST /v2/op/update with append +===================================================================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. GET /v2/entities to see E1/T/A1-A2-A3, E2/T/A1-A3, E3/T/A1-A2-A3 and E4/A2-A3 +================================================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 573 + +[ + { + "A1": { + "metadata": {}, + "type": "Number", + "value": 11 + }, + "A2": { + "metadata": {}, + "type": "Number", + "value": 12 + }, + "A3": { + "metadata": {}, + "type": "Number", + "value": 13 + }, + "id": "E1", + "type": "T" + }, + { + "A1": { + "metadata": {}, + "type": "Number", + "value": 21 + }, + "A3": { + "metadata": {}, + "type": "Number", + "value": 23 + }, + "id": "E2", + "type": "T" + }, + { + "A1": { + "metadata": {}, + "type": "Number", + "value": 31 + }, + "A2": { + "metadata": {}, + "type": "Number", + "value": 32 + }, + "A3": { + "metadata": {}, + "type": "Number", + "value": 33 + }, + "id": "E3", + "type": "T" + }, + { + "A2": { + "metadata": {}, + "type": "Number", + "value": 42 + }, + "A3": { + "metadata": {}, + "type": "Number", + "value": 43 + }, + "id": "E4", + "type": "T" + } +] + + +03. Delete E1-A1/A2=null, E2-A1/A2=null (A2 not existing), E3-A1/A2=null and E4-A1/A2 (A1 not existing) using POST /v2/op/update with delete, get error +======================================================================================================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 125 + +{ + "description": "one or more of the attributes in the request do not exist: E2 - [ A2 ], E4 - [ A1 ]", + "error": "Unprocessable" +} + + +04. GET /v2/entities to see E1/T/A3, E2/T/A1-A3, E3/T/A3 and E4/T/A2-A3 in entities +=================================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 381 + +[ + { + "A3": { + "metadata": {}, + "type": "Number", + "value": 13 + }, + "id": "E1", + "type": "T" + }, + { + "A1": { + "metadata": {}, + "type": "Number", + "value": 21 + }, + "A3": { + "metadata": {}, + "type": "Number", + "value": 23 + }, + "id": "E2", + "type": "T" + }, + { + "A3": { + "metadata": {}, + "type": "Number", + "value": 33 + }, + "id": "E3", + "type": "T" + }, + { + "A2": { + "metadata": {}, + "type": "Number", + "value": 42 + }, + "A3": { + "metadata": {}, + "type": "Number", + "value": 43 + }, + "id": "E4", + "type": "T" + } +] + + +--TEARDOWN-- +brokerStop CB +dbDrop CB diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test index 76af634b17..49fe42d62f 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test @@ -199,15 +199,15 @@ Content-Length: 358 03. Delete E1-A2=null, E2-A2=null (not existing) and E3-A2=null using POST /v2/op/update with delete, get error =============================================================================================================== -HTTP/1.1 404 Not Found +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 79 +Content-Length: 112 { - "description": "The entity does not have such an attribute", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E2 - [ A2 ]", + "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_UPDATE.test b/test/functionalTest/cases/1715_batch_update/batch_update_UPDATE.test index 82454712f4..6c5a4b536d 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_UPDATE.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_UPDATE.test @@ -189,10 +189,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 107 +Content-Length: 112 { - "description": "one or more of the attributes in the request do not exist: [ A3 ]", + "description": "one or more of the attributes in the request do not exist: E1 - [ A3 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test b/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test index b94f11236a..2f11f7300b 100644 --- a/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test +++ b/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test @@ -89,10 +89,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 107 +Content-Length: 112 { - "description": "one or more of the attributes in the request do not exist: [ A2 ]", + "description": "one or more of the attributes in the request do not exist: E1 - [ A2 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test b/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test index 11c4dfac26..0da279cbfb 100644 --- a/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test +++ b/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test @@ -83,10 +83,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 107 +Content-Length: 112 { - "description": "one or more of the attributes in the request do not exist: [ A2 ]", + "description": "one or more of the attributes in the request do not exist: E1 - [ A2 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test b/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test index ea20ea72ed..f1a44d10f5 100644 --- a/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test +++ b/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test @@ -117,10 +117,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 112 +Content-Length: 120 { - "description": "one or more of the attributes in the request do not exist: [ pressur ]", + "description": "one or more of the attributes in the request do not exist: Room1 - [ pressur ]", "error": "Unprocessable" } @@ -131,10 +131,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 120 +Content-Length: 128 { - "description": "one or more of the attributes in the request do not exist: [ temper, pressur ]", + "description": "one or more of the attributes in the request do not exist: Room1 - [ temper, pressur ]", "error": "Unprocessable" } From bd76b31edfbd3dfdd2f390b4f93372069bc7a689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 20 Nov 2023 16:46:16 +0100 Subject: [PATCH 007/390] FIX PartialUpdate cases --- src/lib/mongoBackend/mongoUpdateContext.cpp | 19 +++++++++++++++++++ ...date_DELETE_non_existing_and_existing.test | 2 +- ...xisting_and_existing_several_entities.test | 2 +- ...batch_update_DELETE_non_existing_attr.test | 2 +- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/lib/mongoBackend/mongoUpdateContext.cpp b/src/lib/mongoBackend/mongoUpdateContext.cpp index 5c6dabff7c..e5ce9081f5 100644 --- a/src/lib/mongoBackend/mongoUpdateContext.cpp +++ b/src/lib/mongoBackend/mongoUpdateContext.cpp @@ -150,6 +150,8 @@ HttpStatusCode mongoUpdateContext else { /* Process each ContextElement */ + bool atLeastOneSuccess = false; + bool atLeastOneFail = false; for (unsigned int ix = 0; ix < requestP->entityVector.size(); ++ix) { notifSent += processContextElement(requestP->entityVector[ix], @@ -166,6 +168,23 @@ HttpStatusCode mongoUpdateContext notifSent, apiVersion, ngsiv2Flavour); + + if (responseP->oe.code == SccNone) + { + atLeastOneSuccess = true; + } + else + { + atLeastOneFail = true; + } + } + + // Only the PartialUpdate case (at least one success + at least one fail) needs to be "intercepted" here + // Other cases follow the usual response processing flow (whatever it is :) + if (atLeastOneSuccess && atLeastOneFail) + { + // FIXME PR: move to errorMessages.h + responseP->oe.reasonPhrase = "PartialUpdate"; } LM_T(LmtNotifier, ("total notifications sent during update: %d", notifSent)); diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test index 1a6828476f..5aff10fa29 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test @@ -237,7 +237,7 @@ Content-Length: 112 { "description": "one or more of the attributes in the request do not exist: E2 - [ A2 ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test index a8b308e3d9..59bb725854 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test @@ -275,7 +275,7 @@ Content-Length: 125 { "description": "one or more of the attributes in the request do not exist: E2 - [ A2 ], E4 - [ A1 ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test index 49fe42d62f..aa7b436961 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test @@ -207,7 +207,7 @@ Content-Length: 112 { "description": "one or more of the attributes in the request do not exist: E2 - [ A2 ]", - "error": "Unprocessable" + "error": "PartialUpdate" } From 4ed302297ec0d57f0eba5b4613ec4ebbca294d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 20 Nov 2023 17:09:19 +0100 Subject: [PATCH 008/390] FIX avoid magic string for error response --- src/lib/common/errorMessages.h | 1 + src/lib/mongoBackend/mongoUpdateContext.cpp | 3 +-- src/lib/serviceRoutines/postUpdateContext.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/common/errorMessages.h b/src/lib/common/errorMessages.h index 4a5333e75c..1c9f4c93c6 100644 --- a/src/lib/common/errorMessages.h +++ b/src/lib/common/errorMessages.h @@ -98,6 +98,7 @@ #define ERROR_DESC_NOT_FOUND_REGISTRATION "The requested registration has not been found. Check id" #define ERROR_UNPROCESSABLE "Unprocessable" +#define ERROR_PARTIAL_UPDATE "PartialUpdate" #define ERROR_DESC_UNPROCESSABLE_ALREADY_EXISTS "Already Exists" #define ERROR_NO_RESOURCES_AVAILABLE "NoResourcesAvailable" diff --git a/src/lib/mongoBackend/mongoUpdateContext.cpp b/src/lib/mongoBackend/mongoUpdateContext.cpp index e5ce9081f5..0443fd1682 100644 --- a/src/lib/mongoBackend/mongoUpdateContext.cpp +++ b/src/lib/mongoBackend/mongoUpdateContext.cpp @@ -183,8 +183,7 @@ HttpStatusCode mongoUpdateContext // Other cases follow the usual response processing flow (whatever it is :) if (atLeastOneSuccess && atLeastOneFail) { - // FIXME PR: move to errorMessages.h - responseP->oe.reasonPhrase = "PartialUpdate"; + responseP->oe.reasonPhrase = ERROR_PARTIAL_UPDATE; } LM_T(LmtNotifier, ("total notifications sent during update: %d", notifSent)); diff --git a/src/lib/serviceRoutines/postUpdateContext.cpp b/src/lib/serviceRoutines/postUpdateContext.cpp index e287c2b859..54fd689774 100644 --- a/src/lib/serviceRoutines/postUpdateContext.cpp +++ b/src/lib/serviceRoutines/postUpdateContext.cpp @@ -840,7 +840,7 @@ std::string postUpdateContext failing = failing.substr(0, failing.size() - 2); // If some CER (but not all) fail, then it is a partial update - parseDataP->upcrs.res.oe.fill(SccContextElementNotFound, "Some of the following attributes were not updated: { " + failing + " }", "PartialUpdate"); + parseDataP->upcrs.res.oe.fill(SccContextElementNotFound, "Some of the following attributes were not updated: { " + failing + " }", ERROR_PARTIAL_UPDATE); } else // failures == 0 { From 6dcca23279418ff5bb4b87701bced2661a3fc15b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 Nov 2023 11:16:33 +0100 Subject: [PATCH 009/390] ADD stub for partical update test cases --- .../partial_update_action_appendstrict.test | 138 ++++++++++ ..._action_appendstrict_several_entities.test | 222 +++++++++++++++++ .../partial_update_action_delete.test | 142 +++++++++++ ...update_action_delete_several_entities.test | 235 ++++++++++++++++++ .../partial_update_action_update.test | 138 ++++++++++ ...update_action_update_several_entities.test | 222 +++++++++++++++++ 6 files changed, 1097 insertions(+) create mode 100644 test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test new file mode 100644 index 0000000000..1932dc2d62 --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test @@ -0,0 +1,138 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +PartialUpdate response for appendStrict case + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E-A=1 +# 02. POST /v2/op/update appendStrict E A=10, B=20, C=30, get error about already existing attribute A +# 03. Get E, check A=1, B=20, C=30 +# 04. POST /v2/op/update appendStrict E D=400 B=200, get error about about already existing attribute B +# 05. Get E, check A=1, B=20, C=30, D=400 +# + +echo "01. Create entity E-A=1" +echo "=======================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 1, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. POST /v2/op/update appendStrict E A=10, B=20, C=30, get error about already existing attribute A" +echo "====================================================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Get E, check A=1, B=20, C=30" +echo "================================" +orionCurl --url /v2/entities/E +echo +echo + + +echo "04. POST /v2/op/update appendStrict E D=400 B=200, get error about about already existing attribute B" +echo "=====================================================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E", + "D": { + "value": 400, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Get E, check A=1, B=20, C=30, D=400" +echo "=======================================" +orionCurl --url /v2/entities/E +echo +echo + + +--REGEXPECT-- + +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test new file mode 100644 index 0000000000..a43f6e01e3 --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test @@ -0,0 +1,222 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +PartialUpdate response for appendStrict case for several entities + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1-A=1, E2-A=1/B=2, E3-A=1/B=2/C=3 +# 02. POST /v2/op/update appendStrict E/E1/E2 A=10, B=20, C=30, get error about already existing attribute A/B/C in E3, A/B in E2 and A in E1 +# 03. Get entities, E1-A=1/B=20/C=30, E2-A=1/B=2/C=30, E3-A=1/B=2/C=3 +# 04. POST /v2/op/update update E/E1/E2 D=400 B=200, get error about already existing attribute B in E1/E2/E3 +# 05. Get entities, E1-A=1/B=20/C=30/D=400, E2-A=1/B=2/C=30/D=400, E3-A=1/B=2/C=3/D=400 +# + +echo "01. Create entity E1-A=1, E2-A=1/B=2, E3-A=1/B=2/C=3" +echo "====================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + }, + "C": { + "value": 3, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. POST /v2/op/update appendStrict E/E1/E2 A=10, B=20, C=30, get error about already existing attribute A/B/C in E3, A/B in E2 and A in E1" +echo "===========================================================================================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Get entities, E1-A=1/B=20/C=30, E2-A=1/B=2/C=30, E3-A=1/B=2/C=3" +echo "===================================================================" +orionCurl --url /v2/entities +echo +echo + + +echo "04. POST /v2/op/update appendStrict E/E1/E2 D=400 B=200, get error about already existing attribute B in E1/E2/E3" +echo "=================================================================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "D": { + "value": 400, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "D": { + "value": 400, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "D": { + "value": 400, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Get entities, E1-A=1/B=20/C=30/D=400, E2-A=1/B=2/C=30/D=400, E3-A=1/B=2/C=3/D=400" +echo "=====================================================================================" +orionCurl --url /v2/entities +echo +echo + + +--REGEXPECT-- + +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test new file mode 100644 index 0000000000..575bde447a --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test @@ -0,0 +1,142 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +PartialUpdate response for delete case + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E-A=1/D=4 +# 02. POST /v2/op/update delete E A=10, B=20, C=30, get error about entity does not have attribute B and C +# 03. Get E, check D=4 +# 04. POST /v2/op/update delete E D=400 B=200, get error about entity does not have attribute B +# 05. Get E, check no attr +# + +echo "01. Create entity E-A=1/D=4" +echo "===========================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 1, + "type": "Number" + }, + "D": { + "value": 4, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. POST /v2/op/update delete E A=10, B=20, C=30, get error about entity does not have attribute B and C" +echo "========================================================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Get E, check D=4" +echo "====================" +orionCurl --url /v2/entities/E +echo +echo + + +echo "04. POST /v2/op/update delete E D=400 B=200, get error about entity does not have attribute B" +echo "=============================================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E", + "D": { + "value": 400, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Get E, check no attr" +echo "========================" +orionCurl --url /v2/entities/E +echo +echo + + +--REGEXPECT-- + +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test new file mode 100644 index 0000000000..a9d623628a --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test @@ -0,0 +1,235 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +PartialUpdate response for delete case for several entities + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1-A=1/D=4, E2-A=1/B=2/D=4, E3-A=1/B=2/C=3/D=4 +# 02. POST /v2/op/update delete E/E1/E2 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2 +# 03. Get entities, E1-D=4, E2-D=4, E3-D=4 +# 04. POST /v2/op/update update E/E1/E2 D=400 B=200, get error about entity does not have attribute B in E1/E2/E3 +# 05. Get entities without attributes +# + +echo "01. Create entity E1-A=1/D=4, E2-A=1/B=2/D=4, E3-A=1/B=2/C=3/D=4" +echo "================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "D": { + "value": 4, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + }, + "D": { + "value": 4, + "type": "Number" + } + + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + }, + "C": { + "value": 3, + "type": "Number" + }, + "D": { + "value": 4, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. POST /v2/op/update delete E/E1/E2 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2" +echo "================================================================================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Get entities, E1-D=4, E2-D=4, E3-D=4" +echo "========================================" +orionCurl --url /v2/entities +echo +echo + + +echo "04. POST /v2/op/update update E/E1/E2 D=400 B=200, get error about entity does not have attribute B in E1/E2/E3" +echo "===============================================================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T", + "id": "E1", + "D": { + "value": 400, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 100, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 100, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Get entities without attributes" +echo "===================================" +orionCurl --url /v2/entities +echo +echo + + +--REGEXPECT-- + +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test new file mode 100644 index 0000000000..d22d469a0e --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test @@ -0,0 +1,138 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +PartialUpdate response for update case + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E-A=1 +# 02. POST /v2/op/update update E A=10, B=20, C=30, get error about entity does not have attribute B and C +# 03. Get E, check A=10 +# 04. POST /v2/op/update update E A=100 B=200, get error about entity does not have attribute B +# 05. Get E, check A=100 +# + +echo "01. Create entity E-A=1" +echo "=======================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 1, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. POST /v2/op/update update E A=10, B=20, C=30, get error about entity does not have attribute B and C" +echo "========================================================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Get E, check A=10" +echo "=====================" +orionCurl --url /v2/entities/E +echo +echo + + +echo "04. POST /v2/op/update update E A=100 B=200, get error about entity does not have attribute B" +echo "==============================================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 100, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Get E, check A=100" +echo "======================" +orionCurl --url /v2/entities/E +echo +echo + + +--REGEXPECT-- + +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test new file mode 100644 index 0000000000..8a03ad8d85 --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test @@ -0,0 +1,222 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +PartialUpdate response for update case for several entities + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1-A=1, E2-A=1/B=2, E3-A=1/B=2/C=3 +# 02. POST /v2/op/update update E/E1/E2 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2 +# 03. Get entities, E1-A=10, E2-A=10/B=20, E3-A=10/B=20/C=30 +# 04. POST /v2/op/update update E/E1/E2 A=100 B=200, get error about entity does not have attribute B in E1 +# 05. Get entities, E1-A=100, E2-A=100/B=200, E3-A=100/B=200/C=300 +# + +echo "01. Create entity E1-A=1, E2-A=1/B=2, E3-A=1/B=2/C=3" +echo "====================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + }, + "C": { + "value": 3, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. POST /v2/op/update update E/E1/E2 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2" +echo "================================================================================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Get entities, E1-A=10, E2-A=10/B=20, E3-A=10/B=20/C=30" +echo "==========================================================" +orionCurl --url /v2/entities +echo +echo + + +echo "04. POST /v2/op/update update E/E1/E2 A=100 B=200, get error about entity does not have attribute B in E1" +echo "=========================================================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 100, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 100, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 100, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Get entities, E1-A=100, E2-A=100/B=200, E3-A=100/B=200/C=300" +echo "================================================================" +orionCurl --url /v2/entities +echo +echo + + +--REGEXPECT-- + +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file From a515124bf5405d5e3ddf551fd1c28266af329bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 Nov 2023 13:32:20 +0100 Subject: [PATCH 010/390] FIX delete attributes operation and improve error reporting --- CHANGES_NEXT_RELEASE | 2 +- src/lib/mongoBackend/MongoCommonUpdate.cpp | 6 +- src/lib/rest/OrionError.cpp | 21 ++ src/lib/rest/OrionError.h | 1 + ...date_DELETE_non_existing_and_existing.test | 17 +- ...xisting_and_existing_several_entities.test | 22 +- .../partial_update_action_appendstrict.test | 93 +++++ ..._action_appendstrict_several_entities.test | 195 ++++++++++- .../partial_update_action_delete.test | 65 +++- ...update_action_delete_several_entities.test | 119 ++++++- ..._delete_several_entities_non_existing.test | 298 ++++++++++++++++ .../partial_update_action_update.test | 68 ++++ ...update_action_update_several_entities.test | 144 +++++++- ..._update_several_entities_non_existing.test | 320 ++++++++++++++++++ 14 files changed, 1317 insertions(+), 54 deletions(-) create mode 100644 test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e241145fe2..ba5b1b9d9b 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -3,7 +3,7 @@ - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) -- Fix: improve error traces (#4387) +- Fix: on delete attributes operation the lack of attribute in entity doesn't precules attributes which do exist (this way behaving same way as update attributes operation) - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) - Fix: improve logs in MongoDB query logic - Upgrade Debian version from 11.6 to 12.1 in Dockerfile diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index feb5a99bf2..7859fcc24b 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -2618,7 +2618,7 @@ static bool deleteContextAttributeItem alarmMgr.badInput(clientIp, "attribute to be deleted is not found", targetAttr->getName()); ca->found = false; - return false; + //return false; } return true; @@ -4342,7 +4342,7 @@ unsigned int processContextElement } else { - responseP->oe.fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ERROR_NOT_FOUND); + responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); } } } @@ -4350,7 +4350,7 @@ unsigned int processContextElement { cerP->statusCode.fill(SccContextElementNotFound); - responseP->oe.fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ERROR_NOT_FOUND); + responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); responseP->contextElementResponseVector.push_back(cerP); } else /* APPEND or APPEND_STRICT */ diff --git a/src/lib/rest/OrionError.cpp b/src/lib/rest/OrionError.cpp index 9b25f1014d..c3839b631f 100644 --- a/src/lib/rest/OrionError.cpp +++ b/src/lib/rest/OrionError.cpp @@ -85,6 +85,27 @@ void OrionError::fill(HttpStatusCode _code, const std::string& _details, const s +/* **************************************************************************** +* +* OrionError::fillOrAppend - +*/ +void OrionError::fillOrAppend(HttpStatusCode _code, const std::string& fullDetails, const std::string& appendDetail, const std::string& _reasonPhrase) +{ + if (code == SccInvalidModification) + { + // Already filled by a prevoius operation. This can happen in batch update processing + details += appendDetail; + } + else + { + code = _code; + reasonPhrase = _reasonPhrase.empty()? httpStatusCodeString(code) : _reasonPhrase; + details = fullDetails; + } +} + + + /* **************************************************************************** * * OrionError::fill - diff --git a/src/lib/rest/OrionError.h b/src/lib/rest/OrionError.h index 318cb83790..5f7e828911 100644 --- a/src/lib/rest/OrionError.h +++ b/src/lib/rest/OrionError.h @@ -55,6 +55,7 @@ typedef struct OrionError std::string toJsonV1(void); void fill(HttpStatusCode _code, const std::string& _details, const std::string& _reasonPhrase = ""); void fill(const StatusCode& sc); + void fillOrAppend(HttpStatusCode _code, const std::string& fullDetails, const std::string& appendDetail, const std::string& _reasonPhrase); void appendDetails(const std::string& _details); diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test index 5aff10fa29..0711ce279d 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test @@ -33,7 +33,7 @@ brokerStart CB 0 # 01. Create E1/T/A1-A2-A3, E2/T/A1-A3 and E3/T/A1-A2-A3 using POST /v2/op/update with append # 02. GET /v2/entities to see E1/T/A1-A2-A3, E2/T/A1-A3 and E3/T/A1-A2-A3 # 03. Delete E1-A1/A2=null, E2-A1/A2=null (A2 not existing) and E3-A1/A2=null using POST /v2/op/update with delete, get error -# 04. GET /v2/entities to see E1/T/A3, E2/T/A1-A3 and E3/T/A3 in entities +# 04. GET /v2/entities to see E1/T/A3, E2/T/A3 and E3/T/A3 in entities # echo "01. Create E1/T/A1-A2-A3, E2/T/A1-A3 and E3/T/A1-A2-A3 using POST /v2/op/update with append" @@ -147,8 +147,8 @@ echo echo -echo "04. GET /v2/entities to see E1/T/A3, E2/T/A1-A3 and E3/T/A3 in entities" -echo "=======================================================================" +echo "04. GET /v2/entities to see E1/T/A3, E2/T/A3 and E3/T/A3 in entities" +echo "====================================================================" orionCurl --url /v2/entities echo echo @@ -241,13 +241,13 @@ Content-Length: 112 } -04. GET /v2/entities to see E1/T/A3, E2/T/A1-A3 and E3/T/A3 in entities -======================================================================= +04. GET /v2/entities to see E1/T/A3, E2/T/A3 and E3/T/A3 in entities +==================================================================== HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 262 +Content-Length: 214 [ { @@ -260,11 +260,6 @@ Content-Length: 262 "type": "T" }, { - "A1": { - "metadata": {}, - "type": "Number", - "value": 21 - }, "A3": { "metadata": {}, "type": "Number", diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test index 59bb725854..d6f1638a6f 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test @@ -33,7 +33,7 @@ brokerStart CB 0 # 01. Create E1/T/A1-A2-A3, E2/T/A1-A3, E3/T/A1-A2-A3 and E4/A2-A3 using POST /v2/op/update with append # 02. GET /v2/entities to see E1/T/A1-A2-A3, E2/T/A1-A3, E3/T/A1-A2-A3 and E4/A2-A3 # 03. Delete E1-A1/A2=null, E2-A1/A2=null (A2 not existing), E3-A1/A2=null and E4-A1/A2 (A1 not existing) using POST /v2/op/update with delete, get error -# 04. GET /v2/entities to see E1/T/A3, E2/T/A1-A3, E3/T/A3 and E4/T/A2-A3 in entities +# 04. GET /v2/entities to see E1/T/A3, E2/T/A3, E3/T/A3 and E4/T/A3 in entities # echo "01. Create E1/T/A1-A2-A3, E2/T/A1-A3, E3/T/A1-A2-A3 and E4/A2-A3 using POST /v2/op/update with append" @@ -171,8 +171,8 @@ echo echo -echo "04. GET /v2/entities to see E1/T/A3, E2/T/A1-A3, E3/T/A3 and E4/T/A2-A3 in entities" -echo "===================================================================================" +echo "04. GET /v2/entities to see E1/T/A3, E2/T/A3, E3/T/A3 and E4/T/A3 in entities" +echo "=============================================================================" orionCurl --url /v2/entities echo echo @@ -279,13 +279,13 @@ Content-Length: 125 } -04. GET /v2/entities to see E1/T/A3, E2/T/A1-A3, E3/T/A3 and E4/T/A2-A3 in entities -=================================================================================== +04. GET /v2/entities to see E1/T/A3, E2/T/A3, E3/T/A3 and E4/T/A3 in entities +============================================================================= HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 381 +Content-Length: 285 [ { @@ -298,11 +298,6 @@ Content-Length: 381 "type": "T" }, { - "A1": { - "metadata": {}, - "type": "Number", - "value": 21 - }, "A3": { "metadata": {}, "type": "Number", @@ -321,11 +316,6 @@ Content-Length: 381 "type": "T" }, { - "A2": { - "metadata": {}, - "type": "Number", - "value": 42 - }, "A3": { "metadata": {}, "type": "Number", diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test index 1932dc2d62..07630a0d73 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test @@ -127,10 +127,103 @@ echo --REGEXPECT-- +01. Create entity E-A=1 +======================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + +02. POST /v2/op/update appendStrict E A=10, B=20, C=30, get error about already existing attribute A +==================================================================================================== +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 111 +{ + "description": "one or more of the attributes in the request already exist: E - [ A ]", + "error": "Unprocessable" +} + + +03. Get E, check A=1, B=20, C=30 +================================ +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 161 + +{ + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 20 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "id": "E", + "type": "T" +} + + +04. POST /v2/op/update appendStrict E D=400 B=200, get error about about already existing attribute B +===================================================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 111 + +{ + "description": "one or more of the attributes in the request already exist: E - [ B ]", + "error": "Unprocessable" +} + + +05. Get E, check A=1, B=20, C=30, D=400 +======================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 209 + +{ + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 20 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "D": { + "metadata": {}, + "type": "Number", + "value": 400 + }, + "id": "E", + "type": "T" +} --TEARDOWN-- diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test index a43f6e01e3..3539d2c686 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test @@ -31,9 +31,9 @@ brokerStart CB # # 01. Create entity E1-A=1, E2-A=1/B=2, E3-A=1/B=2/C=3 -# 02. POST /v2/op/update appendStrict E/E1/E2 A=10, B=20, C=30, get error about already existing attribute A/B/C in E3, A/B in E2 and A in E1 +# 02. POST /v2/op/update appendStrict E1/E2/E3 A=10, B=20, C=30, get error about already existing attribute A/B/C in E3, A/B in E2 and A in E1 # 03. Get entities, E1-A=1/B=20/C=30, E2-A=1/B=2/C=30, E3-A=1/B=2/C=3 -# 04. POST /v2/op/update update E/E1/E2 D=400 B=200, get error about already existing attribute B in E1/E2/E3 +# 04. POST /v2/op/update update E1/E2/E3 D=400 B=200, get error about already existing attribute B in E1/E2/E3 # 05. Get entities, E1-A=1/B=20/C=30/D=400, E2-A=1/B=2/C=30/D=400, E3-A=1/B=2/C=3/D=400 # @@ -86,8 +86,8 @@ echo echo -echo "02. POST /v2/op/update appendStrict E/E1/E2 A=10, B=20, C=30, get error about already existing attribute A/B/C in E3, A/B in E2 and A in E1" -echo "===========================================================================================================================================" +echo "02. POST /v2/op/update appendStrict E1/E2/E3 A=10, B=20, C=30, get error about already existing attribute A/B/C in E3, A/B in E2 and A in E1" +echo "============================================================================================================================================" payload='{ "actionType": "appendStrict", "entities": @@ -154,8 +154,8 @@ echo echo -echo "04. POST /v2/op/update appendStrict E/E1/E2 D=400 B=200, get error about already existing attribute B in E1/E2/E3" -echo "=================================================================================================================" +echo "04. POST /v2/op/update appendStrict E1/E2/E3 D=400 B=200, get error about already existing attribute B in E1/E2/E3" +echo "==================================================================================================================" payload='{ "actionType": "appendStrict", "entities": @@ -211,11 +211,194 @@ echo --REGEXPECT-- +01. Create entity E1-A=1, E2-A=1/B=2, E3-A=1/B=2/C=3 +==================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. POST /v2/op/update appendStrict E1/E2/E3 A=10, B=20, C=30, get error about already existing attribute A/B/C in E3, A/B in E2 and A in E1 +============================================================================================================================================ +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 145 + +{ + "description": "one or more of the attributes in the request already exist: E1 - [ A ], E2 - [ A, B ], E3 - [ A, B, C ]", + "error": "Unprocessable" +} + +03. Get entities, E1-A=1/B=20/C=30, E2-A=1/B=2/C=30, E3-A=1/B=2/C=3 +=================================================================== +HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 487 + +[ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 20 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "id": "E1", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "id": "E2", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 3 + }, + "id": "E3", + "type": "T" + } +] +04. POST /v2/op/update appendStrict E1/E2/E3 D=400 B=200, get error about already existing attribute B in E1/E2/E3 +================================================================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 136 + +{ + "description": "one or more of the attributes in the request already exist: E1 - [ B ], E2 - [ B ], E3 - [ B ]", + "error": "Unprocessable" +} + + +05. Get entities, E1-A=1/B=20/C=30/D=400, E2-A=1/B=2/C=30/D=400, E3-A=1/B=2/C=3/D=400 +===================================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 631 + +[ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 20 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "D": { + "metadata": {}, + "type": "Number", + "value": 400 + }, + "id": "E1", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "D": { + "metadata": {}, + "type": "Number", + "value": 400 + }, + "id": "E2", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 3 + }, + "D": { + "metadata": {}, + "type": "Number", + "value": 400 + }, + "id": "E3", + "type": "T" + } +] + --TEARDOWN-- brokerStop CB diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test index 575bde447a..a2762ecd29 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test @@ -101,7 +101,7 @@ echo echo "04. POST /v2/op/update delete E D=400 B=200, get error about entity does not have attribute B" echo "=============================================================================================" payload='{ - "actionType": "update", + "actionType": "delete", "entities": [ { @@ -131,10 +131,73 @@ echo --REGEXPECT-- +01. Create entity E-A=1/D=4 +=========================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. POST /v2/op/update delete E A=10, B=20, C=30, get error about entity does not have attribute B and C +======================================================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 113 + +{ + "description": "one or more of the attributes in the request do not exist: E - [ B, C ]", + "error": "Unprocessable" +} + + +03. Get E, check D=4 +==================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 67 + +{ + "D": { + "metadata": {}, + "type": "Number", + "value": 4 + }, + "id": "E", + "type": "T" +} + + +04. POST /v2/op/update delete E D=400 B=200, get error about entity does not have attribute B +============================================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 110 + +{ + "description": "one or more of the attributes in the request do not exist: E - [ B ]", + "error": "Unprocessable" +} + +05. Get E, check no attr +======================== +HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 21 +{ + "id": "E", + "type": "T" +} --TEARDOWN-- diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test index a9d623628a..fa0504848d 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test @@ -31,10 +31,10 @@ brokerStart CB # # 01. Create entity E1-A=1/D=4, E2-A=1/B=2/D=4, E3-A=1/B=2/C=3/D=4 -# 02. POST /v2/op/update delete E/E1/E2 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2 +# 02. POST /v2/op/update delete E1/E2/E3 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2 # 03. Get entities, E1-D=4, E2-D=4, E3-D=4 -# 04. POST /v2/op/update update E/E1/E2 D=400 B=200, get error about entity does not have attribute B in E1/E2/E3 -# 05. Get entities without attributes +# 04. POST /v2/op/update update E1/E2/E3 D=400 B=200, get error about entity does not have attribute B in E1/E2/E3 +# 05. Get entities E1, E2 and E3 without attributes # echo "01. Create entity E1-A=1/D=4, E2-A=1/B=2/D=4, E3-A=1/B=2/C=3/D=4" @@ -99,8 +99,8 @@ echo echo -echo "02. POST /v2/op/update delete E/E1/E2 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2" -echo "================================================================================================================================" +echo "02. POST /v2/op/update delete E1/E2/E3 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2" +echo "=================================================================================================================================" payload='{ "actionType": "delete", "entities": @@ -167,8 +167,8 @@ echo echo -echo "04. POST /v2/op/update update E/E1/E2 D=400 B=200, get error about entity does not have attribute B in E1/E2/E3" -echo "===============================================================================================================" +echo "04. POST /v2/op/update update E1/E2/E3 D=400 B=200, get error about entity does not have attribute B in E1/E2/E3" +echo "================================================================================================================" payload='{ "actionType": "delete", "entities": @@ -188,8 +188,8 @@ payload='{ { "type": "T", "id": "E2", - "A": { - "value": 100, + "D": { + "value": 400, "type": "Number" }, "B": { @@ -200,8 +200,8 @@ payload='{ { "type": "T", "id": "E3", - "A": { - "value": 100, + "D": { + "value": 400, "type": "Number" }, "B": { @@ -216,18 +216,111 @@ echo echo -echo "05. Get entities without attributes" -echo "===================================" +echo "05. Get entities E1, E2 and E3 without attributes" +echo "=================================================" orionCurl --url /v2/entities echo echo --REGEXPECT-- +01. Create entity E1-A=1/D=4, E2-A=1/B=2/D=4, E3-A=1/B=2/C=3/D=4 +================================================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + +02. POST /v2/op/update delete E1/E2/E3 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2 +================================================================================================================================= +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 126 +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 - [ C ]", + "error": "Unprocessable" +} + + +03. Get entities, E1-D=4, E2-D=4, E3-D=4 +======================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 208 + +[ + { + "D": { + "metadata": {}, + "type": "Number", + "value": 4 + }, + "id": "E1", + "type": "T" + }, + { + "D": { + "metadata": {}, + "type": "Number", + "value": 4 + }, + "id": "E2", + "type": "T" + }, + { + "D": { + "metadata": {}, + "type": "Number", + "value": 4 + }, + "id": "E3", + "type": "T" + } +] + + +04. POST /v2/op/update update E1/E2/E3 D=400 B=200, get error about entity does not have attribute B in E1/E2/E3 +================================================================================================================ +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 135 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ B ], E2 - [ B ], E3 - [ B ]", + "error": "Unprocessable" +} + + +05. Get entities E1, E2 and E3 without attributes +================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 70 + +[ + { + "id": "E1", + "type": "T" + }, + { + "id": "E2", + "type": "T" + }, + { + "id": "E3", + "type": "T" + } +] --TEARDOWN-- diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test new file mode 100644 index 0000000000..299df9f73a --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test @@ -0,0 +1,298 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +PartialUpdate response for delete case for several entities one of them non existing + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1-A=1/D=4, E3-A=1/B=2/C=3/D=4 +# 02. POST /v2/op/update delete E1/E2/E3 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and E2 does not exists +# 03. Get entities, E1-D=4, E3-D=4 +# 04. POST /v2/op/update update E1/E2/E3 D=400 B=200, get error about entity does not have attribute B in E1/E3 and entity E2 does not exist +# 05. Get entities E1 and E3 attributes +# + +echo "01. Create entity E1-A=1/D=4, E3-A=1/B=2/C=3/D=4" +echo "================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "D": { + "value": 4, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + }, + "C": { + "value": 3, + "type": "Number" + }, + "D": { + "value": 4, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. POST /v2/op/update delete E1/E2/E3 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and E2 does not exists" +echo "============================================================================================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Get entities, E1-D=4, E3-D=4" +echo "================================" +orionCurl --url /v2/entities +echo +echo + + +echo "04. POST /v2/op/update update E1/E2/E3 D=400 B=200, get error about entity does not have attribute B in E1/E3 and entity E2 does not exist" +echo "==========================================================================================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T", + "id": "E1", + "D": { + "value": 400, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "D": { + "value": 400, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "D": { + "value": 400, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Get entities E1 and E3 without attributes" +echo "=============================================" +orionCurl --url /v2/entities +echo +echo + + +--REGEXPECT-- +01. Create entity E1-A=1/D=4, E3-A=1/B=2/C=3/D=4 +================================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. POST /v2/op/update delete E1/E2/E3 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and E2 does not exists +============================================================================================================================================ +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 134 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 [entity itself]", + "error": "Unprocessable" +} + + +03. Get entities, E1-D=4, E3-D=4 +================================ +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 139 + +[ + { + "D": { + "metadata": {}, + "type": "Number", + "value": 4 + }, + "id": "E1", + "type": "T" + }, + { + "D": { + "metadata": {}, + "type": "Number", + "value": 4 + }, + "id": "E3", + "type": "T" + } +] + + +04. POST /v2/op/update update E1/E2/E3 D=400 B=200, get error about entity does not have attribute B in E1/E3 and entity E2 does not exist +========================================================================================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 143 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ B ], E2 [entity itself], E3 - [ B ]", + "error": "Unprocessable" +} + + +05. Get entities E1 and E3 without attributes +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 47 + +[ + { + "id": "E1", + "type": "T" + }, + { + "id": "E3", + "type": "T" + } +] + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test index d22d469a0e..39307f0ea7 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test @@ -127,10 +127,78 @@ echo --REGEXPECT-- +01. Create entity E-A=1 +======================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + +02. POST /v2/op/update update E A=10, B=20, C=30, get error about entity does not have attribute B and C +======================================================================================================== +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 113 +{ + "description": "one or more of the attributes in the request do not exist: E - [ B, C ]", + "error": "Unprocessable" +} + + +03. Get E, check A=10 +===================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "A": { + "metadata": {}, + "type": "Number", + "value": 10 + }, + "id": "E", + "type": "T" +} + + +04. POST /v2/op/update update E A=100 B=200, get error about entity does not have attribute B +============================================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 110 + +{ + "description": "one or more of the attributes in the request do not exist: E - [ B ]", + "error": "Unprocessable" +} + + +05. Get E, check A=100 +====================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 69 + +{ + "A": { + "metadata": {}, + "type": "Number", + "value": 100 + }, + "id": "E", + "type": "T" +} --TEARDOWN-- diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test index 8a03ad8d85..a9b2e2887c 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test @@ -34,7 +34,7 @@ brokerStart CB # 02. POST /v2/op/update update E/E1/E2 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2 # 03. Get entities, E1-A=10, E2-A=10/B=20, E3-A=10/B=20/C=30 # 04. POST /v2/op/update update E/E1/E2 A=100 B=200, get error about entity does not have attribute B in E1 -# 05. Get entities, E1-A=100, E2-A=100/B=200, E3-A=100/B=200/C=300 +# 05. Get entities, E1-A=100, E2-A=100/B=200, E3-A=100/B=200/C=30 # echo "01. Create entity E1-A=1, E2-A=1/B=2, E3-A=1/B=2/C=3" @@ -203,19 +203,157 @@ echo echo -echo "05. Get entities, E1-A=100, E2-A=100/B=200, E3-A=100/B=200/C=300" -echo "================================================================" +echo "05. Get entities, E1-A=100, E2-A=100/B=200, E3-A=100/B=200/C=30" +echo "===============================================================" orionCurl --url /v2/entities echo echo --REGEXPECT-- +01. Create entity E1-A=1, E2-A=1/B=2, E3-A=1/B=2/C=3 +==================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. POST /v2/op/update update E/E1/E2 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and C in E2 +================================================================================================================================ +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 126 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 - [ C ]", + "error": "Unprocessable" +} + +03. Get entities, E1-A=10, E2-A=10/B=20, E3-A=10/B=20/C=30 +========================================================== +HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 352 + +[ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 10 + }, + "id": "E1", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 10 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 20 + }, + "id": "E2", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 10 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 20 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "id": "E3", + "type": "T" + } +] +04. POST /v2/op/update update E/E1/E2 A=100 B=200, get error about entity does not have attribute B in E1 +========================================================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 111 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ B ]", + "error": "Unprocessable" +} + + +05. Get entities, E1-A=100, E2-A=100/B=200, E3-A=100/B=200/C=30 +=============================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 357 + +[ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 100 + }, + "id": "E1", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 100 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 200 + }, + "id": "E2", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 100 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 200 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "id": "E3", + "type": "T" + } +] + --TEARDOWN-- brokerStop CB diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test new file mode 100644 index 0000000000..dcadc3d607 --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test @@ -0,0 +1,320 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +PartialUpdate response for update case for several entities one of them non existing + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1-A=1, E3-A=1/B=2/C=3 +# 02. POST /v2/op/update update E1/E2/E3 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and E2 does not exist +# 03. Get entities, E1-A=10, E3-A=10/B=20/C=30 +# 04. POST /v2/op/update update E1/E2/E3 A=100 B=200, get error about entity does not have attribute B in E1 and E2 does not exist +# 05. Get entities, E1-A=100, E3-A=100/B=200/C=30 +# + +echo "01. Create entity E1-A=1, E3-A=1/B=2/C=3" +echo "========================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + }, + "C": { + "value": 3, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. POST /v2/op/update update E1/E2/E3 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and E2 does not exist" +echo "===========================================================================================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 10, + "type": "Number" + }, + "B": { + "value": 20, + "type": "Number" + }, + "C": { + "value": 30, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Get entities, E1-A=10, E3-A=10/B=20/C=30" +echo "============================================" +orionCurl --url /v2/entities +echo +echo + + +echo "04. POST /v2/op/update update E1/E2/E3 A=100 B=200, get error about entity does not have attribute B in E1 and E2 does not exist" +echo "================================================================================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 100, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 100, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 100, + "type": "Number" + }, + "B": { + "value": 200, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Get entities, E1-A=100, E3-A=100/B=200/C=30" +echo "===============================================" +orionCurl --url /v2/entities +echo +echo + + +--REGEXPECT-- +01. Create entity E1-A=1, E3-A=1/B=2/C=3 +======================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. POST /v2/op/update update E1/E2/E3 A=10, B=20, C=30, get error about entity does not have attribute B and C in E1 and E2 does not exist +=========================================================================================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 134 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 [entity itself]", + "error": "Unprocessable" +} + + +03. Get entities, E1-A=10, E3-A=10/B=20/C=30 +============================================ +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 235 + +[ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 10 + }, + "id": "E1", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 10 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 20 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "id": "E3", + "type": "T" + } +] + + +04. POST /v2/op/update update E1/E2/E3 A=100 B=200, get error about entity does not have attribute B in E1 and E2 does not exist +================================================================================================================================ +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 131 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ B ], E2 [entity itself]", + "error": "Unprocessable" +} + + +05. Get entities, E1-A=100, E3-A=100/B=200/C=30 +=============================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 238 + +[ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 100 + }, + "id": "E1", + "type": "T" + }, + { + "A": { + "metadata": {}, + "type": "Number", + "value": 100 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 200 + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 30 + }, + "id": "E3", + "type": "T" + } +] + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file From 39b338caaf28583d3200d3540086d6fe93cde10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 Nov 2023 15:12:40 +0100 Subject: [PATCH 011/390] FIX refactor --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 31 +++++----------------- src/lib/rest/OrionError.cpp | 11 -------- src/lib/rest/OrionError.h | 1 - 3 files changed, 7 insertions(+), 36 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 7859fcc24b..287320d6c2 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4451,34 +4451,17 @@ unsigned int processContextElement if (attributeAlreadyExistsError == true) { - if (responseP->oe.code == SccInvalidModification) - { - // Another previous entity had problems. Use appendDetails() - responseP->oe.appendDetails(", " + eP->id + " - " + attributeAlreadyExistsList); - } - else - { - // First entity with this problem. Use fill() - std::string details = "one or more of the attributes in the request already exist: " + eP->id + " - " + attributeAlreadyExistsList; - buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); - responseP->oe.fill(SccInvalidModification, details, ERROR_UNPROCESSABLE); - } + std::string details = "one or more of the attributes in the request already exist: " + eP->id + " - " + attributeAlreadyExistsList; + buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); + responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeAlreadyExistsList, ERROR_UNPROCESSABLE); } if (attributeNotExistingError == true) { - if (responseP->oe.code == SccInvalidModification) - { - // Another previous entity had problems. Use appendDetails() - responseP->oe.appendDetails(", " + eP->id + " - " + attributeNotExistingList); - } - else - { - // First entity with this problem. Use fill() - std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - " + attributeNotExistingList; - buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); - responseP->oe.fill(SccInvalidModification, details, ERROR_UNPROCESSABLE); - } + + std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - " + attributeNotExistingList; + buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); + responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeNotExistingList, ERROR_UNPROCESSABLE); } // Response in responseP diff --git a/src/lib/rest/OrionError.cpp b/src/lib/rest/OrionError.cpp index c3839b631f..180764bfa1 100644 --- a/src/lib/rest/OrionError.cpp +++ b/src/lib/rest/OrionError.cpp @@ -119,17 +119,6 @@ void OrionError::fill(const StatusCode& sc) -/* **************************************************************************** -* -* OrionError::appendDetails - -*/ -void OrionError::appendDetails(const std::string& _details) -{ - details += _details; -} - - - /* **************************************************************************** * * OrionError::smartRender - diff --git a/src/lib/rest/OrionError.h b/src/lib/rest/OrionError.h index 5f7e828911..d656bac8c9 100644 --- a/src/lib/rest/OrionError.h +++ b/src/lib/rest/OrionError.h @@ -56,7 +56,6 @@ typedef struct OrionError void fill(HttpStatusCode _code, const std::string& _details, const std::string& _reasonPhrase = ""); void fill(const StatusCode& sc); void fillOrAppend(HttpStatusCode _code, const std::string& fullDetails, const std::string& appendDetail, const std::string& _reasonPhrase); - void appendDetails(const std::string& _details); private: From d8214e5448205d1404b7fc0e41412bbd890f3b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 Nov 2023 15:48:20 +0100 Subject: [PATCH 012/390] FIX refactor --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 66 ++++++++++------------ 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 287320d6c2..4b034e947c 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -3616,53 +3616,45 @@ static unsigned int updateEntity } // - // Before calling processContextAttributeVector and actually do the work, let's check if the - // request is of type 'append-only' and if we have any problem with attributes already existing. - // - if (action == ActionTypeAppendStrict) + // Calculate list of existing and not existing attributes (used by decide if this was a partial update, full update or full fail + // depending on the operation) + + for (unsigned int ix = 0; ix < eP->attributeVector.size(); ++ix) { - for (unsigned int ix = 0; ix < eP->attributeVector.size(); ++ix) + if (attrs.hasField(dbEncode(eP->attributeVector[ix]->name))) { - if (attrs.hasField(dbEncode(eP->attributeVector[ix]->name))) - { - alarmMgr.badInput(clientIp, "attribute already exists", eP->attributeVector[ix]->name); - *attributeAlreadyExistsError = true; + *attributeAlreadyExistsError = true; - // - // This attribute should now be removed from the 'query' ... - // processContextAttributeVector looks at the 'skip' field - // + // + // If action is ActionTypeAppendStrict, this attribute should now be removed from the 'query' ... + // processContextAttributeVector looks at the 'skip' field + // + if (action == ActionTypeAppendStrict) + { eP->attributeVector[ix]->skip = true; + } - // Add to the list of existing attributes - for the error response - if (*attributeAlreadyExistsList != "[ ") - { - *attributeAlreadyExistsList += ", "; - } - *attributeAlreadyExistsList += eP->attributeVector[ix]->name; + // Add to the list of existing attributes + if (*attributeAlreadyExistsList != "[ ") + { + *attributeAlreadyExistsList += ", "; } + *attributeAlreadyExistsList += eP->attributeVector[ix]->name; } - *attributeAlreadyExistsList += " ]"; - } - - if ((apiVersion == V2) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) - { - for (unsigned int ix = 0; ix < eP->attributeVector.size(); ++ix) + else // !attrs.hasField(dbEncode(eP->attributeVector[ix]->name)) { - if (!attrs.hasField(dbEncode(eP->attributeVector[ix]->name))) - { - *attributeNotExistingError = true; + *attributeNotExistingError = true; - // Add to the list of non existing attributes - for the error response - if (*attributeNotExistingList != "[ ") - { - *attributeNotExistingList += ", "; - } - *attributeNotExistingList += eP->attributeVector[ix]->name; + // Add to the list of non existing attributes + if (*attributeNotExistingList != "[ ") + { + *attributeNotExistingList += ", "; } + *attributeNotExistingList += eP->attributeVector[ix]->name; } - *attributeNotExistingList += " ]"; } + *attributeNotExistingList += " ]"; + *attributeAlreadyExistsList += " ]"; // The logic to detect notification loops is to check that the correlator in the request differs from the last one seen for the entity and, // in addition, the request was sent due to a custom notification @@ -4449,14 +4441,14 @@ unsigned int processContextElement } } - if (attributeAlreadyExistsError == true) + if ((attributeAlreadyExistsError == true) && (action == ActionTypeAppendStrict)) { std::string details = "one or more of the attributes in the request already exist: " + eP->id + " - " + attributeAlreadyExistsList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeAlreadyExistsList, ERROR_UNPROCESSABLE); } - if (attributeNotExistingError == true) + if ((attributeNotExistingError == true) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) { std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - " + attributeNotExistingList; From 2399c31a2411e2cd1dc8e6ab2b4f29ed5a74ccae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 Nov 2023 16:09:40 +0100 Subject: [PATCH 013/390] FIX refactor --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 33 +++++++++++----------- src/lib/mongoBackend/MongoCommonUpdate.h | 17 ++++++++++- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 4b034e947c..4a4ebf10ec 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -3473,9 +3473,9 @@ static unsigned int updateEntity const std::string& xauthToken, Entity* eP, UpdateContextResponse* responseP, - bool* attributeAlreadyExistsError, + unsigned int* attributeAlreadyExistsNumber, std::string* attributeAlreadyExistsList, - bool* attributeNotExistingError, + unsigned int* attributeNotExistingNumber, std::string* attributeNotExistingList, const bool& forcedUpdate, const bool& overrideMetadata, @@ -3486,10 +3486,10 @@ static unsigned int updateEntity ) { // Used to accumulate error response information - *attributeAlreadyExistsError = false; + *attributeAlreadyExistsNumber = 0; *attributeAlreadyExistsList = "[ "; - *attributeNotExistingError = false; + *attributeNotExistingNumber = 0; *attributeNotExistingList = "[ "; const std::string idString = "_id." ENT_ENTITY_ID; @@ -3623,7 +3623,7 @@ static unsigned int updateEntity { if (attrs.hasField(dbEncode(eP->attributeVector[ix]->name))) { - *attributeAlreadyExistsError = true; + (*attributeAlreadyExistsNumber)++;; // // If action is ActionTypeAppendStrict, this attribute should now be removed from the 'query' ... @@ -3643,7 +3643,7 @@ static unsigned int updateEntity } else // !attrs.hasField(dbEncode(eP->attributeVector[ix]->name)) { - *attributeNotExistingError = true; + (*attributeNotExistingNumber)++; // Add to the list of non existing attributes if (*attributeNotExistingList != "[ ") @@ -3694,7 +3694,7 @@ static unsigned int updateEntity // searchContextProviders(tenant, servicePathV, en, eP->attributeVector, cerP); - if (!(attributeAlreadyExistsError && (action == ActionTypeAppendStrict))) + if (!(attributeAlreadyExistsNumber && (action == ActionTypeAppendStrict))) { // Note that CER generation in the case of attributeAlreadyExistsError has its own logic at // processContextElement() function so we need to skip this addition or we will get duplicated @@ -4107,7 +4107,8 @@ unsigned int processContextElement const bool& overrideMetadata, unsigned int notifStartCounter, ApiVersion apiVersion, - Ngsiv2Flavour ngsiv2Flavour + Ngsiv2Flavour ngsiv2Flavour, + UpdateCoverage* updateCoverageP ) { /* Check preconditions */ @@ -4255,11 +4256,11 @@ unsigned int processContextElement unsigned int notifSent = 0; // Used to accumulate error response information, checked at the end - bool attributeAlreadyExistsError = false; - std::string attributeAlreadyExistsList = "[ "; + unsigned int attributeAlreadyExistsNumber = 0; + std::string attributeAlreadyExistsList = "[ "; - bool attributeNotExistingError = false; - std::string attributeNotExistingList = "[ "; + unsigned int attributeNotExistingNumber = 0; + std::string attributeNotExistingList = "[ "; /* Note that the following loop is not executed if result size is 0, which leads to the * 'if' just below to create a new entity */ @@ -4272,9 +4273,9 @@ unsigned int processContextElement xauthToken, eP, responseP, - &attributeAlreadyExistsError, + &attributeAlreadyExistsNumber, &attributeAlreadyExistsList, - &attributeNotExistingError, + &attributeNotExistingNumber, &attributeNotExistingList, forcedUpdate, overrideMetadata, @@ -4441,14 +4442,14 @@ unsigned int processContextElement } } - if ((attributeAlreadyExistsError == true) && (action == ActionTypeAppendStrict)) + if ((attributeAlreadyExistsNumber > 0) && (action == ActionTypeAppendStrict)) { std::string details = "one or more of the attributes in the request already exist: " + eP->id + " - " + attributeAlreadyExistsList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeAlreadyExistsList, ERROR_UNPROCESSABLE); } - if ((attributeNotExistingError == true) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) + if ((attributeNotExistingNumber > 0) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) { std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - " + attributeNotExistingList; diff --git a/src/lib/mongoBackend/MongoCommonUpdate.h b/src/lib/mongoBackend/MongoCommonUpdate.h index 5a22880c26..baefd1a9ec 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.h +++ b/src/lib/mongoBackend/MongoCommonUpdate.h @@ -36,6 +36,20 @@ #define MONGODB_ERROR_WRONGJSON "Can't extract geo keys" + +/* **************************************************************************** +* +* UpdateCoverage - +*/ +typedef enum UpdateCoverage +{ + FULL_SUCCESS = 0, + FULL_FAILURE = 1, + PARTIAL = 2 +} UpdateCoverage; + + + /* **************************************************************************** * * processContextElement - @@ -55,7 +69,8 @@ extern unsigned int processContextElement const bool& overrideMetadata, unsigned int notifStartCounter, ApiVersion apiVersion = V1, - Ngsiv2Flavour ngsiV2Flavour = NGSIV2_NO_FLAVOUR + Ngsiv2Flavour ngsiV2Flavour = NGSIV2_NO_FLAVOUR, + UpdateCoverage* updateCoverageP = NULL ); #endif // SRC_LIB_MONGOBACKEND_MONGOCOMMONUPDATE_H_ From a14931c40b0f9907d5cd8a279bb36d71d48ed0d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 Nov 2023 16:55:14 +0100 Subject: [PATCH 014/390] FIX PartialUpdate detection logic --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 36 +++++++++++++++++++ src/lib/mongoBackend/MongoCommonUpdate.h | 7 ++-- src/lib/mongoBackend/mongoUpdateContext.cpp | 35 ++++++++++++------ .../partial_update_action_appendstrict.test | 4 +-- ..._action_appendstrict_several_entities.test | 4 +-- .../partial_update_action_delete.test | 4 +-- ...update_action_delete_several_entities.test | 4 +-- ..._delete_several_entities_non_existing.test | 4 +-- .../partial_update_action_update.test | 4 +-- ...update_action_update_several_entities.test | 4 +-- ..._update_several_entities_non_existing.test | 4 +-- 11 files changed, 80 insertions(+), 30 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 4a4ebf10ec..e4c1e18a85 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4457,6 +4457,42 @@ unsigned int processContextElement responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeNotExistingList, ERROR_UNPROCESSABLE); } + if (updateCoverageP != NULL) + { + // By default, everyghing gone ok. Next we are going to detect some partial/failure cases + *updateCoverageP = UC_FULL_SUCCESS; + + if ((action == ActionTypeUpdate) || (action == ActionTypeDelete)) + { + // Full failure if the number of the non-existing attributes are all the ones in the request + if (attributeNotExistingNumber == eP->attributeVector.size()) + { + *updateCoverageP = UC_FULL_FAILURE; + } + // Partial failure/success if the number of the non-existing attributes is less than the ones in the request (else branch) + // and at least there was one existing attribute + else if (attributeNotExistingNumber > 0) + { + *updateCoverageP = UC_PARTIAL; + } + } + + if (action == ActionTypeAppendStrict) + { + // Full failure if the number of the existing attributes are all the ones in the request + if (attributeAlreadyExistsNumber == eP->attributeVector.size()) + { + *updateCoverageP = UC_FULL_FAILURE; + } + // Partial failure/success if the number of the existing attributes is less than the ones in the request (else branch) + // and at least there was one non-existing attribute + else if (attributeAlreadyExistsNumber > 0) + { + *updateCoverageP = UC_PARTIAL; + } + } + } + // Response in responseP return notifSent; } diff --git a/src/lib/mongoBackend/MongoCommonUpdate.h b/src/lib/mongoBackend/MongoCommonUpdate.h index baefd1a9ec..dc4463a470 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.h +++ b/src/lib/mongoBackend/MongoCommonUpdate.h @@ -43,9 +43,10 @@ */ typedef enum UpdateCoverage { - FULL_SUCCESS = 0, - FULL_FAILURE = 1, - PARTIAL = 2 + UC_NONE = 0, + UC_FULL_SUCCESS = 1, + UC_FULL_FAILURE = 2, + UC_PARTIAL = 3 } UpdateCoverage; diff --git a/src/lib/mongoBackend/mongoUpdateContext.cpp b/src/lib/mongoBackend/mongoUpdateContext.cpp index 0443fd1682..860a7cdf46 100644 --- a/src/lib/mongoBackend/mongoUpdateContext.cpp +++ b/src/lib/mongoBackend/mongoUpdateContext.cpp @@ -150,10 +150,10 @@ HttpStatusCode mongoUpdateContext else { /* Process each ContextElement */ - bool atLeastOneSuccess = false; - bool atLeastOneFail = false; + UpdateCoverage updateCoverage = UC_NONE; for (unsigned int ix = 0; ix < requestP->entityVector.size(); ++ix) { + UpdateCoverage entityUpdateCoverage; notifSent += processContextElement(requestP->entityVector[ix], responseP, requestP->updateActionType, @@ -167,21 +167,34 @@ HttpStatusCode mongoUpdateContext overrideMetadata, notifSent, apiVersion, - ngsiv2Flavour); - - if (responseP->oe.code == SccNone) - { - atLeastOneSuccess = true; - } - else + ngsiv2Flavour, + &entityUpdateCoverage); + switch(updateCoverage) { - atLeastOneFail = true; + case UC_NONE: + updateCoverage = entityUpdateCoverage; + break; + case UC_FULL_SUCCESS: + if (entityUpdateCoverage != UC_FULL_SUCCESS) + { + updateCoverage = UC_PARTIAL; + } + break; + case UC_FULL_FAILURE: + if (entityUpdateCoverage != UC_FULL_FAILURE) + { + updateCoverage = UC_PARTIAL; + } + break; + case UC_PARTIAL: + updateCoverage = UC_PARTIAL; + break; } } // Only the PartialUpdate case (at least one success + at least one fail) needs to be "intercepted" here // Other cases follow the usual response processing flow (whatever it is :) - if (atLeastOneSuccess && atLeastOneFail) + if (updateCoverage == UC_PARTIAL) { responseP->oe.reasonPhrase = ERROR_PARTIAL_UPDATE; } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test index 07630a0d73..ba43328e5e 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test @@ -145,7 +145,7 @@ Content-Length: 111 { "description": "one or more of the attributes in the request already exist: E - [ A ]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -188,7 +188,7 @@ Content-Length: 111 { "description": "one or more of the attributes in the request already exist: E - [ B ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test index 3539d2c686..0c949c092d 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test @@ -229,7 +229,7 @@ Content-Length: 145 { "description": "one or more of the attributes in the request already exist: E1 - [ A ], E2 - [ A, B ], E3 - [ A, B, C ]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -312,7 +312,7 @@ Content-Length: 136 { "description": "one or more of the attributes in the request already exist: E1 - [ B ], E2 - [ B ], E3 - [ B ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test index a2762ecd29..8feabbe5be 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test @@ -149,7 +149,7 @@ Content-Length: 113 { "description": "one or more of the attributes in the request do not exist: E - [ B, C ]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -182,7 +182,7 @@ Content-Length: 110 { "description": "one or more of the attributes in the request do not exist: E - [ B ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test index fa0504848d..aec5c87f14 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test @@ -242,7 +242,7 @@ Content-Length: 126 { "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 - [ C ]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -295,7 +295,7 @@ Content-Length: 135 { "description": "one or more of the attributes in the request do not exist: E1 - [ B ], E2 - [ B ], E3 - [ B ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test index 299df9f73a..1bd68a7642 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test @@ -225,7 +225,7 @@ Content-Length: 134 { "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 [entity itself]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -269,7 +269,7 @@ Content-Length: 143 { "description": "one or more of the attributes in the request do not exist: E1 - [ B ], E2 [entity itself], E3 - [ B ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test index 39307f0ea7..93999321d6 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test @@ -145,7 +145,7 @@ Content-Length: 113 { "description": "one or more of the attributes in the request do not exist: E - [ B, C ]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -178,7 +178,7 @@ Content-Length: 110 { "description": "one or more of the attributes in the request do not exist: E - [ B ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test index a9b2e2887c..3059d751be 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test @@ -229,7 +229,7 @@ Content-Length: 126 { "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 - [ C ]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -297,7 +297,7 @@ Content-Length: 111 { "description": "one or more of the attributes in the request do not exist: E1 - [ B ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test index dcadc3d607..01904cc414 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test @@ -217,7 +217,7 @@ Content-Length: 134 { "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 [entity itself]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -271,7 +271,7 @@ Content-Length: 131 { "description": "one or more of the attributes in the request do not exist: E1 - [ B ], E2 [entity itself]", - "error": "Unprocessable" + "error": "PartialUpdate" } From 4c2d7b55c94a05bfb762330da6bf04182907eb81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 Nov 2023 17:12:40 +0100 Subject: [PATCH 015/390] FIX ftest cases --- .../append_and_append_only_combinations.test | 2 +- .../ngsiv2_entity_id_revisited.test | 10 +++++----- .../patch_non_existing_attribute.test | 2 +- .../PATCH_offending_attributes.test | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/functionalTest/cases/0981_POST_v2_attributes/append_and_append_only_combinations.test b/test/functionalTest/cases/0981_POST_v2_attributes/append_and_append_only_combinations.test index de25c1bcc8..7410ba41cd 100644 --- a/test/functionalTest/cases/0981_POST_v2_attributes/append_and_append_only_combinations.test +++ b/test/functionalTest/cases/0981_POST_v2_attributes/append_and_append_only_combinations.test @@ -267,7 +267,7 @@ Content-Length: 116 { "description": "one or more of the attributes in the request already exist: E1 - [ attr4 ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/1682_ngsiv2_entity_id/ngsiv2_entity_id_revisited.test b/test/functionalTest/cases/1682_ngsiv2_entity_id/ngsiv2_entity_id_revisited.test index 3f08385cc0..9b23e25341 100644 --- a/test/functionalTest/cases/1682_ngsiv2_entity_id/ngsiv2_entity_id_revisited.test +++ b/test/functionalTest/cases/1682_ngsiv2_entity_id/ngsiv2_entity_id_revisited.test @@ -74,8 +74,8 @@ orionCurl --url /v2/entities/E1/attrs --payload "$payload" echo echo -echo "04. POST /v2/entities/E1?options=append A and B (and check 400 Bad Request)" -echo "===========================================================================" +echo "04. POST /v2/entities/E1?options=append A and B (and check 422 Unprocessable Content)" +echo "=====================================================================================" payload='{ "A": { "value": 30 @@ -142,8 +142,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -04. POST /v2/entities/E1?options=append A and B (and check 400 Bad Request) -=========================================================================== +04. POST /v2/entities/E1?options=append A and B (and check 422 Unprocessable Content) +===================================================================================== HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -152,7 +152,7 @@ Content-Length: 112 { "description": "one or more of the attributes in the request already exist: E1 - [ A ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test b/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test index 2f11f7300b..7c84e5ef49 100644 --- a/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test +++ b/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test @@ -93,7 +93,7 @@ Content-Length: 112 { "description": "one or more of the attributes in the request do not exist: E1 - [ A2 ]", - "error": "Unprocessable" + "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test b/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test index f1a44d10f5..0ccf23b652 100644 --- a/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test +++ b/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test @@ -77,7 +77,7 @@ echo '03. PATCH /v2/entities/Room1/attrs, to update existing attribute (temperat echo '====================================================================================================================================================================' payload='{ "temperature": { - "value": 24, + "value": 25, "type": "Float" }, "temper": { @@ -121,7 +121,7 @@ Content-Length: 120 { "description": "one or more of the attributes in the request do not exist: Room1 - [ pressur ]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -135,7 +135,7 @@ Content-Length: 128 { "description": "one or more of the attributes in the request do not exist: Room1 - [ temper, pressur ]", - "error": "Unprocessable" + "error": "PartialUpdate" } @@ -158,7 +158,7 @@ Content-Length: 141 "temperature": { "metadata": {}, "type": "Float", - "value": 24 + "value": 25 }, "type": "Thing" } From 59cf0771ef36bdbc50b31d8e084751ab90097b29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 Nov 2023 17:34:48 +0100 Subject: [PATCH 016/390] FIX broker NGSIv1 test case --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index e4c1e18a85..be35550e7b 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4442,14 +4442,14 @@ unsigned int processContextElement } } - if ((attributeAlreadyExistsNumber > 0) && (action == ActionTypeAppendStrict)) + if ((apiVersion == V2) && (attributeAlreadyExistsNumber > 0) && (action == ActionTypeAppendStrict)) { std::string details = "one or more of the attributes in the request already exist: " + eP->id + " - " + attributeAlreadyExistsList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeAlreadyExistsList, ERROR_UNPROCESSABLE); } - if ((attributeNotExistingNumber > 0) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) + if ((apiVersion == V2) && (attributeNotExistingNumber > 0) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) { std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - " + attributeNotExistingList; From 141dc915d1227b2d2217a055bbb172f2ba766a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 23 Nov 2023 11:24:05 +0100 Subject: [PATCH 017/390] FIX minor fixes --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index be35550e7b..cc5f16e4f7 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -3694,9 +3694,9 @@ static unsigned int updateEntity // searchContextProviders(tenant, servicePathV, en, eP->attributeVector, cerP); - if (!(attributeAlreadyExistsNumber && (action == ActionTypeAppendStrict))) + if (!((*attributeAlreadyExistsNumber) > 0 && (action == ActionTypeAppendStrict))) { - // Note that CER generation in the case of attributeAlreadyExistsError has its own logic at + // Note that CER generation in the case of attributeAlreadyExistsNumber has its own logic at // processContextElement() function so we need to skip this addition or we will get duplicated // CER responseP->contextElementResponseVector.push_back(cerP); @@ -4442,7 +4442,7 @@ unsigned int processContextElement } } - if ((apiVersion == V2) && (attributeAlreadyExistsNumber > 0) && (action == ActionTypeAppendStrict)) + if ((attributeAlreadyExistsNumber > 0) && (action == ActionTypeAppendStrict)) { std::string details = "one or more of the attributes in the request already exist: " + eP->id + " - " + attributeAlreadyExistsList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); From 692b788631364c15ba6d3039c92421be1debda68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Nov 2023 13:51:42 +0100 Subject: [PATCH 018/390] ADD ftest covering all combination cases --- .../all_combinations_appendstrict.test | 390 ++++++++++++++ .../all_combinations_delete.test | 510 ++++++++++++++++++ .../all_combinations_update.test | 391 ++++++++++++++ 3 files changed, 1291 insertions(+) create mode 100644 test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test new file mode 100644 index 0000000000..d72fcb6cc8 --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test @@ -0,0 +1,390 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +All fail/success/partial combinations for appendStrict actionType + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create E1-A/B, E2-A/B +# 02. Update E1-C/D (full success) + E2-C/D (full success): OK +# 03. Update E1-A/B (full fail) + E2-A/B (full fail): Unprocessable +# 04. Update E1-E/F (full success) + E2-A/B (full fail): PartialUpdate +# 05. Update E1-A/G (partial) + E2-E/F (full success): PartialUpdate +# 06. Update E1-A/H (partial) + E3-A/B (full fail): PartialUpdate +# 07. Update E1-A/I (partial) + E2-A/G (partial): PartialUpdate +# + +echo "01. Create E1-A/B, E2-A/B" +echo "=========================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. Update E1-C/D (full success) + E2-C/D (full success): OK" +echo "============================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "C": { + "value": 1, + "type": "Number" + }, + "D": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "C": { + "value": 1, + "type": "Number" + }, + "D": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Update E1-A/B (full fail) + E2-A/B (full fail): Unprocessable" +echo "=======================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "04. Update E1-E/F (full success) + E2-A/B (full fail): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "E": { + "value": 1, + "type": "Number" + }, + "F": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Update E1-A/G (partial) + E2-E/F (full success): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "G": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "E": { + "value": 1, + "type": "Number" + }, + "F": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "06. Update E1-A/H (partial) + E3-A/B (full fail): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "H": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "07. Update E1-A/I (partial) + E2-A/G (partial): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "I": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "G": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. Create E1-A/B, E2-A/B +========================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. Update E1-C/D (full success) + E2-C/D (full success): OK +============================================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +03. Update E1-A/B (full fail) + E2-A/B (full fail): Unprocessable +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 130 + +{ + "description": "one or more of the attributes in the request already exist: E1 - [ A, B ], E2 - [ A, B ]", + "error": "Unprocessable" +} + + +04. Update E1-E/F (full success) + E2-A/B (full fail): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 115 + +{ + "description": "one or more of the attributes in the request already exist: E2 - [ A, B ]", + "error": "PartialUpdate" +} + + +05. Update E1-A/G (partial) + E2-E/F (full success): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 112 + +{ + "description": "one or more of the attributes in the request already exist: E1 - [ A ]", + "error": "PartialUpdate" +} + + +06. Update E1-A/H (partial) + E3-A/B (full fail): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 127 + +{ + "description": "one or more of the attributes in the request already exist: E1 - [ A ], E2 - [ A, B ]", + "error": "PartialUpdate" +} + + +07. Update E1-A/I (partial) + E2-A/G (partial): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 124 + +{ + "description": "one or more of the attributes in the request already exist: E1 - [ A ], E2 - [ A ]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test b/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test new file mode 100644 index 0000000000..afad667f8c --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test @@ -0,0 +1,510 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +All fail/success/partial combinations for delete actionType + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create E1-A/B, E2-A/B (6 times, each one with a different type so no interference between update) +# 02. Update E1-A/B (full success) + E2-A/B (full success): OK +# 03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable +# 04. Update E1-A/B (full success) + E3-A/B (full fail): PartialUpdate +# 05. Update E1-A/C (partial) + E2-A/B (full success): PartialUpdate +# 06. Update E1-A/C (partial) + E3-A/A (full fail): PartialUpdate +# 07. Update E1-A/C (partial) + E2-A/C (partial): PartialUpdate +# + +echo "01. Create E1-A/B, E2-A/B (6 times, each one with a different type so no interference between update)" +echo "=====================================================================================================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T2", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T2", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T3", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T3", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T4", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T4", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T5", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T5", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T6", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T6", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T7", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T7", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. Update E1-A/B (full success) + E2-A/B (full success): OK" +echo "============================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T2", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T2", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable" +echo "=======================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T3", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T3", + "id": "E4", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "04. Update E1-A/B (full success) + E3-A/B (full fail): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T4", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T4", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Update E1-A/C (partial) + E2-A/B (full success): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T5", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T5", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "06. Update E1-A/C (partial) + E3-A/B (full fail): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T6", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T6", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "07. Update E1-A/C (partial) + E2-A/C (partial): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "delete", + "entities": + [ + { + "type": "T7", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T7", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. Create E1-A/B, E2-A/B (6 times, each one with a different type so no interference between update) +===================================================================================================== +HTTP/1.1 204 No Content +Date: Fri, 24 Nov 2023 12:11:12 GMT +Fiware-Correlator: 8f1e6af0-8ac2-11ee-a0dd-000c29e06b83 + + + +02. Update E1-A/B (full success) + E2-A/B (full success): OK +============================================================ +HTTP/1.1 204 No Content +Date: Fri, 24 Nov 2023 12:11:12 GMT +Fiware-Correlator: 8f30e464-8ac2-11ee-a2a7-000c29e06b83 + + + +03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable +======================================================================= +HTTP/1.1 404 Not Found +Date: Fri, 24 Nov 2023 12:11:12 GMT +Fiware-Correlator: 8f3833cc-8ac2-11ee-96ba-000c29e06b83 +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id. FIX?", + "error": "NotFound" +} + + +04. Update E1-A/B (full success) + E3-A/B (full fail): PartialUpdate +======================================================================= +HTTP/1.1 404 Not Found +Date: Fri, 24 Nov 2023 12:11:12 GMT +Fiware-Correlator: 8f54372a-8ac2-11ee-a6a7-000c29e06b83 +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id. FIX", + "error": "NotFound" +} + + +05. Update E1-A/C (partial) + E2-A/B (full success): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: Fri, 24 Nov 2023 12:11:12 GMT +Fiware-Correlator: 8f6c9cc0-8ac2-11ee-8293-000c29e06b83 +Content-Type: application/json +Content-Length: 111 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ C ]", + "error": "PartialUpdate" +} + + +06. Update E1-A/C (partial) + E3-A/B (full fail): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: Fri, 24 Nov 2023 12:11:13 GMT +Fiware-Correlator: 8f89545a-8ac2-11ee-aa04-000c29e06b83 +Content-Type: application/json +Content-Length: 131 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ C ], E3 [entity itself]", + "error": "PartialUpdate" +} + + +07. Update E1-A/C (partial) + E2-A/C (partial): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 123 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ C ], E2 - [ C ]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test b/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test new file mode 100644 index 0000000000..8a632022a5 --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test @@ -0,0 +1,391 @@ + +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +All fail/success/partial combinations for update actionType + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create E1-A/B, E2-A/B +# 02. Update E1-A/B (full success) + E2-A/B (full success): OK +# 03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable +# 04. Update E1-A/B (full success) + E3-A/B (full fail): PartialUpdate +# 05. Update E1-A/C (partial) + E2-A/B (full success): PartialUpdate +# 06. Update E1-A/C (partial) + E3-A/B (full fail): PartialUpdate +# 07. Update E1-A/C (partial) + E2-A/C (partial): PartialUpdate +# + +echo "01. Create E1-A/B, E2-A/B" +echo "=========================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. Update E1-A/B (full success) + E2-A/B (full success): OK" +echo "============================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable" +echo "=======================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } + }, + { + "type": "T", + "id": "E4", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "04. Update E1-A/B (full success) + E3-A/B (full fail): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "05. Update E1-A/C (partial) + E2-A/B (full success): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "06. Update E1-A/C (partial) + E3-A/C (full fail): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } + }, + { + "type": "T", + "id": "E3", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "07. Update E1-A/C (partial) + E2-A/C (partial): PartialUpdate" +echo "=======================================================================" +payload='{ + "actionType": "update", + "entities": + [ + { + "type": "T", + "id": "E1", + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } + }, + { + "type": "T", + "id": "E2", + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. Create E1-A/B, E2-A/B +========================= +HTTP/1.1 204 No Content +Date: Fri, 24 Nov 2023 12:44:45 GMT +Fiware-Correlator: 3ebe9756-8ac7-11ee-8889-000c29e06b83 + + + +02. Update E1-A/B (full success) + E2-A/B (full success): OK +============================================================ +HTTP/1.1 204 No Content +Date: Fri, 24 Nov 2023 12:44:45 GMT +Fiware-Correlator: 3ec7f8dc-8ac7-11ee-935d-000c29e06b83 + + + +03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable +======================================================================= +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id. FIX?", + "error": "NotFound" +} + + +04. Update E1-A/B (full success) + E3-A/B (full fail): PartialUpdate +======================================================================= +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id. FIX", + "error": "NotFound" +} + + +05. Update E1-A/C (partial) + E2-A/B (full success): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 111 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ C ]", + "error": "PartialUpdate" +} + + +06. Update E1-A/C (partial) + E3-A/C (full fail): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 131 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ C ], E3 [entity itself]", + "error": "PartialUpdate" +} + + +07. Update E1-A/C (partial) + E2-A/C (partial): PartialUpdate +======================================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 123 + +{ + "description": "one or more of the attributes in the request do not exist: E1 - [ C ], E2 - [ C ]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file From ebf656c22aaaf228e671fc3633077025fcf65f73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Nov 2023 17:09:40 +0100 Subject: [PATCH 019/390] FIX PartialUpdate cases --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 24 ++++++++--- src/lib/rest/OrionError.cpp | 4 +- .../batch_update_DELETE.test | 8 ++-- .../batch_update_error_entity_not_found.test | 24 +++++------ .../all_combinations_delete.test | 40 +++++++++---------- .../all_combinations_update.test | 24 +++++------ 6 files changed, 68 insertions(+), 56 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index cc5f16e4f7..9196f30a51 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4111,6 +4111,9 @@ unsigned int processContextElement UpdateCoverage* updateCoverageP ) { + // By default, assume everything is gone to be ok. Next in this function we are going to check for some partial/failure cases + *updateCoverageP = UC_FULL_SUCCESS; + /* Check preconditions */ if (!contextElementPreconditionsCheck(eP, responseP, action, apiVersion)) { @@ -4335,7 +4338,13 @@ unsigned int processContextElement } else { - responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); + //responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); + std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - [entity itself]"; + responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " [entity itself]", ERROR_UNPROCESSABLE); + if (updateCoverageP != NULL) + { + *updateCoverageP = UC_FULL_FAILURE; + } } } } @@ -4343,8 +4352,14 @@ unsigned int processContextElement { cerP->statusCode.fill(SccContextElementNotFound); - responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); + //responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); + std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - [entity itself]"; + responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " [entity itself]", ERROR_UNPROCESSABLE); responseP->contextElementResponseVector.push_back(cerP); + if (updateCoverageP != NULL) + { + *updateCoverageP = UC_FULL_FAILURE; + } } else /* APPEND or APPEND_STRICT */ { @@ -4444,6 +4459,7 @@ unsigned int processContextElement if ((attributeAlreadyExistsNumber > 0) && (action == ActionTypeAppendStrict)) { + // FIXME PR move to errorsMesage.h std::string details = "one or more of the attributes in the request already exist: " + eP->id + " - " + attributeAlreadyExistsList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeAlreadyExistsList, ERROR_UNPROCESSABLE); @@ -4451,7 +4467,6 @@ unsigned int processContextElement if ((apiVersion == V2) && (attributeNotExistingNumber > 0) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) { - std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - " + attributeNotExistingList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeNotExistingList, ERROR_UNPROCESSABLE); @@ -4459,9 +4474,6 @@ unsigned int processContextElement if (updateCoverageP != NULL) { - // By default, everyghing gone ok. Next we are going to detect some partial/failure cases - *updateCoverageP = UC_FULL_SUCCESS; - if ((action == ActionTypeUpdate) || (action == ActionTypeDelete)) { // Full failure if the number of the non-existing attributes are all the ones in the request diff --git a/src/lib/rest/OrionError.cpp b/src/lib/rest/OrionError.cpp index 180764bfa1..eed3f0b7c0 100644 --- a/src/lib/rest/OrionError.cpp +++ b/src/lib/rest/OrionError.cpp @@ -91,9 +91,9 @@ void OrionError::fill(HttpStatusCode _code, const std::string& _details, const s */ void OrionError::fillOrAppend(HttpStatusCode _code, const std::string& fullDetails, const std::string& appendDetail, const std::string& _reasonPhrase) { - if (code == SccInvalidModification) + if (code == _code) { - // Already filled by a prevoius operation. This can happen in batch update processing + // Already filled by a previous operation. This can happen in batch update processing details += appendDetail; } else diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE.test index 8568b6fbfb..73c69b1b76 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE.test @@ -132,15 +132,15 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) 04. Delete E1 again, see it fail ================================ -HTTP/1.1 404 Not Found +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 95 +Content-Length: 121 { - "description": "The requested entity has not been found. Check type and id", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E1 - [entity itself]", + "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_error_entity_not_found.test b/test/functionalTest/cases/1715_batch_update/batch_update_error_entity_not_found.test index 503dfb3356..e2bd1228a7 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_error_entity_not_found.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_error_entity_not_found.test @@ -86,43 +86,43 @@ echo --REGEXPECT-- 01. Batch Update with actionType UPDATE on non-existing entity, see error ========================================================================= -HTTP/1.1 404 Not Found +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 95 +Content-Length: 120 { - "description": "The requested entity has not been found. Check type and id", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E - [entity itself]", + "error": "Unprocessable" } 02. Batch Update with actionType DELETE on non-existing entity, see error ========================================================================= -HTTP/1.1 404 Not Found +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 95 +Content-Length: 120 { - "description": "The requested entity has not been found. Check type and id", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E - [entity itself]", + "error": "Unprocessable" } 03. Batch Update with actionType REPLACE on non-existing entity, see error ========================================================================== -HTTP/1.1 404 Not Found +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 95 +Content-Length: 120 { - "description": "The requested entity has not been found. Check type and id", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E - [entity itself]", + "error": "Unprocessable" } diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test b/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test index afad667f8c..c6fa5be4bc 100644 --- a/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test +++ b/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test @@ -422,52 +422,52 @@ echo 01. Create E1-A/B, E2-A/B (6 times, each one with a different type so no interference between update) ===================================================================================================== HTTP/1.1 204 No Content -Date: Fri, 24 Nov 2023 12:11:12 GMT -Fiware-Correlator: 8f1e6af0-8ac2-11ee-a0dd-000c29e06b83 +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) 02. Update E1-A/B (full success) + E2-A/B (full success): OK ============================================================ HTTP/1.1 204 No Content -Date: Fri, 24 Nov 2023 12:11:12 GMT -Fiware-Correlator: 8f30e464-8ac2-11ee-a2a7-000c29e06b83 +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) 03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable ======================================================================= -HTTP/1.1 404 Not Found -Date: Fri, 24 Nov 2023 12:11:12 GMT -Fiware-Correlator: 8f3833cc-8ac2-11ee-96ba-000c29e06b83 +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 95 +Content-Length: 141 { - "description": "The requested entity has not been found. Check type and id. FIX?", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E3 - [entity itself], E4 [entity itself]", + "error": "Unprocessable" } 04. Update E1-A/B (full success) + E3-A/B (full fail): PartialUpdate ======================================================================= -HTTP/1.1 404 Not Found -Date: Fri, 24 Nov 2023 12:11:12 GMT -Fiware-Correlator: 8f54372a-8ac2-11ee-a6a7-000c29e06b83 +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 95 +Content-Length: 121 { - "description": "The requested entity has not been found. Check type and id. FIX", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E3 - [entity itself]", + "error": "PartialUpdate" } 05. Update E1-A/C (partial) + E2-A/B (full success): PartialUpdate ======================================================================= HTTP/1.1 422 Unprocessable Content -Date: Fri, 24 Nov 2023 12:11:12 GMT -Fiware-Correlator: 8f6c9cc0-8ac2-11ee-8293-000c29e06b83 +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json Content-Length: 111 @@ -480,8 +480,8 @@ Content-Length: 111 06. Update E1-A/C (partial) + E3-A/B (full fail): PartialUpdate ======================================================================= HTTP/1.1 422 Unprocessable Content -Date: Fri, 24 Nov 2023 12:11:13 GMT -Fiware-Correlator: 8f89545a-8ac2-11ee-aa04-000c29e06b83 +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json Content-Length: 131 diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test b/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test index 8a632022a5..dff1cdc418 100644 --- a/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test +++ b/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test @@ -303,44 +303,44 @@ echo 01. Create E1-A/B, E2-A/B ========================= HTTP/1.1 204 No Content -Date: Fri, 24 Nov 2023 12:44:45 GMT -Fiware-Correlator: 3ebe9756-8ac7-11ee-8889-000c29e06b83 +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) 02. Update E1-A/B (full success) + E2-A/B (full success): OK ============================================================ HTTP/1.1 204 No Content -Date: Fri, 24 Nov 2023 12:44:45 GMT -Fiware-Correlator: 3ec7f8dc-8ac7-11ee-935d-000c29e06b83 +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) 03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable ======================================================================= -HTTP/1.1 404 Not Found +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 95 +Content-Length: 141 { - "description": "The requested entity has not been found. Check type and id. FIX?", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E3 - [entity itself], E4 [entity itself]", + "error": "Unprocessable" } 04. Update E1-A/B (full success) + E3-A/B (full fail): PartialUpdate ======================================================================= -HTTP/1.1 404 Not Found +HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 95 +Content-Length: 121 { - "description": "The requested entity has not been found. Check type and id. FIX", - "error": "NotFound" + "description": "one or more of the attributes in the request do not exist: E3 - [entity itself]", + "error": "PartialUpdate" } From 06ca9c952e0b34960ba6be97f015e007722e69ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 Nov 2023 09:24:17 +0100 Subject: [PATCH 020/390] FIX improve error response for cases 1715 and 3499 --- src/lib/common/errorMessages.h | 3 ++ src/lib/mongoBackend/MongoCommonUpdate.cpp | 27 +++++++------- src/lib/mongoBackend/MongoCommonUpdate.h | 9 ++--- src/lib/mongoBackend/mongoUpdateContext.cpp | 36 ++++++++++++++++--- src/lib/rest/OrionError.cpp | 34 ++++++++++-------- src/lib/rest/OrionError.h | 1 + .../batch_update_APPEND.test | 4 +-- .../batch_update_DELETE.test | 8 ++--- ...date_DELETE_non_existing_and_existing.test | 4 +-- ...xisting_and_existing_several_entities.test | 4 +-- ...batch_update_DELETE_non_existing_attr.test | 4 +-- .../batch_update_UPDATE.test | 4 +-- .../batch_update_error_entity_not_found.test | 24 ++++++------- .../all_combinations_appendstrict.test | 20 +++++------ .../all_combinations_delete.test | 32 ++++++++--------- .../all_combinations_update.test | 32 ++++++++--------- .../partial_update_action_appendstrict.test | 8 ++--- ..._action_appendstrict_several_entities.test | 8 ++--- .../partial_update_action_delete.test | 8 ++--- ...update_action_delete_several_entities.test | 8 ++--- ..._delete_several_entities_non_existing.test | 8 ++--- .../partial_update_action_update.test | 8 ++--- ...update_action_update_several_entities.test | 8 ++--- ..._update_several_entities_non_existing.test | 8 ++--- 24 files changed, 173 insertions(+), 137 deletions(-) diff --git a/src/lib/common/errorMessages.h b/src/lib/common/errorMessages.h index 1c9f4c93c6..a638981806 100644 --- a/src/lib/common/errorMessages.h +++ b/src/lib/common/errorMessages.h @@ -100,6 +100,9 @@ #define ERROR_UNPROCESSABLE "Unprocessable" #define ERROR_PARTIAL_UPDATE "PartialUpdate" #define ERROR_DESC_UNPROCESSABLE_ALREADY_EXISTS "Already Exists" +#define ERROR_DESC_UNPROCESSABLE_ATTR_ALREADY_EXISTS "one or more of the attributes in the request already exist: " + +#define ERROR_DESC_DO_NOT_EXIT "do not exist: " #define ERROR_NO_RESOURCES_AVAILABLE "NoResourcesAvailable" #define ERROR_DESC_NO_RESOURCES_AVAILABLE_GEOLOC "You cannot use more than one geo location attribute when creating an entity. Use ignoreType metadata if you want to add additional informative locations." diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 9196f30a51..6af0df9bcd 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4112,7 +4112,7 @@ unsigned int processContextElement ) { // By default, assume everything is gone to be ok. Next in this function we are going to check for some partial/failure cases - *updateCoverageP = UC_FULL_SUCCESS; + *updateCoverageP = UC_SUCCESS; /* Check preconditions */ if (!contextElementPreconditionsCheck(eP, responseP, action, apiVersion)) @@ -4339,11 +4339,11 @@ unsigned int processContextElement else { //responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); - std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - [entity itself]"; - responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " [entity itself]", ERROR_UNPROCESSABLE); + std::string details = ERROR_DESC_DO_NOT_EXIT + eP->id + "/" + eP->type + " - [entity itself]"; + responseP->oe.fillOrAppend(SccContextElementNotFound, details, ", " + eP->id + "/" + eP->type + " [entity itself]", ERROR_NOT_FOUND); if (updateCoverageP != NULL) { - *updateCoverageP = UC_FULL_FAILURE; + *updateCoverageP = UC_ENTITY_NOT_FOUND; } } } @@ -4353,12 +4353,12 @@ unsigned int processContextElement cerP->statusCode.fill(SccContextElementNotFound); //responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); - std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - [entity itself]"; - responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " [entity itself]", ERROR_UNPROCESSABLE); + std::string details = ERROR_DESC_DO_NOT_EXIT + eP->id + "/" + eP->type + " - [entity itself]"; + responseP->oe.fillOrAppend(SccContextElementNotFound, details, ", " + eP->id + "/" + eP->type + " [entity itself]", ERROR_NOT_FOUND); responseP->contextElementResponseVector.push_back(cerP); if (updateCoverageP != NULL) { - *updateCoverageP = UC_FULL_FAILURE; + *updateCoverageP = UC_ENTITY_NOT_FOUND; } } else /* APPEND or APPEND_STRICT */ @@ -4459,17 +4459,16 @@ unsigned int processContextElement if ((attributeAlreadyExistsNumber > 0) && (action == ActionTypeAppendStrict)) { - // FIXME PR move to errorsMesage.h - std::string details = "one or more of the attributes in the request already exist: " + eP->id + " - " + attributeAlreadyExistsList; + std::string details = ERROR_DESC_UNPROCESSABLE_ATTR_ALREADY_EXISTS + eP->id + "/" + eP->type + " - " + attributeAlreadyExistsList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); - responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeAlreadyExistsList, ERROR_UNPROCESSABLE); + responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + "/" + eP->type + " - " + attributeAlreadyExistsList, ERROR_UNPROCESSABLE); } if ((apiVersion == V2) && (attributeNotExistingNumber > 0) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) { - std::string details = "one or more of the attributes in the request do not exist: " + eP->id + " - " + attributeNotExistingList; + std::string details = ERROR_DESC_DO_NOT_EXIT + eP->id + "/" + eP->type + " - " + attributeNotExistingList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); - responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + " - " + attributeNotExistingList, ERROR_UNPROCESSABLE); + responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + "/" + eP->type + " - " + attributeNotExistingList, ERROR_UNPROCESSABLE); } if (updateCoverageP != NULL) @@ -4479,7 +4478,7 @@ unsigned int processContextElement // Full failure if the number of the non-existing attributes are all the ones in the request if (attributeNotExistingNumber == eP->attributeVector.size()) { - *updateCoverageP = UC_FULL_FAILURE; + *updateCoverageP = UC_FULL_ATTRS_FAIL; } // Partial failure/success if the number of the non-existing attributes is less than the ones in the request (else branch) // and at least there was one existing attribute @@ -4494,7 +4493,7 @@ unsigned int processContextElement // Full failure if the number of the existing attributes are all the ones in the request if (attributeAlreadyExistsNumber == eP->attributeVector.size()) { - *updateCoverageP = UC_FULL_FAILURE; + *updateCoverageP = UC_FULL_ATTRS_FAIL; } // Partial failure/success if the number of the existing attributes is less than the ones in the request (else branch) // and at least there was one non-existing attribute diff --git a/src/lib/mongoBackend/MongoCommonUpdate.h b/src/lib/mongoBackend/MongoCommonUpdate.h index dc4463a470..3873b7a4f8 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.h +++ b/src/lib/mongoBackend/MongoCommonUpdate.h @@ -43,10 +43,11 @@ */ typedef enum UpdateCoverage { - UC_NONE = 0, - UC_FULL_SUCCESS = 1, - UC_FULL_FAILURE = 2, - UC_PARTIAL = 3 + UC_NONE = 0, + UC_SUCCESS = 1, + UC_FULL_ATTRS_FAIL = 2, + UC_ENTITY_NOT_FOUND = 3, + UC_PARTIAL = 4 } UpdateCoverage; diff --git a/src/lib/mongoBackend/mongoUpdateContext.cpp b/src/lib/mongoBackend/mongoUpdateContext.cpp index 860a7cdf46..80ebfad8b3 100644 --- a/src/lib/mongoBackend/mongoUpdateContext.cpp +++ b/src/lib/mongoBackend/mongoUpdateContext.cpp @@ -172,22 +172,47 @@ HttpStatusCode mongoUpdateContext switch(updateCoverage) { case UC_NONE: + // If global UC is not set yet, then take the UC corresponding to the (first) processed entity updateCoverage = entityUpdateCoverage; break; - case UC_FULL_SUCCESS: - if (entityUpdateCoverage != UC_FULL_SUCCESS) + case UC_SUCCESS: + // If global UC is success, we need also success in the processed entity to keep global success + // Otherwise (full attrs fail, partial, not found entity), the global UC changes to partial + if (entityUpdateCoverage != UC_SUCCESS) { updateCoverage = UC_PARTIAL; } break; - case UC_FULL_FAILURE: - if (entityUpdateCoverage != UC_FULL_FAILURE) + case UC_FULL_ATTRS_FAIL: + // If global UC is full attrs fail, we need also full attrs fail in the processed entity to keep global full attrs fail + // Otherwise (success, partial, not found entity), the global UC changes to partial + if (entityUpdateCoverage != UC_FULL_ATTRS_FAIL) { updateCoverage = UC_PARTIAL; } break; + case UC_ENTITY_NOT_FOUND: + // If global UC is entity not found, we need also entity not found in the processed entity to keep entity not found + // Otherwise, two possibilities: 1) if processed entity is full attrs fail, global UC changes so, or 2) if processed entity is + // success/partial, the global UC changes to partial + if (entityUpdateCoverage == UC_ENTITY_NOT_FOUND) + { + // do nothing (explicity block here for the sake of clearness) + } + else + { + if (entityUpdateCoverage == UC_FULL_ATTRS_FAIL) + { + updateCoverage = UC_FULL_ATTRS_FAIL; + } + else + { + updateCoverage = UC_PARTIAL; + } + } + break; case UC_PARTIAL: - updateCoverage = UC_PARTIAL; + // If global UC is partial, we keep partial (no matter the result of processed entity) break; } } @@ -196,6 +221,7 @@ HttpStatusCode mongoUpdateContext // Other cases follow the usual response processing flow (whatever it is :) if (updateCoverage == UC_PARTIAL) { + responseP->oe.code = SccInvalidModification; responseP->oe.reasonPhrase = ERROR_PARTIAL_UPDATE; } diff --git a/src/lib/rest/OrionError.cpp b/src/lib/rest/OrionError.cpp index eed3f0b7c0..3f178f690e 100644 --- a/src/lib/rest/OrionError.cpp +++ b/src/lib/rest/OrionError.cpp @@ -42,6 +42,7 @@ OrionError::OrionError() code = SccNone; reasonPhrase = ""; details = ""; + filled = false; } @@ -55,6 +56,7 @@ OrionError::OrionError(HttpStatusCode _code, const std::string& _details, const code = _code; reasonPhrase = _reasonPhrase.empty() ? httpStatusCodeString(code) : _reasonPhrase; details = _details; + filled = true; } @@ -68,6 +70,7 @@ OrionError::OrionError(StatusCode& sc) code = sc.code; reasonPhrase = httpStatusCodeString(code); details = sc.details; + filled = true; } @@ -81,6 +84,21 @@ void OrionError::fill(HttpStatusCode _code, const std::string& _details, const s code = _code; reasonPhrase = _reasonPhrase.empty()? httpStatusCodeString(code) : _reasonPhrase; details = _details; + filled = true; +} + + + +/* **************************************************************************** +* +* OrionError::fill - +*/ +void OrionError::fill(const StatusCode& sc) +{ + code = sc.code; + reasonPhrase = (sc.reasonPhrase.empty())? httpStatusCodeString(code) : sc.reasonPhrase; + details = sc.details; + filled = true; } @@ -91,7 +109,7 @@ void OrionError::fill(HttpStatusCode _code, const std::string& _details, const s */ void OrionError::fillOrAppend(HttpStatusCode _code, const std::string& fullDetails, const std::string& appendDetail, const std::string& _reasonPhrase) { - if (code == _code) + if (filled) { // Already filled by a previous operation. This can happen in batch update processing details += appendDetail; @@ -101,24 +119,12 @@ void OrionError::fillOrAppend(HttpStatusCode _code, const std::string& fullDetai code = _code; reasonPhrase = _reasonPhrase.empty()? httpStatusCodeString(code) : _reasonPhrase; details = fullDetails; + filled = true; } } -/* **************************************************************************** -* -* OrionError::fill - -*/ -void OrionError::fill(const StatusCode& sc) -{ - code = sc.code; - reasonPhrase = (sc.reasonPhrase.empty())? httpStatusCodeString(code) : sc.reasonPhrase; - details = sc.details; -} - - - /* **************************************************************************** * * OrionError::smartRender - diff --git a/src/lib/rest/OrionError.h b/src/lib/rest/OrionError.h index d656bac8c9..ab6110c944 100644 --- a/src/lib/rest/OrionError.h +++ b/src/lib/rest/OrionError.h @@ -44,6 +44,7 @@ typedef struct OrionError HttpStatusCode code; std::string reasonPhrase; std::string details; + bool filled; OrionError(); OrionError(StatusCode& statusCode); diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_APPEND.test b/test/functionalTest/cases/1715_batch_update/batch_update_APPEND.test index b9e1c5da97..4cf92b1dfd 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_APPEND.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_APPEND.test @@ -202,10 +202,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 113 +Content-Length: 116 { - "description": "one or more of the attributes in the request already exist: E1 - [ A1 ]", + "description": "one or more of the attributes in the request already exist: E1/T1 - [ A1 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE.test index 73c69b1b76..f18c6bb798 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE.test @@ -132,15 +132,15 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) 04. Delete E1 again, see it fail ================================ -HTTP/1.1 422 Unprocessable Content +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 121 +Content-Length: 74 { - "description": "one or more of the attributes in the request do not exist: E1 - [entity itself]", - "error": "Unprocessable" + "description": "do not exist: E1/T1 - [entity itself]", + "error": "NotFound" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test index 0711ce279d..c08cdd99d1 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing.test @@ -233,10 +233,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 112 +Content-Length: 69 { - "description": "one or more of the attributes in the request do not exist: E2 - [ A2 ]", + "description": "do not exist: E2/T - [ A2 ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test index d6f1638a6f..16a31aa672 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_and_existing_several_entities.test @@ -271,10 +271,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 125 +Content-Length: 84 { - "description": "one or more of the attributes in the request do not exist: E2 - [ A2 ], E4 - [ A1 ]", + "description": "do not exist: E2/T - [ A2 ], E4/T - [ A1 ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test index aa7b436961..e53db2c35c 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_DELETE_non_existing_attr.test @@ -203,10 +203,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 112 +Content-Length: 69 { - "description": "one or more of the attributes in the request do not exist: E2 - [ A2 ]", + "description": "do not exist: E2/T - [ A2 ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_UPDATE.test b/test/functionalTest/cases/1715_batch_update/batch_update_UPDATE.test index 6c5a4b536d..e1f6702472 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_UPDATE.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_UPDATE.test @@ -189,10 +189,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 112 +Content-Length: 70 { - "description": "one or more of the attributes in the request do not exist: E1 - [ A3 ]", + "description": "do not exist: E1/T1 - [ A3 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1715_batch_update/batch_update_error_entity_not_found.test b/test/functionalTest/cases/1715_batch_update/batch_update_error_entity_not_found.test index e2bd1228a7..8efbe70a3f 100644 --- a/test/functionalTest/cases/1715_batch_update/batch_update_error_entity_not_found.test +++ b/test/functionalTest/cases/1715_batch_update/batch_update_error_entity_not_found.test @@ -86,43 +86,43 @@ echo --REGEXPECT-- 01. Batch Update with actionType UPDATE on non-existing entity, see error ========================================================================= -HTTP/1.1 422 Unprocessable Content +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 120 +Content-Length: 72 { - "description": "one or more of the attributes in the request do not exist: E - [entity itself]", - "error": "Unprocessable" + "description": "do not exist: E/T - [entity itself]", + "error": "NotFound" } 02. Batch Update with actionType DELETE on non-existing entity, see error ========================================================================= -HTTP/1.1 422 Unprocessable Content +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 120 +Content-Length: 72 { - "description": "one or more of the attributes in the request do not exist: E - [entity itself]", - "error": "Unprocessable" + "description": "do not exist: E/T - [entity itself]", + "error": "NotFound" } 03. Batch Update with actionType REPLACE on non-existing entity, see error ========================================================================== -HTTP/1.1 422 Unprocessable Content +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 120 +Content-Length: 72 { - "description": "one or more of the attributes in the request do not exist: E - [entity itself]", - "error": "Unprocessable" + "description": "do not exist: E/T - [entity itself]", + "error": "NotFound" } diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test index d72fcb6cc8..1f0891f10b 100644 --- a/test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test +++ b/test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test @@ -321,10 +321,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 130 +Content-Length: 134 { - "description": "one or more of the attributes in the request already exist: E1 - [ A, B ], E2 - [ A, B ]", + "description": "one or more of the attributes in the request already exist: E1/T - [ A, B ], E2/T - [ A, B ]", "error": "Unprocessable" } @@ -335,10 +335,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 115 +Content-Length: 117 { - "description": "one or more of the attributes in the request already exist: E2 - [ A, B ]", + "description": "one or more of the attributes in the request already exist: E2/T - [ A, B ]", "error": "PartialUpdate" } @@ -349,10 +349,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 112 +Content-Length: 114 { - "description": "one or more of the attributes in the request already exist: E1 - [ A ]", + "description": "one or more of the attributes in the request already exist: E1/T - [ A ]", "error": "PartialUpdate" } @@ -363,10 +363,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 127 +Content-Length: 131 { - "description": "one or more of the attributes in the request already exist: E1 - [ A ], E2 - [ A, B ]", + "description": "one or more of the attributes in the request already exist: E1/T - [ A ], E2/T - [ A, B ]", "error": "PartialUpdate" } @@ -377,10 +377,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 124 +Content-Length: 128 { - "description": "one or more of the attributes in the request already exist: E1 - [ A ], E2 - [ A ]", + "description": "one or more of the attributes in the request already exist: E1/T - [ A ], E2/T - [ A ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test b/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test index c6fa5be4bc..87c733b3c2 100644 --- a/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test +++ b/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test @@ -233,8 +233,8 @@ echo echo -echo "03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable" -echo "=======================================================================" +echo "03. Update E3-A/B (full fail) + E4-A/B (full fail): NotFound" +echo "==================================================================" payload='{ "actionType": "delete", "entities": @@ -435,17 +435,17 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable -======================================================================= -HTTP/1.1 422 Unprocessable Content +03. Update E3-A/B (full fail) + E4-A/B (full fail): NotFound +================================================================== +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 141 +Content-Length: 97 { - "description": "one or more of the attributes in the request do not exist: E3 - [entity itself], E4 [entity itself]", - "error": "Unprocessable" + "description": "do not exist: E3/T3 - [entity itself], E4/T3 [entity itself]", + "error": "NotFound" } @@ -455,10 +455,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 121 +Content-Length: 79 { - "description": "one or more of the attributes in the request do not exist: E3 - [entity itself]", + "description": "do not exist: E3/T4 - [entity itself]", "error": "PartialUpdate" } @@ -469,10 +469,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 111 +Content-Length: 69 { - "description": "one or more of the attributes in the request do not exist: E1 - [ C ]", + "description": "do not exist: E1/T5 - [ C ]", "error": "PartialUpdate" } @@ -483,10 +483,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 131 +Content-Length: 92 { - "description": "one or more of the attributes in the request do not exist: E1 - [ C ], E3 [entity itself]", + "description": "do not exist: E1/T6 - [ C ], E3/T6 [entity itself]", "error": "PartialUpdate" } @@ -497,10 +497,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 123 +Content-Length: 84 { - "description": "one or more of the attributes in the request do not exist: E1 - [ C ], E2 - [ C ]", + "description": "do not exist: E1/T7 - [ C ], E2/T7 - [ C ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test b/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test index dff1cdc418..459bf73bc6 100644 --- a/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test +++ b/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test @@ -114,8 +114,8 @@ echo echo -echo "03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable" -echo "=======================================================================" +echo "03. Update E3-A/B (full fail) + E4-A/B (full fail): NotFound" +echo "==================================================================" payload='{ "actionType": "update", "entities": @@ -316,17 +316,17 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -03. Update E3-A/B (full fail) + E4-A/B (full fail): Unprocessable -======================================================================= -HTTP/1.1 422 Unprocessable Content +03. Update E3-A/B (full fail) + E4-A/B (full fail): NotFound +================================================================== +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 141 +Content-Length: 95 { - "description": "one or more of the attributes in the request do not exist: E3 - [entity itself], E4 [entity itself]", - "error": "Unprocessable" + "description": "do not exist: E3/T - [entity itself], E4/T [entity itself]", + "error": "NotFound" } @@ -336,10 +336,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 121 +Content-Length: 78 { - "description": "one or more of the attributes in the request do not exist: E3 - [entity itself]", + "description": "do not exist: E3/T - [entity itself]", "error": "PartialUpdate" } @@ -350,10 +350,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 111 +Content-Length: 68 { - "description": "one or more of the attributes in the request do not exist: E1 - [ C ]", + "description": "do not exist: E1/T - [ C ]", "error": "PartialUpdate" } @@ -364,10 +364,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 131 +Content-Length: 90 { - "description": "one or more of the attributes in the request do not exist: E1 - [ C ], E3 [entity itself]", + "description": "do not exist: E1/T - [ C ], E3/T [entity itself]", "error": "PartialUpdate" } @@ -378,10 +378,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 123 +Content-Length: 82 { - "description": "one or more of the attributes in the request do not exist: E1 - [ C ], E2 - [ C ]", + "description": "do not exist: E1/T - [ C ], E2/T - [ C ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test index ba43328e5e..729203db11 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test @@ -141,10 +141,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 111 +Content-Length: 113 { - "description": "one or more of the attributes in the request already exist: E - [ A ]", + "description": "one or more of the attributes in the request already exist: E/T - [ A ]", "error": "PartialUpdate" } @@ -184,10 +184,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 111 +Content-Length: 113 { - "description": "one or more of the attributes in the request already exist: E - [ B ]", + "description": "one or more of the attributes in the request already exist: E/T - [ B ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test index 0c949c092d..5e0162caa4 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test @@ -225,10 +225,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 145 +Content-Length: 151 { - "description": "one or more of the attributes in the request already exist: E1 - [ A ], E2 - [ A, B ], E3 - [ A, B, C ]", + "description": "one or more of the attributes in the request already exist: E1/T - [ A ], E2/T - [ A, B ], E3/T - [ A, B, C ]", "error": "PartialUpdate" } @@ -308,10 +308,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 136 +Content-Length: 142 { - "description": "one or more of the attributes in the request already exist: E1 - [ B ], E2 - [ B ], E3 - [ B ]", + "description": "one or more of the attributes in the request already exist: E1/T - [ B ], E2/T - [ B ], E3/T - [ B ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test index 8feabbe5be..9916059ffc 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test @@ -145,10 +145,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 113 +Content-Length: 70 { - "description": "one or more of the attributes in the request do not exist: E - [ B, C ]", + "description": "do not exist: E/T - [ B, C ]", "error": "PartialUpdate" } @@ -178,10 +178,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 110 +Content-Length: 67 { - "description": "one or more of the attributes in the request do not exist: E - [ B ]", + "description": "do not exist: E/T - [ B ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test index aec5c87f14..4eaba2cf4f 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test @@ -238,10 +238,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 126 +Content-Length: 85 { - "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 - [ C ]", + "description": "do not exist: E1/T - [ B, C ], E2/T - [ C ]", "error": "PartialUpdate" } @@ -291,10 +291,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 135 +Content-Length: 96 { - "description": "one or more of the attributes in the request do not exist: E1 - [ B ], E2 - [ B ], E3 - [ B ]", + "description": "do not exist: E1/T - [ B ], E2/T - [ B ], E3/T - [ B ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test index 1bd68a7642..8141fb7a12 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test @@ -221,10 +221,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 134 +Content-Length: 93 { - "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 [entity itself]", + "description": "do not exist: E1/T - [ B, C ], E2/T [entity itself]", "error": "PartialUpdate" } @@ -265,10 +265,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 143 +Content-Length: 104 { - "description": "one or more of the attributes in the request do not exist: E1 - [ B ], E2 [entity itself], E3 - [ B ]", + "description": "do not exist: E1/T - [ B ], E2/T [entity itself], E3/T - [ B ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test index 93999321d6..61492c7492 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test @@ -141,10 +141,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 113 +Content-Length: 70 { - "description": "one or more of the attributes in the request do not exist: E - [ B, C ]", + "description": "do not exist: E/T - [ B, C ]", "error": "PartialUpdate" } @@ -174,10 +174,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 110 +Content-Length: 67 { - "description": "one or more of the attributes in the request do not exist: E - [ B ]", + "description": "do not exist: E/T - [ B ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test index 3059d751be..0e8a9e5fef 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test @@ -225,10 +225,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 126 +Content-Length: 85 { - "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 - [ C ]", + "description": "do not exist: E1/T - [ B, C ], E2/T - [ C ]", "error": "PartialUpdate" } @@ -293,10 +293,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 111 +Content-Length: 68 { - "description": "one or more of the attributes in the request do not exist: E1 - [ B ]", + "description": "do not exist: E1/T - [ B ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test index 01904cc414..f22dd0dc35 100644 --- a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test +++ b/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test @@ -213,10 +213,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 134 +Content-Length: 93 { - "description": "one or more of the attributes in the request do not exist: E1 - [ B, C ], E2 [entity itself]", + "description": "do not exist: E1/T - [ B, C ], E2/T [entity itself]", "error": "PartialUpdate" } @@ -267,10 +267,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 131 +Content-Length: 90 { - "description": "one or more of the attributes in the request do not exist: E1 - [ B ], E2 [entity itself]", + "description": "do not exist: E1/T - [ B ], E2/T [entity itself]", "error": "PartialUpdate" } From e41242548b3587ac3f8a233f875bcb7db9e9bb53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 Nov 2023 16:28:19 +0100 Subject: [PATCH 021/390] FIX response error string --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 6af0df9bcd..515b7c0105 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4124,14 +4124,16 @@ unsigned int processContextElement const std::string idString = "_id." ENT_ENTITY_ID; const std::string typeString = "_id." ENT_ENTITY_TYPE; - EntityId en(eP->id, eP->type); - orion::BSONObjBuilder bob; + orion::BSONObjBuilder bob; + EntityId en(eP->id, eP->type); + std::string enStr = eP->id; bob.append(idString, eP->id); if (!eP->type.empty()) { bob.append(typeString, eP->type); + enStr += '/' + eP->type; } // Service path @@ -4338,9 +4340,8 @@ unsigned int processContextElement } else { - //responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); - std::string details = ERROR_DESC_DO_NOT_EXIT + eP->id + "/" + eP->type + " - [entity itself]"; - responseP->oe.fillOrAppend(SccContextElementNotFound, details, ", " + eP->id + "/" + eP->type + " [entity itself]", ERROR_NOT_FOUND); + std::string details = ERROR_DESC_DO_NOT_EXIT + enStr + " - [entity itself]"; + responseP->oe.fillOrAppend(SccContextElementNotFound, details, ", " + enStr + " [entity itself]", ERROR_NOT_FOUND); if (updateCoverageP != NULL) { *updateCoverageP = UC_ENTITY_NOT_FOUND; @@ -4352,9 +4353,8 @@ unsigned int processContextElement { cerP->statusCode.fill(SccContextElementNotFound); - //responseP->oe.fillOrAppend(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ", " + eP->id + " [entity itself]", ERROR_NOT_FOUND); - std::string details = ERROR_DESC_DO_NOT_EXIT + eP->id + "/" + eP->type + " - [entity itself]"; - responseP->oe.fillOrAppend(SccContextElementNotFound, details, ", " + eP->id + "/" + eP->type + " [entity itself]", ERROR_NOT_FOUND); + std::string details = ERROR_DESC_DO_NOT_EXIT + enStr + " - [entity itself]"; + responseP->oe.fillOrAppend(SccContextElementNotFound, details, ", " + enStr + " [entity itself]", ERROR_NOT_FOUND); responseP->contextElementResponseVector.push_back(cerP); if (updateCoverageP != NULL) { @@ -4459,16 +4459,16 @@ unsigned int processContextElement if ((attributeAlreadyExistsNumber > 0) && (action == ActionTypeAppendStrict)) { - std::string details = ERROR_DESC_UNPROCESSABLE_ATTR_ALREADY_EXISTS + eP->id + "/" + eP->type + " - " + attributeAlreadyExistsList; + std::string details = ERROR_DESC_UNPROCESSABLE_ATTR_ALREADY_EXISTS + enStr + " - " + attributeAlreadyExistsList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); - responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + "/" + eP->type + " - " + attributeAlreadyExistsList, ERROR_UNPROCESSABLE); + responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + enStr + " - " + attributeAlreadyExistsList, ERROR_UNPROCESSABLE); } if ((apiVersion == V2) && (attributeNotExistingNumber > 0) && ((action == ActionTypeUpdate) || (action == ActionTypeDelete))) { - std::string details = ERROR_DESC_DO_NOT_EXIT + eP->id + "/" + eP->type + " - " + attributeNotExistingList; + std::string details = ERROR_DESC_DO_NOT_EXIT + enStr + " - " + attributeNotExistingList; buildGeneralErrorResponse(eP, NULL, responseP, SccBadRequest, details); - responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + eP->id + "/" + eP->type + " - " + attributeNotExistingList, ERROR_UNPROCESSABLE); + responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + enStr + " - " + attributeNotExistingList, ERROR_UNPROCESSABLE); } if (updateCoverageP != NULL) From c5e74e8818a81a6a552a258fcb6f60adeccf0071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 Nov 2023 17:45:44 +0100 Subject: [PATCH 022/390] FIX single entity error description in responses --- src/lib/serviceRoutinesV2/patchEntity.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib/serviceRoutinesV2/patchEntity.cpp b/src/lib/serviceRoutinesV2/patchEntity.cpp index 8c505d4bc0..c31c5749b5 100644 --- a/src/lib/serviceRoutinesV2/patchEntity.cpp +++ b/src/lib/serviceRoutinesV2/patchEntity.cpp @@ -85,6 +85,12 @@ std::string patchEntity // 02. Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); + // Adjust NotFound description (to avoid redundant missing entity information) + if (parseDataP->upcrs.res.oe.code == SccContextElementNotFound) + { + parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; + } + // 03. Check output from mongoBackend - any errors? if (parseDataP->upcrs.res.oe.code != SccNone ) { From 32bec9f650a394ebee021ff6140a20601474c999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 13:16:20 +0100 Subject: [PATCH 023/390] FIX error reponses for the NotFound case with CPrs in place --- src/lib/serviceRoutinesV2/patchEntity.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/serviceRoutinesV2/patchEntity.cpp b/src/lib/serviceRoutinesV2/patchEntity.cpp index c31c5749b5..2d30a9acaa 100644 --- a/src/lib/serviceRoutinesV2/patchEntity.cpp +++ b/src/lib/serviceRoutinesV2/patchEntity.cpp @@ -86,7 +86,7 @@ std::string patchEntity postUpdateContext(ciP, components, compV, parseDataP); // Adjust NotFound description (to avoid redundant missing entity information) - if (parseDataP->upcrs.res.oe.code == SccContextElementNotFound) + if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) { parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; } From d6e484d635ab7ed4ab3b49053805bba5d1164d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 13:34:47 +0100 Subject: [PATCH 024/390] FIX single error description responses --- src/lib/serviceRoutinesV2/putEntityAttribute.cpp | 6 ++++++ .../invalid_error_updating_non_existing_attribute.test | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp index 59ca2e8d4e..b3ef79c042 100644 --- a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp +++ b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp @@ -84,6 +84,12 @@ std::string putEntityAttribute // 02. Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); + // Adjust NotFound description (to avoid redundant missing entity information) + if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) + { + parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; + } + // 03. Check error std::string answer = ""; if (parseDataP->upcrs.res.oe.code != SccNone ) diff --git a/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test b/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test index 0da279cbfb..2592785cec 100644 --- a/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test +++ b/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test @@ -83,10 +83,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 112 +Content-Length: 67 { - "description": "one or more of the attributes in the request do not exist: E1 - [ A2 ]", + "description": "do not exist: E1 - [ A2 ]", "error": "Unprocessable" } From 34b23149b4e0c64bcda9ed718f58eb3cd68fb69c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 15:21:51 +0100 Subject: [PATCH 025/390] FIX old ftest --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 5 ++++- src/lib/serviceRoutinesV2/deleteEntity.cpp | 7 +++++++ src/lib/serviceRoutinesV2/patchEntity.cpp | 1 + src/lib/serviceRoutinesV2/putEntity.cpp | 7 +++++++ src/lib/serviceRoutinesV2/putEntityAttribute.cpp | 1 + src/lib/serviceRoutinesV2/putEntityAttributeValue.cpp | 8 ++++++++ .../0979_patch_v2_entities_eid/patch_v2_entities_eid.test | 4 ++-- .../patch_entity_with_param_type.test | 4 ++-- .../982_post_entity_with_type_param.test | 4 ++-- .../0993_DELETE_v2_entity_attr/delete_not_exist_attr.test | 4 ++-- .../cases/1127_v1_append_strict/v1_append_strict.test | 8 ++++---- .../patch_unknown_attribute.test | 4 ++-- .../error_response_for_entity_not_found.test | 4 ++-- .../patch_non_existing_attribute.test | 4 ++-- .../PATCH_offending_attributes.test | 8 ++++---- 15 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 515b7c0105..6fba6a2fb9 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4112,7 +4112,10 @@ unsigned int processContextElement ) { // By default, assume everything is gone to be ok. Next in this function we are going to check for some partial/failure cases - *updateCoverageP = UC_SUCCESS; + if (updateCoverageP != NULL) + { + *updateCoverageP = UC_SUCCESS; + } /* Check preconditions */ if (!contextElementPreconditionsCheck(eP, responseP, action, apiVersion)) diff --git a/src/lib/serviceRoutinesV2/deleteEntity.cpp b/src/lib/serviceRoutinesV2/deleteEntity.cpp index 45015a83d4..0330694791 100644 --- a/src/lib/serviceRoutinesV2/deleteEntity.cpp +++ b/src/lib/serviceRoutinesV2/deleteEntity.cpp @@ -90,6 +90,13 @@ std::string deleteEntity // Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); + // Adjust NotFound description (to avoid redundant missing entity information) + // FIXME PR: duplicated code in several places + if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) + { + parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; + } + ciP->outMimeType = JSON; // Check for potential error diff --git a/src/lib/serviceRoutinesV2/patchEntity.cpp b/src/lib/serviceRoutinesV2/patchEntity.cpp index 2d30a9acaa..dbc750930f 100644 --- a/src/lib/serviceRoutinesV2/patchEntity.cpp +++ b/src/lib/serviceRoutinesV2/patchEntity.cpp @@ -86,6 +86,7 @@ std::string patchEntity postUpdateContext(ciP, components, compV, parseDataP); // Adjust NotFound description (to avoid redundant missing entity information) + // FIXME PR: duplicated code in several places if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) { parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; diff --git a/src/lib/serviceRoutinesV2/putEntity.cpp b/src/lib/serviceRoutinesV2/putEntity.cpp index 5df120b028..d44b23190a 100644 --- a/src/lib/serviceRoutinesV2/putEntity.cpp +++ b/src/lib/serviceRoutinesV2/putEntity.cpp @@ -84,6 +84,13 @@ std::string putEntity // 02. Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); + // Adjust NotFound description (to avoid redundant missing entity information) + // FIXME PR: duplicated code in several places + if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) + { + parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; + } + // 03. Check error std::string answer = ""; if (parseDataP->upcrs.res.oe.code != SccNone ) diff --git a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp index b3ef79c042..53204127ac 100644 --- a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp +++ b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp @@ -85,6 +85,7 @@ std::string putEntityAttribute postUpdateContext(ciP, components, compV, parseDataP); // Adjust NotFound description (to avoid redundant missing entity information) + // FIXME PR: duplicated code in several places if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) { parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; diff --git a/src/lib/serviceRoutinesV2/putEntityAttributeValue.cpp b/src/lib/serviceRoutinesV2/putEntityAttributeValue.cpp index ce14f00684..c4e8ea8f18 100644 --- a/src/lib/serviceRoutinesV2/putEntityAttributeValue.cpp +++ b/src/lib/serviceRoutinesV2/putEntityAttributeValue.cpp @@ -91,6 +91,14 @@ std::string putEntityAttributeValue // 02. Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); + // Adjust NotFound description (to avoid redundant missing entity information) + // FIXME PR: duplicated code in several places + if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) + { + parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; + } + + // 03. Check output from mongoBackend std::string answer = ""; if (parseDataP->upcrs.res.oe.code != SccNone) diff --git a/test/functionalTest/cases/0979_patch_v2_entities_eid/patch_v2_entities_eid.test b/test/functionalTest/cases/0979_patch_v2_entities_eid/patch_v2_entities_eid.test index 5848be90e5..2c5540c1ad 100644 --- a/test/functionalTest/cases/0979_patch_v2_entities_eid/patch_v2_entities_eid.test +++ b/test/functionalTest/cases/0979_patch_v2_entities_eid/patch_v2_entities_eid.test @@ -171,10 +171,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 115 +Content-Length: 70 { - "description": "one or more of the attributes in the request do not exist: E1 - [ attr3 ]", + "description": "do not exist: E1 - [ attr3 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/0980_patch_entity_v2_wihth_param_type/patch_entity_with_param_type.test b/test/functionalTest/cases/0980_patch_entity_v2_wihth_param_type/patch_entity_with_param_type.test index 41c90e2615..2d2c87bbbe 100644 --- a/test/functionalTest/cases/0980_patch_entity_v2_wihth_param_type/patch_entity_with_param_type.test +++ b/test/functionalTest/cases/0980_patch_entity_v2_wihth_param_type/patch_entity_with_param_type.test @@ -169,10 +169,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 112 +Content-Length: 70 { - "description": "one or more of the attributes in the request do not exist: E1 - [ A2 ]", + "description": "do not exist: E1/T1 - [ A2 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/0982_post_entity_with_type_param/982_post_entity_with_type_param.test b/test/functionalTest/cases/0982_post_entity_with_type_param/982_post_entity_with_type_param.test index 3208886ffb..943a3c5436 100644 --- a/test/functionalTest/cases/0982_post_entity_with_type_param/982_post_entity_with_type_param.test +++ b/test/functionalTest/cases/0982_post_entity_with_type_param/982_post_entity_with_type_param.test @@ -238,10 +238,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 113 +Content-Length: 116 { - "description": "one or more of the attributes in the request already exist: E1 - [ A3 ]", + "description": "one or more of the attributes in the request already exist: E1/T2 - [ A3 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test b/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test index ae337a1564..1f6638006d 100644 --- a/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test +++ b/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test @@ -106,10 +106,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 114 +Content-Length: 69 { - "description": "one or more of the attributes in the request do not exist: E1 - [ none ]", + "description": "do not exist: E1 - [ none ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test b/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test index 6588f5b23e..746ea2c18c 100644 --- a/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test +++ b/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test @@ -239,7 +239,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 229 +Content-Length: 232 { "contextResponses": [ @@ -251,7 +251,7 @@ Content-Length: 229 }, "statusCode": { "code": "400", - "details": "one or more of the attributes in the request already exist: E1 - [ attr2 ]", + "details": "one or more of the attributes in the request already exist: E1/T1 - [ attr2 ]", "reasonPhrase": "Bad Request" } } @@ -289,7 +289,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 393 +Content-Length: 396 { "contextResponses": [ @@ -319,7 +319,7 @@ Content-Length: 393 }, "statusCode": { "code": "400", - "details": "one or more of the attributes in the request already exist: E1 - [ attr2 ]", + "details": "one or more of the attributes in the request already exist: E1/T1 - [ attr2 ]", "reasonPhrase": "Bad Request" } } diff --git a/test/functionalTest/cases/1278_patch_unknown_attribute/patch_unknown_attribute.test b/test/functionalTest/cases/1278_patch_unknown_attribute/patch_unknown_attribute.test index 555d8cefda..5a91ddffa1 100644 --- a/test/functionalTest/cases/1278_patch_unknown_attribute/patch_unknown_attribute.test +++ b/test/functionalTest/cases/1278_patch_unknown_attribute/patch_unknown_attribute.test @@ -75,10 +75,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 112 +Content-Length: 67 { - "description": "one or more of the attributes in the request do not exist: E1 - [ A2 ]", + "description": "do not exist: E1 - [ A2 ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test b/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test index 11cca7a827..70b67f7e68 100644 --- a/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test +++ b/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test @@ -97,10 +97,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 122 +Content-Length: 77 { - "description": "one or more of the attributes in the request do not exist: room_2 - [ dfgdfgdf ]", + "description": "do not exist: room_2 - [ dfgdfgdf ]", "error": "Unprocessable" } diff --git a/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test b/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test index 7c84e5ef49..d57bda6493 100644 --- a/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test +++ b/test/functionalTest/cases/1784_patch_non_existing_attribute/patch_non_existing_attribute.test @@ -89,10 +89,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 112 +Content-Length: 67 { - "description": "one or more of the attributes in the request do not exist: E1 - [ A2 ]", + "description": "do not exist: E1 - [ A2 ]", "error": "PartialUpdate" } diff --git a/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test b/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test index 0ccf23b652..a69fc89d16 100644 --- a/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test +++ b/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test @@ -117,10 +117,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 120 +Content-Length: 75 { - "description": "one or more of the attributes in the request do not exist: Room1 - [ pressur ]", + "description": "do not exist: Room1 - [ pressur ]", "error": "PartialUpdate" } @@ -131,10 +131,10 @@ HTTP/1.1 422 Unprocessable Content Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 128 +Content-Length: 83 { - "description": "one or more of the attributes in the request do not exist: Room1 - [ temper, pressur ]", + "description": "do not exist: Room1 - [ temper, pressur ]", "error": "PartialUpdate" } From 8163e116a07492350c2bbdc22d8b5ee24737a3a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 16:42:09 +0100 Subject: [PATCH 026/390] FIX rename ftest --- ...appendstrict.test => batch_all_combinations_appendstrict.test} | 0 ...ombinations_delete.test => batch_all_combinations_delete.test} | 0 ...ombinations_update.test => batch_all_combinations_update.test} | 0 ...dstrict.test => batch_partial_update_action_appendstrict.test} | 0 ...atch_partial_update_action_appendstrict_several_entities.test} | 0 ...action_delete.test => batch_partial_update_action_delete.test} | 0 ...t => batch_partial_update_action_delete_several_entities.test} | 0 ...rtial_update_action_delete_several_entities_non_existing.test} | 0 ...action_update.test => batch_partial_update_action_update.test} | 0 ...t => batch_partial_update_action_update_several_entities.test} | 0 ...rtial_update_action_update_several_entities_non_existing.test} | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename test/functionalTest/cases/3499_partial_update_response/{all_combinations_appendstrict.test => batch_all_combinations_appendstrict.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{all_combinations_delete.test => batch_all_combinations_delete.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{all_combinations_update.test => batch_all_combinations_update.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{partial_update_action_appendstrict.test => batch_partial_update_action_appendstrict.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{partial_update_action_appendstrict_several_entities.test => batch_partial_update_action_appendstrict_several_entities.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{partial_update_action_delete.test => batch_partial_update_action_delete.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{partial_update_action_delete_several_entities.test => batch_partial_update_action_delete_several_entities.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{partial_update_action_delete_several_entities_non_existing.test => batch_partial_update_action_delete_several_entities_non_existing.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{partial_update_action_update.test => batch_partial_update_action_update.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{partial_update_action_update_several_entities.test => batch_partial_update_action_update_several_entities.test} (100%) rename test/functionalTest/cases/3499_partial_update_response/{partial_update_action_update_several_entities_non_existing.test => batch_partial_update_action_update_several_entities_non_existing.test} (100%) diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_appendstrict.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/all_combinations_appendstrict.test rename to test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_appendstrict.test diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test b/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_delete.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/all_combinations_delete.test rename to test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_delete.test diff --git a/test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test b/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_update.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/all_combinations_update.test rename to test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_update.test diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict.test rename to test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict.test diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict_several_entities.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/partial_update_action_appendstrict_several_entities.test rename to test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict_several_entities.test diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete.test rename to test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete.test diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities.test rename to test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities.test diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities_non_existing.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/partial_update_action_delete_several_entities_non_existing.test rename to test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities_non_existing.test diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/partial_update_action_update.test rename to test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update.test diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities.test rename to test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities.test diff --git a/test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities_non_existing.test similarity index 100% rename from test/functionalTest/cases/3499_partial_update_response/partial_update_action_update_several_entities_non_existing.test rename to test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities_non_existing.test From 3def719df46f4eb1be49e08910c9a8212b76bd7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 17:18:25 +0100 Subject: [PATCH 027/390] ADD test caes for single update cases --- .../batch_all_combinations_appendstrict.test | 2 +- .../batch_all_combinations_delete.test | 2 +- .../batch_all_combinations_update.test | 2 +- ...ch_partial_update_action_appendstrict.test | 2 +- ..._action_appendstrict_several_entities.test | 2 +- .../batch_partial_update_action_delete.test | 2 +- ...update_action_delete_several_entities.test | 2 +- ..._delete_several_entities_non_existing.test | 2 +- .../batch_partial_update_action_update.test | 2 +- ...update_action_update_several_entities.test | 2 +- ..._update_several_entities_non_existing.test | 2 +- .../single_all_combinations_appendstrict.test | 200 ++++++++++++++++++ .../single_all_combinations_delete.test | 116 ++++++++++ .../single_all_combinations_update.test | 200 ++++++++++++++++++ 14 files changed, 527 insertions(+), 11 deletions(-) create mode 100644 test/functionalTest/cases/3499_partial_update_response/single_all_combinations_appendstrict.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/single_all_combinations_delete.test create mode 100644 test/functionalTest/cases/3499_partial_update_response/single_all_combinations_update.test diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_appendstrict.test index 1f0891f10b..f8b2e1c410 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_appendstrict.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_appendstrict.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -All fail/success/partial combinations for appendStrict actionType +All fail/success/partial combinations for appendStrict actionType in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_delete.test b/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_delete.test index 87c733b3c2..4f807d90df 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_delete.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_delete.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -All fail/success/partial combinations for delete actionType +All fail/success/partial combinations for delete actionType in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_update.test b/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_update.test index 459bf73bc6..1f95ea79e5 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_update.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_all_combinations_update.test @@ -22,7 +22,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -All fail/success/partial combinations for update actionType +All fail/success/partial combinations for update actionType in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict.test index 729203db11..6c15add5d7 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -PartialUpdate response for appendStrict case +PartialUpdate response for appendStrict case in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict_several_entities.test index 5e0162caa4..5dc96ef8f6 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_appendstrict_several_entities.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -PartialUpdate response for appendStrict case for several entities +PartialUpdate response for appendStrict case for several entities in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete.test index 9916059ffc..c7add1534d 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -PartialUpdate response for delete case +PartialUpdate response for delete case in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities.test index 4eaba2cf4f..4a8e4536c9 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -PartialUpdate response for delete case for several entities +PartialUpdate response for delete case for several entities in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities_non_existing.test index 8141fb7a12..1728cf0292 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities_non_existing.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_delete_several_entities_non_existing.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -PartialUpdate response for delete case for several entities one of them non existing +PartialUpdate response for delete case for several entities one of them non existing in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update.test index 61492c7492..2ffe7c4fe8 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -PartialUpdate response for update case +PartialUpdate response for update case in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities.test index 0e8a9e5fef..4543a819fb 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -PartialUpdate response for update case for several entities +PartialUpdate response for update case for several entities in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities_non_existing.test b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities_non_existing.test index f22dd0dc35..2073cc7391 100644 --- a/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities_non_existing.test +++ b/test/functionalTest/cases/3499_partial_update_response/batch_partial_update_action_update_several_entities_non_existing.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -PartialUpdate response for update case for several entities one of them non existing +PartialUpdate response for update case for several entities one of them non existing in batch operation --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_appendstrict.test b/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_appendstrict.test new file mode 100644 index 0000000000..ddeb1e4024 --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_appendstrict.test @@ -0,0 +1,200 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +All fail/success/partial combinations for append strict in single update operation + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create E-A/B +# 02. POST /v2/entities/E/attrs?options=append A/B (full fail) +# 03. POST /v2/entities/E/attrs?options=append B/C (partial fail) +# 04. POST /v2/entities/E/attrs?type=T&options=append A/B (full fail) +# 05. POST /v2/entities/E/attrs?type=T&options=append B/D (partial fail) +# + +echo "01. Create E-A/B" +echo "================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. POST /v2/entities/E/attrs?options=append A/B (full fail)" +echo "============================================================" +payload='{ + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url '/v2/entities/E/attrs?options=append' --payload "$payload" +echo +echo + + +echo "03. POST /v2/entities/E/attrs?options=append B/C (partial fail)" +echo "===============================================================" +payload='{ + "B": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url '/v2/entities/E/attrs?options=append' --payload "$payload" +echo +echo + + +echo "04. POST /v2/entities/E/attrs?type=T&options=append A/B (full fail)" +echo "===================================================================" +payload='{ + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url '/v2/entities/E/attrs?type=T&options=append' --payload "$payload" +echo +echo + + +echo "05. POST /v2/entities/E/attrs?type=T&options=append B/D (partial fail)" +echo "======================================================================" +payload='{ + "B": { + "value": 1, + "type": "Number" + }, + "D": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url '/v2/entities/E/attrs?type=T&options=append' --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. Create E-A/B +================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. POST /v2/entities/E/attrs?options=append A/B (full fail) +============================================================ +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 114 + +{ + "description": "one or more of the attributes in the request already exist: E - [ A, B ]", + "error": "Unprocessable" +} + + +03. POST /v2/entities/E/attrs?options=append B/C (partial fail) +=============================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 111 + +{ + "description": "one or more of the attributes in the request already exist: E - [ B ]", + "error": "PartialUpdate" +} + + +04. POST /v2/entities/E/attrs?type=T&options=append A/B (full fail) +=================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 116 + +{ + "description": "one or more of the attributes in the request already exist: E/T - [ A, B ]", + "error": "Unprocessable" +} + + +05. POST /v2/entities/E/attrs?type=T&options=append B/D (partial fail) +====================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 113 + +{ + "description": "one or more of the attributes in the request already exist: E/T - [ B ]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_delete.test b/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_delete.test new file mode 100644 index 0000000000..edd9ab999c --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_delete.test @@ -0,0 +1,116 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +All fail/success/partial combinations for update in single update operation + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create E-A/B +# 02. DELETE /v2/entities/E/attrs/C (full fail) +# 03. DELETE /v2/entities/E/attrs/C?type=T (full fail) +# + +echo "01. Create E-A/B" +echo "================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. DELETE /v2/entities/E/attrs/C (full fail)" +echo "=============================================" +orionCurl --url '/v2/entities/E/attrs/C' -X DELETE +echo +echo + + +echo "03. DELETE /v2/entities/E/attrs/C?type=T (full fail)" +echo "====================================================" +orionCurl --url '/v2/entities/E/attrs/C?type=T' -X DELETE +echo +echo + + +--REGEXPECT-- +01. Create E-A/B +================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. DELETE /v2/entities/E/attrs/C (full fail) +============================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 65 + +{ + "description": "do not exist: E - [ C ]", + "error": "Unprocessable" +} + + +03. DELETE /v2/entities/E/attrs/C?type=T (full fail) +==================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 67 + +{ + "description": "do not exist: E/T - [ C ]", + "error": "Unprocessable" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_update.test b/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_update.test new file mode 100644 index 0000000000..00ee708439 --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_update.test @@ -0,0 +1,200 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +All fail/success/partial combinations for update in single update operation + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create E-A/B +# 02. PATCH /v2/entities/E/attrs C/D (full fail) +# 03. PATCH /v2/entities/E/attrs A/C (partial fail) +# 04. PATCH /v2/entities/E/attrs?type=T C/D (full fail) +# 05. PATCH /v2/entities/E/attrs?type=T A/C (partial fail) +# + +echo "01. Create E-A/B" +echo "================" +payload='{ + "actionType": "appendStrict", + "entities": + [ + { + "type": "T", + "id": "E", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. PATCH /v2/entities/E/attrs C/D (full fail)" +echo "==============================================" +payload='{ + "C": { + "value": 1, + "type": "Number" + }, + "D": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url '/v2/entities/E/attrs' --payload "$payload" -X PATCH +echo +echo + + +echo "03. PATCH /v2/entities/E/attrs A/C (partial fail)" +echo "=================================================" +payload='{ + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url '/v2/entities/E/attrs' --payload "$payload" -X PATCH +echo +echo + + +echo "04. PATCH /v2/entities/E/attrs?type=T C/D (full fail)" +echo "=====================================================" +payload='{ + "C": { + "value": 1, + "type": "Number" + }, + "D": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url '/v2/entities/E/attrs?type=T' --payload "$payload" -X PATCH +echo +echo + + +echo "05. PATCH /v2/entities/E/attrs?type=T A/C (partial fail)" +echo "========================================================" +payload='{ + "A": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url '/v2/entities/E/attrs?type=T' --payload "$payload" -X PATCH +echo +echo + + +--REGEXPECT-- +01. Create E-A/B +================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. PATCH /v2/entities/E/attrs C/D (full fail) +============================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "do not exist: E - [ C, D ]", + "error": "Unprocessable" +} + + +03. PATCH /v2/entities/E/attrs A/C (partial fail) +================================================= +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 65 + +{ + "description": "do not exist: E - [ C ]", + "error": "PartialUpdate" +} + + +04. PATCH /v2/entities/E/attrs?type=T C/D (full fail) +===================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 70 + +{ + "description": "do not exist: E/T - [ C, D ]", + "error": "Unprocessable" +} + + +05. PATCH /v2/entities/E/attrs?type=T A/C (partial fail) +======================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 67 + +{ + "description": "do not exist: E/T - [ C ]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file From 3ac1bfc1567cbc373512f68158ad7ab63c666b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 17:39:55 +0100 Subject: [PATCH 028/390] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index ba5b1b9d9b..4038fd3329 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -3,7 +3,8 @@ - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) -- Fix: on delete attributes operation the lack of attribute in entity doesn't precules attributes which do exist (this way behaving same way as update attributes operation) +- Fix: improve error traces (#4387) +- Fix: on delete attributes operation the lack of attribute in entity doesn't precule attributes which do exist (this way behaving same way as update attributes operation) - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) - Fix: improve logs in MongoDB query logic - Upgrade Debian version from 11.6 to 12.1 in Dockerfile From 48c5ca456e58949d25588841d7b5f19d625681b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 30 Nov 2023 17:21:08 +0100 Subject: [PATCH 029/390] FIX adjust response codes --- CHANGES_NEXT_RELEASE | 2 +- src/lib/serviceRoutinesV2/deleteEntity.cpp | 11 ++++----- src/lib/serviceRoutinesV2/patchEntity.cpp | 9 +++---- .../serviceRoutinesV2/putEntityAttribute.cpp | 9 +++---- .../serviceRoutinesCommon.cpp | 24 +++++++++++++++++++ .../serviceRoutinesV2/serviceRoutinesCommon.h | 10 ++++++++ .../delete_not_exist_attr.test | 8 +++---- .../single_all_combinations_delete.test | 16 ++++++------- 8 files changed, 57 insertions(+), 32 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 4038fd3329..7f815e5662 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -4,7 +4,7 @@ - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) - Fix: improve error traces (#4387) -- Fix: on delete attributes operation the lack of attribute in entity doesn't precule attributes which do exist (this way behaving same way as update attributes operation) +- Fix: on delete attributes operation, the lack of attribute in entity doesn't preclude the deletion of attributes which do exist (this way behaving same way as update attributes operation) - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) - Fix: improve logs in MongoDB query logic - Upgrade Debian version from 11.6 to 12.1 in Dockerfile diff --git a/src/lib/serviceRoutinesV2/deleteEntity.cpp b/src/lib/serviceRoutinesV2/deleteEntity.cpp index 0330694791..f70eaa226c 100644 --- a/src/lib/serviceRoutinesV2/deleteEntity.cpp +++ b/src/lib/serviceRoutinesV2/deleteEntity.cpp @@ -35,6 +35,7 @@ #include "rest/OrionError.h" #include "rest/EntityTypeInfo.h" #include "serviceRoutinesV2/deleteEntity.h" +#include "serviceRoutinesV2/serviceRoutinesCommon.h" #include "serviceRoutines/postUpdateContext.h" #include "parse/forbiddenChars.h" @@ -44,7 +45,7 @@ * * deleteEntity - * -* DELETE /v2/entities/{entityId} +* DELETE /v2/entities/{entityId}[/attrs/{attrName}] * * Payload In: None * Payload Out: None @@ -90,12 +91,8 @@ std::string deleteEntity // Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); - // Adjust NotFound description (to avoid redundant missing entity information) - // FIXME PR: duplicated code in several places - if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) - { - parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; - } + // Adjust error code if needed + adaptErrorCodeForSingleEntityOperation(&(parseDataP->upcrs.res.oe), true); ciP->outMimeType = JSON; diff --git a/src/lib/serviceRoutinesV2/patchEntity.cpp b/src/lib/serviceRoutinesV2/patchEntity.cpp index dbc750930f..ad767b632c 100644 --- a/src/lib/serviceRoutinesV2/patchEntity.cpp +++ b/src/lib/serviceRoutinesV2/patchEntity.cpp @@ -34,6 +34,7 @@ #include "apiTypesV2/Entities.h" #include "rest/EntityTypeInfo.h" #include "serviceRoutinesV2/patchEntity.h" +#include "serviceRoutinesV2/serviceRoutinesCommon.h" #include "serviceRoutines/postUpdateContext.h" #include "rest/OrionError.h" @@ -85,12 +86,8 @@ std::string patchEntity // 02. Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); - // Adjust NotFound description (to avoid redundant missing entity information) - // FIXME PR: duplicated code in several places - if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) - { - parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; - } + // Adjust error code if needed + adaptErrorCodeForSingleEntityOperation(&(parseDataP->upcrs.res.oe), false); // 03. Check output from mongoBackend - any errors? if (parseDataP->upcrs.res.oe.code != SccNone ) diff --git a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp index 53204127ac..83d9af39ca 100644 --- a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp +++ b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp @@ -34,6 +34,7 @@ #include "rest/EntityTypeInfo.h" #include "serviceRoutines/postUpdateContext.h" #include "serviceRoutinesV2/putEntityAttribute.h" +#include "serviceRoutinesV2/serviceRoutinesCommon.h" #include "rest/OrionError.h" #include "parse/forbiddenChars.h" #include "alarmMgr/alarmMgr.h" @@ -84,12 +85,8 @@ std::string putEntityAttribute // 02. Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); - // Adjust NotFound description (to avoid redundant missing entity information) - // FIXME PR: duplicated code in several places - if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) - { - parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; - } + // Adjust error code if needed + adaptErrorCodeForSingleEntityOperation(&(parseDataP->upcrs.res.oe), false); // 03. Check error std::string answer = ""; diff --git a/src/lib/serviceRoutinesV2/serviceRoutinesCommon.cpp b/src/lib/serviceRoutinesV2/serviceRoutinesCommon.cpp index 1acc888ee4..083072e0cf 100644 --- a/src/lib/serviceRoutinesV2/serviceRoutinesCommon.cpp +++ b/src/lib/serviceRoutinesV2/serviceRoutinesCommon.cpp @@ -32,6 +32,7 @@ #include "common/string.h" #include "common/RenderFormat.h" +#include "common/errorMessages.h" #include "ngsi/StringList.h" #include "rest/uriParamNames.h" @@ -124,3 +125,26 @@ RenderFormat getRenderFormat(std::map& uriParamOptions) return renderFormat; } + + + +/* **************************************************************************** +* +* adaptErrorCodeForSingleEntityOperation - +* +*/ +void adaptErrorCodeForSingleEntityOperation(OrionError* oeP, bool singleAttributeCheck) +{ + if ((oeP->code == SccContextElementNotFound) & (oeP->reasonPhrase == ERROR_NOT_FOUND)) + { + // In single entity attribute operations (e.g. DELETE /v2/entities/E/attrs/A) when the entity doesn't exist + oeP->details = ERROR_DESC_NOT_FOUND_ENTITY; + } + else if (singleAttributeCheck && (oeP->code == SccInvalidModification) & (oeP->reasonPhrase == ERROR_UNPROCESSABLE)) + { + // In single entity attribute operations (e.g. DELETE /v2/entities/E/attrs/A) when the entity exists but the attribute doesn't + oeP->code = SccContextElementNotFound; + oeP->reasonPhrase = ERROR_NOT_FOUND; + oeP->details = ERROR_DESC_NOT_FOUND_ATTRIBUTE; + } +} \ No newline at end of file diff --git a/src/lib/serviceRoutinesV2/serviceRoutinesCommon.h b/src/lib/serviceRoutinesV2/serviceRoutinesCommon.h index 10e8ee89dd..82dc209738 100644 --- a/src/lib/serviceRoutinesV2/serviceRoutinesCommon.h +++ b/src/lib/serviceRoutinesV2/serviceRoutinesCommon.h @@ -34,6 +34,7 @@ #include "common/string.h" #include "common/RenderFormat.h" #include "ngsi/StringList.h" +#include "rest/OrionError.h" #include "rest/uriParamNames.h" @@ -72,5 +73,14 @@ extern void setMetadataFilter */ extern RenderFormat getRenderFormat(std::map& uriParamOptions); + + +/* **************************************************************************** +* +* adaptErrorCodeForSingleEntityOperation - +* +*/ +extern void adaptErrorCodeForSingleEntityOperation(OrionError* oeP, bool singleAttributeCheck); + #endif // SRC_LIB_SERVICEROUTINESV2_SERVICEROUTINESCOMMON_H_ diff --git a/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test b/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test index 1f6638006d..405123b429 100644 --- a/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test +++ b/test/functionalTest/cases/0993_DELETE_v2_entity_attr/delete_not_exist_attr.test @@ -102,15 +102,15 @@ Content-Length: 121 03. DELETE /v2/entities/E1/attrs/none ===================================== -HTTP/1.1 422 Unprocessable Content +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 69 +Content-Length: 79 { - "description": "do not exist: E1 - [ none ]", - "error": "Unprocessable" + "description": "The entity does not have such an attribute", + "error": "NotFound" } diff --git a/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_delete.test b/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_delete.test index edd9ab999c..1c8eb4281a 100644 --- a/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_delete.test +++ b/test/functionalTest/cases/3499_partial_update_response/single_all_combinations_delete.test @@ -85,29 +85,29 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) 02. DELETE /v2/entities/E/attrs/C (full fail) ============================================= -HTTP/1.1 422 Unprocessable Content +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 65 +Content-Length: 79 { - "description": "do not exist: E - [ C ]", - "error": "Unprocessable" + "description": "The entity does not have such an attribute", + "error": "NotFound" } 03. DELETE /v2/entities/E/attrs/C?type=T (full fail) ==================================================== -HTTP/1.1 422 Unprocessable Content +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 67 +Content-Length: 79 { - "description": "do not exist: E/T - [ C ]", - "error": "Unprocessable" + "description": "The entity does not have such an attribute", + "error": "NotFound" } From 3a3d7a5671bd4cde233b0e184480a5c8ad82eab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 13 Dec 2023 09:58:53 +0100 Subject: [PATCH 030/390] ADD ftest for error responses on updates and doc and code changes to align --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/orion-api.md | 239 ++++++++++++++++-- src/lib/common/globals.h | 2 +- src/lib/mongoBackend/MongoCommonUpdate.cpp | 3 +- src/lib/serviceRoutinesV2/postEntity.cpp | 5 +- src/lib/serviceRoutinesV2/putEntity.cpp | 11 +- .../serviceRoutinesV2/putEntityAttribute.cpp | 2 +- .../putEntityAttributeValue.cpp | 10 +- ...s_00_get_entity_and_entity_attributes.test | 101 ++++++++ ...01_update_or_append_entity_attributes.test | 221 ++++++++++++++++ ..._02_update_existing_entity_attributes.test | 193 ++++++++++++++ ...r_responses_03_replace_all_attributes.test | 86 +++++++ .../error_responses_04_delete_entity.test | 80 ++++++ .../error_responses_05_update_attribute.test | 146 +++++++++++ .../error_responses_06_delete_attribute.test | 138 ++++++++++ ...rror_responses_07_get_attribute_value.test | 139 ++++++++++ ...r_responses_08_update_attribute_value.test | 145 +++++++++++ 17 files changed, 1480 insertions(+), 42 deletions(-) create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_00_get_entity_and_entity_attributes.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_01_update_or_append_entity_attributes.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_02_update_existing_entity_attributes.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_03_replace_all_attributes.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_04_delete_entity.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_05_update_attribute.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_06_delete_attribute.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_07_get_attribute_value.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_08_update_attribute_value.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 7f815e5662..ef5fd3affb 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -5,6 +5,7 @@ - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) - Fix: improve error traces (#4387) - Fix: on delete attributes operation, the lack of attribute in entity doesn't preclude the deletion of attributes which do exist (this way behaving same way as update attributes operation) +- Fix: POST /v2/entities/E/attrs?options=append was wrongly creating entities - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) - Fix: improve logs in MongoDB query logic - Upgrade Debian version from 11.6 to 12.1 in Dockerfile diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index d45e1305f6..577d88b5d7 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2816,7 +2816,8 @@ _**Request headers**_ _**Response code**_ * Successful operation uses 200 OK -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for more details. +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity _**Response headers**_ @@ -2824,7 +2825,16 @@ Successful operations return `Content-Type` header with `application/json` value _**Response payload**_ -The response is an object representing the entity identified by the ID. The object follows +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +If the entity is found, the response is an object representing the entity identified by the ID. The object follows the JSON entity representation format (described in [JSON Entity Representation](#json-entity-representation) section and side [Simplified Entity Representation](#simplified-entity-representation) and [Partial Representations](#partial-representations) sections). @@ -2899,8 +2909,8 @@ _**Request headers**_ _**Response code**_ * Successful operation uses 200 OK -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity _**Response headers**_ @@ -2908,7 +2918,16 @@ Successful operations return `Content-Type` header with `application/json` value _**Response payload**_ -The payload is an object representing the entity identified by the ID in the URL parameter. The object follows +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +If the entity is found, the payload is an object representing the entity identified by the ID in the URL parameter. The object follows the JSON entity representation format (described in [JSON Entity Representation](#json-entity-representation) section and side [Simplified Entity Representation](#simplified-entity-representation) and [Partial Representations](#partial-representations) sections), but omitting `id` and `type` fields. @@ -3005,8 +3024,44 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity + * 422 Unprocessable Content for existing attributes when `append` options is used + +_**Response payload**_ + +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +In the case of *all* attributes exist when `append` options is used: + +``` +{ + "description": "one or more of the attributes in the request already exist: E/T - [ A, B ]", + "error": "Unprocessable" +} +``` + +In the case of *some* (but not all) attributes exist when `append` options is used (partial update): + +``` +{ + "description": "one or more of the attributes in the request already exist: E/T - [ B ]", + "error": "PartialUpdate" +} +``` + +The entity type in `description` is shown only if the request includes it. Otherwise, it is omitted: + +``` +"description": "one or more of the attributes in the request already exist: E - [ B ]", +``` #### Update Existing Entity Attributes `PATCH /v2/entities/{entityId}/attrs` @@ -3068,8 +3123,44 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity + * 422 Unprocessable Content for non existing attributes + +_**Response payload**_ + +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +In the case of *none* of the attributes in the request exist: + +``` +{ + "description": "do not exist: E/T - [ C, D ]", + "error": "Unprocessable" +} +``` + +In the case of *some* (but not all) attributes does not exist (partial update): + +``` +{ + "description": "do not exist: E/T - [ C ]", + "error": "PartialUpdate" +} +``` + +The entity type in `description` is shown only if the request includes it. Otherwise, it is omitted: + +``` +"description": "do not exist: E - [ C ]", +``` #### Replace all entity attributes `PUT /v2/entities/{entityId}/attrs` @@ -3130,8 +3221,19 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity + +_**Response payload**_ + +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` #### Remove Entity `DELETE /v2/entities/{entityId}` @@ -3161,8 +3263,19 @@ _**Request headers**_ _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity + +_**Response payload**_ + +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` ### Attributes @@ -3287,8 +3400,28 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity or not found attribute + +_**Response payload**_ + +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +In the case of not found attribute: + +``` +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} +``` #### Remove a Single Attribute `DELETE /v2/entities/{entityId}/attrs/{attrName}` @@ -3319,8 +3452,28 @@ _**Request headers**_ _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity or not found attribute + +_**Response payload**_ + +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +In the case of not found attribute: + +``` +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} +``` ### Attribute Value @@ -3359,9 +3512,10 @@ _**Request headers**_ _**Response code**_ -* Successful operation uses 200 OK. -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Successful operation uses 200 OK +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity or not found attribute + * 406 Not Acceptable in the case of not acceptable content _**Response headers**_ @@ -3369,7 +3523,26 @@ _**Response headers**_ _**Response payload**_ -The response payload can be an object, array, string, number, boolean or null with the value of the attribute. +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +In the case of not found attribute: + +``` +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} +``` + +In the case of entity and attribute both are found, the response payload can be an object, array, string, +number, boolean or null with the value of the attribute. * If attribute value is JSON Array or Object: * If `Accept` header can be expanded to `application/json` or `text/plain` return the value as a JSON with a @@ -3458,8 +3631,28 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found for not found entity or not found attribute + +_**Response payload**_ + +In the case of not found entity: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +In the case of not found attribute: + +``` +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} +``` ### Types diff --git a/src/lib/common/globals.h b/src/lib/common/globals.h index 73c68c7187..2d385d9092 100644 --- a/src/lib/common/globals.h +++ b/src/lib/common/globals.h @@ -150,7 +150,7 @@ typedef enum Ngsiv2Flavour NGSIV2_NO_FLAVOUR = 0, NGSIV2_FLAVOUR_ONCREATE = 1, NGSIV2_FLAVOUR_ONAPPEND = 2, - NGSIV2_FLAVOUR_ONUPDATE = 3 + NGSIV2_FLAVOUR_ONPUREAPPEND = 3 } Ngsiv2Flavour; diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 6fba6a2fb9..cea1c10471 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4188,7 +4188,8 @@ unsigned int processContextElement } // This is the case of POST /v2/entities/, in order to check that entity previously exist - if ((entitiesNumber == 0) && (ngsiv2Flavour == NGSIV2_FLAVOUR_ONAPPEND)) + // both for regular case (NGSIV2_FLAVOUR_ONAPPEND) and when ?options=append is used (NGSIV2_FLAVOUR_ONPUREAPPEND) + if ((entitiesNumber == 0) && ((ngsiv2Flavour == NGSIV2_FLAVOUR_ONAPPEND)||(ngsiv2Flavour == NGSIV2_FLAVOUR_ONPUREAPPEND))) { buildGeneralErrorResponse(eP, NULL, responseP, SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY); responseP->oe.fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ERROR_NOT_FOUND); diff --git a/src/lib/serviceRoutinesV2/postEntity.cpp b/src/lib/serviceRoutinesV2/postEntity.cpp index 7797bc7e87..c81b095cc9 100644 --- a/src/lib/serviceRoutinesV2/postEntity.cpp +++ b/src/lib/serviceRoutinesV2/postEntity.cpp @@ -76,10 +76,11 @@ std::string postEntity return oe.toJson(); } - if (ciP->uriParamOptions["append"] == true) // pure-append + // FIXME PR: maybe NGSIV2_FLAVOUR_ONPUREAPPEND and NGSIV2_FLAVOUR_ONAPPEND can be merged into a single one + if (ciP->uriParamOptions[OPT_APPEND] == true) // pure-append { op = ActionTypeAppendStrict; - flavor = NGSIV2_FLAVOUR_ONUPDATE; + flavor = NGSIV2_FLAVOUR_ONPUREAPPEND; } else { diff --git a/src/lib/serviceRoutinesV2/putEntity.cpp b/src/lib/serviceRoutinesV2/putEntity.cpp index d44b23190a..0a6f88580d 100644 --- a/src/lib/serviceRoutinesV2/putEntity.cpp +++ b/src/lib/serviceRoutinesV2/putEntity.cpp @@ -34,6 +34,7 @@ #include "apiTypesV2/Entities.h" #include "rest/EntityTypeInfo.h" #include "serviceRoutinesV2/putEntity.h" +#include "serviceRoutinesV2/serviceRoutinesCommon.h" #include "serviceRoutines/postUpdateContext.h" #include "rest/OrionError.h" #include "parse/forbiddenChars.h" @@ -43,7 +44,7 @@ * * putEntity - * -* PUT /v2/entities +* PUT /v2/entities//attrs * * Payload In: Entity * Payload Out: None @@ -84,12 +85,8 @@ std::string putEntity // 02. Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); - // Adjust NotFound description (to avoid redundant missing entity information) - // FIXME PR: duplicated code in several places - if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) - { - parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; - } +// Adjust error code if needed + adaptErrorCodeForSingleEntityOperation(&(parseDataP->upcrs.res.oe), false); // 03. Check error std::string answer = ""; diff --git a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp index 83d9af39ca..25713cb229 100644 --- a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp +++ b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp @@ -86,7 +86,7 @@ std::string putEntityAttribute postUpdateContext(ciP, components, compV, parseDataP); // Adjust error code if needed - adaptErrorCodeForSingleEntityOperation(&(parseDataP->upcrs.res.oe), false); + adaptErrorCodeForSingleEntityOperation(&(parseDataP->upcrs.res.oe), true); // 03. Check error std::string answer = ""; diff --git a/src/lib/serviceRoutinesV2/putEntityAttributeValue.cpp b/src/lib/serviceRoutinesV2/putEntityAttributeValue.cpp index c4e8ea8f18..0f41e87f36 100644 --- a/src/lib/serviceRoutinesV2/putEntityAttributeValue.cpp +++ b/src/lib/serviceRoutinesV2/putEntityAttributeValue.cpp @@ -33,6 +33,7 @@ #include "rest/EntityTypeInfo.h" #include "serviceRoutines/postUpdateContext.h" #include "serviceRoutinesV2/putEntityAttributeValue.h" +#include "serviceRoutinesV2/serviceRoutinesCommon.h" #include "rest/OrionError.h" #include "parse/forbiddenChars.h" @@ -91,13 +92,8 @@ std::string putEntityAttributeValue // 02. Call standard op postUpdateContext postUpdateContext(ciP, components, compV, parseDataP); - // Adjust NotFound description (to avoid redundant missing entity information) - // FIXME PR: duplicated code in several places - if ((parseDataP->upcrs.res.oe.code == SccContextElementNotFound) & (parseDataP->upcrs.res.oe.reasonPhrase == ERROR_NOT_FOUND)) - { - parseDataP->upcrs.res.oe.details = ERROR_DESC_NOT_FOUND_ENTITY; - } - + // Adjust error code if needed + adaptErrorCodeForSingleEntityOperation(&(parseDataP->upcrs.res.oe), true); // 03. Check output from mongoBackend std::string answer = ""; diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_00_get_entity_and_entity_attributes.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_00_get_entity_and_entity_attributes.test new file mode 100644 index 0000000000..26686104d7 --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_00_get_entity_and_entity_attributes.test @@ -0,0 +1,101 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Get entity (GET /v2/entities/{entityId}) and get entity attributes (GET /v2/entities/{entityId}) error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found for not found entity or not found attribute +# +# Response payload: +# +# In the case of not found entity: +# +# { +# "description": "The requested entity has not been found. Check type and id", +# "error": "NotFound" +# } +# + +# +# 01. GET /v2/entities/F, see error (NotFound) +# 02. GET /v2/entities/F/attrs, see error (NotFound) +# + + +echo "01. GET /v2/entities/F, see error (NotFound)" +echo "============================================" +orionCurl --url /v2/entities/F +echo +echo + + +echo "02. GET /v2/entities/F/attrs, see error (NotFound)" +echo "==================================================" +orionCurl --url /v2/entities/F/attrs +echo +echo + + +--REGEXPECT-- +01. GET /v2/entities/F, see error (NotFound) +============================================ +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +02. GET /v2/entities/F/attrs, see error (NotFound) +================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_01_update_or_append_entity_attributes.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_01_update_or_append_entity_attributes.test new file mode 100644 index 0000000000..6348a0d60a --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_01_update_or_append_entity_attributes.test @@ -0,0 +1,221 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Update or Append Entity Attributes (POST /v2/entities/{entityId}/attrs) error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response codes: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found for not found entity +# * 422 Unprocessable Content for existing attributes when `append` options is used +# +# Response payload: +# +# In the case of not found entity: +# +# { +# "description": "The requested entity has not been found. Check type and id", +# "error": "NotFound" +# } +# +# In the case of *all* attributes exist when `append` options is used: +# +# { +# "description": "one or more of the attributes in the request already exist: E/T - [ A, B ]", +# "error": "Unprocessable" +# } +# +# In the case of *some* (but not all) attributes exist when `append` options is used (partial update): +# +# { +# "description": "one or more of the attributes in the request already exist: E/T - [ B ]", +# "error": "PartialUpdate" +# } +# + +# +# 01. POST /v2/entities/F/attrs, see error (NotFound) +# 02. POST /v2/entities/F/attrs?options=append, see error (NotFound) +# 03. Create entity E/A-B +# 04. POST /v2/entities/E/attrs?options=append with A and B, see error (Unprocessable) +# 05. POST /v2/entities/E/attrs?options=append with B and C, see error (PartialUpdate) +# + +echo "01. POST /v2/entities/F/attrs, see error (NotFound)" +echo "===================================================" +payload='{ + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/F/attrs --payload "$payload" +echo +echo + + +echo "02. POST /v2/entities/F/attrs?options=append, see error (NotFound)" +echo "==================================================================" +payload='{ + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/F/attrs?options=append --payload "$payload" +echo +echo + + +echo "03. Create entity E/A-B" +echo "=======================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "04. POST /v2/entities/E/attrs?options=append with A and B, see error (Unprocessable)" +echo "====================================================================================" +payload='{ + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs?options=append --payload "$payload" +echo +echo + + +echo "05. POST /v2/entities/E/attrs?options=append with B and C, see error (PartialUpdate)" +echo "====================================================================================" +payload='{ + "B": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs?options=append --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. POST /v2/entities/F/attrs, see error (NotFound) +=================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +02. POST /v2/entities/F/attrs?options=append, see error (NotFound) +================================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +03. Create entity E/A-B +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +04. POST /v2/entities/E/attrs?options=append with A and B, see error (Unprocessable) +==================================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 114 + +{ + "description": "one or more of the attributes in the request already exist: E - [ A, B ]", + "error": "Unprocessable" +} + + +05. POST /v2/entities/E/attrs?options=append with B and C, see error (PartialUpdate) +==================================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 111 + +{ + "description": "one or more of the attributes in the request already exist: E - [ B ]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_02_update_existing_entity_attributes.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_02_update_existing_entity_attributes.test new file mode 100644 index 0000000000..3f585d26ae --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_02_update_existing_entity_attributes.test @@ -0,0 +1,193 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Update Existing Entity Attributes (PATCH /v2/entities/{entityId}/attrs) error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found for not found entity +# * 422 Unprocessable Content for non existing attributes +# +# Response payload: +# +# In the case of not found entity: +# +# { +# "description": "The requested entity has not been found. Check type and id", +# "error": "NotFound" +# } +# +# In the case of *none* of the attributes in the request exist: +# +# { +# "description": "do not exist: E/T - [ C, D ]", +# "error": "Unprocessable" +# } +# +# In the case of *some* (but not all) attributes does not exist (partial update) +# +# { +# "description": "do not exist: E/T - [ C ]", +# "error": "PartialUpdate" +# } +# + +# +# 01. PATCH /v2/entities/F/attrs, see error (NotFound) +# 02. Create entity E/A-B +# 03. PATCH /v2/entities/E/attrs with C and D, see error (Unprocessable) +# 04. PATCH /v2/entities/E/attrs with B and C, see error (PartialUpdate) +# + +echo "01. PATCH /v2/entities/F/attrs, see error (NotFound)" +echo "====================================================" +payload='{ + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/F/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "02. Create entity E/A-B" +echo "=======================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. PATCH /v2/entities/E/attrs with C and D, see error (Unprocessable)" +echo "======================================================================" +payload='{ + "C": { + "value": 1, + "type": "Number" + }, + "D": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs?options=append --payload "$payload" -X PATCH +echo +echo + + +echo "04. PATCH /v2/entities/E/attrs with B and C, see error (PartialUpdate)" +echo "======================================================================" +payload='{ + "B": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs?options=append --payload "$payload" -X PATCH +echo +echo + + +--REGEXPECT-- +01. PATCH /v2/entities/F/attrs, see error (NotFound) +==================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +02. Create entity E/A-B +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. PATCH /v2/entities/E/attrs with C and D, see error (Unprocessable) +====================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "do not exist: E - [ C, D ]", + "error": "Unprocessable" +} + + +04. PATCH /v2/entities/E/attrs with B and C, see error (PartialUpdate) +====================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 65 + +{ + "description": "do not exist: E - [ C ]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_03_replace_all_attributes.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_03_replace_all_attributes.test new file mode 100644 index 0000000000..1829afed4a --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_03_replace_all_attributes.test @@ -0,0 +1,86 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Replace all entity attributes (PUT /v2/entities/{entityId}/attrs) error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found for not found entity +# +# Response payload: +# +# In the case of not found entity: +# +# { +# "description": "The requested entity has not been found. Check type and id", +# "error": "NotFound" +# } +# + +# +# 01. PUT /v2/entities/F/attrs, see error (NotFound) +# + +echo "01. PUT /v2/entities/F/attrs, see error (NotFound)" +echo "==================================================" +payload='{ + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/F/attrs --payload "$payload" -X PUT +echo +echo + + + + +--REGEXPECT-- +01. PUT /v2/entities/F/attrs, see error (NotFound) +================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_04_delete_entity.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_04_delete_entity.test new file mode 100644 index 0000000000..7642c28b47 --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_04_delete_entity.test @@ -0,0 +1,80 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Delete entity (DELETE /v2/entities/{entityId}) error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found for not found entity +# +# Response payload: +# +# In the case of not found entity: +# +# { +# "description": "The requested entity has not been found. Check type and id", +# "error": "NotFound" +# } +# + +# +# 01. DELETE /v2/entities/F, see error (NotFound) +# + +echo "01. DELETE /v2/entities/F, see error (NotFound)" +echo "===============================================" +orionCurl --url /v2/entities/F -X DELETE +echo +echo + + + + +--REGEXPECT-- +01. DELETE /v2/entities/F, see error (NotFound) +=============================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_05_update_attribute.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_05_update_attribute.test new file mode 100644 index 0000000000..830d0e027e --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_05_update_attribute.test @@ -0,0 +1,146 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Update attribute (PUT /v2/entities/{entityId}/attrs/{attrName}) error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found for not found entity or not found attribute +# +# Response payload: +# +# In the case of not found entity: +# +# { +# "description": "The requested entity has not been found. Check type and id", +# "error": "NotFound" +# } +# +# In the case of not found attribute: +# +# { +# "description": "The entity does not have such an attribute", +# "error": "NotFound" +# } +# + +# +# 01. PUT /v2/entities/F/attrs/C, see error (NotFound) +# 02. Create entity E/A-B +# 03. PUT /v2/entities/E/attrs/C, see error (NotFound) +# + + +echo "01. PUT /v2/entities/F/attrs/C, see error (NotFound)" +echo "====================================================" +payload='{ + "type": "Number", + "value": 1 +}' +orionCurl --url /v2/entities/F/attrs/C --payload "$payload" -X PUT +echo +echo + + +echo "02. Create entity E/A-B" +echo "=======================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. PUT /v2/entities/E/attrs/C, see error (NotFound)" +echo "====================================================" +payload='{ + "type": "Number", + "value": 1 +}' +orionCurl --url /v2/entities/E/attrs/C --payload "$payload" -X PUT +echo +echo + + +--REGEXPECT-- +01. PUT /v2/entities/F/attrs/C, see error (NotFound) +==================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +02. Create entity E/A-B +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. PUT /v2/entities/E/attrs/C, see error (NotFound) +==================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 79 + +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_06_delete_attribute.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_06_delete_attribute.test new file mode 100644 index 0000000000..5bbf88ef5f --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_06_delete_attribute.test @@ -0,0 +1,138 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Update attribute (PUT /v2/entities/{entityId}/attrs/{attrName}) error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found for not found entity or not found attribute +# +# Response payload: +# +# In the case of not found entity: +# +# { +# "description": "The requested entity has not been found. Check type and id", +# "error": "NotFound" +# } +# +# In the case of not found attribute: +# +# { +# "description": "The entity does not have such an attribute", +# "error": "NotFound" +# } +# + +# +# 01. DELETE /v2/entities/F/attrs/C, see error (NotFound) +# 02. Create entity E/A-B +# 03. DELETE /v2/entities/E/attrs/C, see error (NotFound) +# + + +echo "01. DELETE /v2/entities/F/attrs/C, see error (NotFound)" +echo "=======================================================" +orionCurl --url /v2/entities/F/attrs/C -X DELETE +echo +echo + + +echo "02. Create entity E/A-B" +echo "=======================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. DELETE /v2/entities/E/attrs/C, see error (NotFound)" +echo "=======================================================" +orionCurl --url /v2/entities/E/attrs/C -X DELETE +echo +echo + + +--REGEXPECT-- +01. DELETE /v2/entities/F/attrs/C, see error (NotFound) +======================================================= +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +02. Create entity E/A-B +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. DELETE /v2/entities/E/attrs/C, see error (NotFound) +======================================================= +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 79 + +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_07_get_attribute_value.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_07_get_attribute_value.test new file mode 100644 index 0000000000..230695cb1a --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_07_get_attribute_value.test @@ -0,0 +1,139 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Get attribute value (GET /v2/entities/{entityId}/attrs/{attrName}/value) error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found for not found entity or not found attribute +# ... +# +# Response payload: +# +# In the case of not found entity: +# +# { +# "description": "The requested entity has not been found. Check type and id", +# "error": "NotFound" +# } +# +# In the case of not found attribute: +# +# { +# "description": "The entity does not have such an attribute", +# "error": "NotFound" +# } +# + +# +# 01. GET /v2/entities/F/attrs/C/value, see error (NotFound) +# 02. Create entity E/A-B +# 03. GET /v2/entities/E/attrs/C/value, see error (NotFound) +# + + +echo "01. GET /v2/entities/F/attrs/C/value, see error (NotFound)" +echo "==========================================================" +orionCurl --url /v2/entities/F/attrs/C/value +echo +echo + + +echo "02. Create entity E/A-B" +echo "=======================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. GET /v2/entities/E/attrs/C/value, see error (NotFound)" +echo "==========================================================" +orionCurl --url /v2/entities/E/attrs/C +echo +echo + + +--REGEXPECT-- +01. GET /v2/entities/F/attrs/C/value, see error (NotFound) +========================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +02. Create entity E/A-B +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. GET /v2/entities/E/attrs/C/value, see error (NotFound) +========================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 79 + +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_08_update_attribute_value.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_08_update_attribute_value.test new file mode 100644 index 0000000000..2997f46d0f --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_08_update_attribute_value.test @@ -0,0 +1,145 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Update attribute value (PUT /v2/entities/{entityId}/attrs/{attrName}/value) error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found for not found entity or not found attribute +# ... +# +# Response payload: +# +# In the case of not found entity: +# +# { +# "description": "The requested entity has not been found. Check type and id", +# "error": "NotFound" +# } +# +# In the case of not found attribute: +# +# { +# "description": "The entity does not have such an attribute", +# "error": "NotFound" +# } +# + +# +# 01. PUT /v2/entities/F/attrs/C/value, see error (NotFound) +# 02. Create entity E/A-B +# 03. PUT /v2/entities/E/attrs/C/value, see error (NotFound) +# + + +echo "01. PUT /v2/entities/F/attrs/C/value, see error (NotFound)" +echo "==========================================================" +payload='{ + "x": 1 +}' +orionCurl --url /v2/entities/F/attrs/C/value --payload "$payload" -X PUT +echo +echo + + +echo "02. Create entity E/A-B" +echo "=======================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. PUT /v2/entities/E/attrs/C/value, see error (NotFound)" +echo "==========================================================" +payload='{ + "x": 1 +}' +orionCurl --url /v2/entities/E/attrs/C/value --payload "$payload" -X PUT +echo +echo + + +--REGEXPECT-- +01. PUT /v2/entities/F/attrs/C/value, see error (NotFound) +========================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +02. Create entity E/A-B +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. PUT /v2/entities/E/attrs/C/value, see error (NotFound) +========================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 79 + +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file From 2bd828c9cd4d50170595f437dfaa531dca52acce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 13 Dec 2023 11:18:35 +0100 Subject: [PATCH 031/390] FIX align some existing test cases with recent changes --- .../error_response_for_entity_not_found.test | 8 ++++---- .../invalid_error_updating_non_existing_attribute.test | 8 ++++---- .../PATCH_offending_attributes.test | 8 ++++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test b/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test index 70b67f7e68..8fafc70198 100644 --- a/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test +++ b/test/functionalTest/cases/1360_error_response_for_entity_not_found/error_response_for_entity_not_found.test @@ -93,15 +93,15 @@ Content-Length: 0 03. Attempt to update attribute dfgdfgdf of entity room_2 (the attribute does not exist) ======================================================================================== -HTTP/1.1 422 Unprocessable Content +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 77 +Content-Length: 79 { - "description": "do not exist: room_2 - [ dfgdfgdf ]", - "error": "Unprocessable" + "description": "The entity does not have such an attribute", + "error": "NotFound" } diff --git a/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test b/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test index 2592785cec..b546c7e6e5 100644 --- a/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test +++ b/test/functionalTest/cases/2221_invalid_error_updating_non_existing_attribute/invalid_error_updating_non_existing_attribute.test @@ -79,15 +79,15 @@ Content-Length: 0 02. Attempt to replace E1/A2, see error about attribute not found ================================================================= -HTTP/1.1 422 Unprocessable Content +HTTP/1.1 404 Not Found Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 67 +Content-Length: 79 { - "description": "do not exist: E1 - [ A2 ]", - "error": "Unprocessable" + "description": "The entity does not have such an attribute", + "error": "NotFound" } diff --git a/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test b/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test index a69fc89d16..22be5d5436 100644 --- a/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test +++ b/test/functionalTest/cases/2933_PATCH_offending_attributes/PATCH_offending_attributes.test @@ -42,6 +42,8 @@ brokerStart CB echo '01. POST /v2/entities, to create Room1 with temperature and pressure.' echo '=====================================================================' payload='{ + "id": "Room1", + "type": "Thing", "temperature": { "value": 23, "type": "Float" @@ -51,7 +53,7 @@ payload='{ "type": "Float" } }' -orionCurl --url /v2/entities/Room1/attrs?options=append --payload "$payload" +orionCurl --url /v2/entities --payload "$payload" echo echo @@ -105,9 +107,11 @@ echo --REGEXPECT-- 01. POST /v2/entities, to create Room1 with temperature and pressure. ===================================================================== -HTTP/1.1 204 No Content +HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/Room1?type=Thing +Content-Length: 0 From b240965aa6a3c7bcea3c8c11bdfe790d961af367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 13 Dec 2023 14:14:51 +0100 Subject: [PATCH 032/390] FIX additional test cases for error responses in batch update and align in the code --- doc/manuals/orion-api.md | 84 +++++- src/lib/mongoBackend/mongoUpdateContext.cpp | 4 +- ...or_responses_09a_batch_update_replace.test | 182 +++++++++++++ ...ror_responses_09b_batch_update_update.test | 241 ++++++++++++++++++ ...ror_responses_09c_batch_update_delete.test | 241 ++++++++++++++++++ ...ponses_09d_batch_update_append_strict.test | 230 +++++++++++++++++ 6 files changed, 977 insertions(+), 5 deletions(-) create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_09a_batch_update_replace.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_09b_batch_update_update.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_09c_batch_update_delete.test create mode 100644 test/functionalTest/cases/0000_update_error_responses/error_responses_09d_batch_update_append_strict.test diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 577d88b5d7..967a185cc3 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -4635,9 +4635,87 @@ Example: _**Response code**_ -* Successful operation uses 204 No Content. -* Errors use a non-2xx and (optionally) an error payload. See subsection on [Error Responses](#error-responses) for - more details. +* Successful operation uses 204 No Content +* Errors use a non-2xx code and error payload (see next subsection): + * 404 Not Found if none of the entities in the `entities` field exists in `update`, `delete` or `replace` cases + * 422 Unprocessable Content for other cases + +_**Response payload**_ + +For action type `replace`: + +* If *none* of the entities in `entities` exist: + +``` +{ + "description": "do not exist: F/T - [entity itself], G/T [entity itself]", + "error": "NotFound" +} +``` + +* If *any (but not all)* of the entities in `entities` does not exist (partial update): + +``` +{ + "description": "do not exist: G/T - [entity itself]", + "error": "PartialUpdate" +} +``` + +For action type `update` or `delete`: + +* If *none* of the entities in `entities` exist: + +``` +{ + "description": "do not exist: F/T - [entity itself], G/T [entity itself]", + "error": "NotFound" +} +``` + +* If at least one entity in `entities` exists and in *all* of existing entities there was a *full fail* due to missing attributes: + +``` +{ + "description": "do not exist: E/T - [ C, D ], G/T [entity itself]", + "error": "Unprocessable" +} +``` + +* If at least one entity in `entities` exists and in *at least one* of the existing entities *at least* one attribute exists + but not all entities exist or all entities exist but in at least one entity there is at least one missing attribute (partial update): + +``` +{ + "description": "do not exist: E/T - [ D ], G/T [entity itself]", + "error": "PartialUpdate" +} +``` + +For action type `appendStrict`:` + +* If in *all* entities in `entities` there was a *full fail* due to existing attributes: + +{ + "description": "one or more of the attributes in the request already exist: E1/T - [ A, B ], E2/T - [ A, B ]", + "error": "Unprocessable" +} + +* If *in some (but not in all)* entities in `entities` there was a fail due to existing attributes (partial update): + +* If in *at least one entity* in `entities` in *at least* one attribute there was a success but not all entities in `entities` have + a full success (partial update): + +{ + "description": "one or more of the attributes in the request already exist: E2/T - [ A, B ]", + "error": "PartialUpdate" +} + +The entity type in `description` is shown only if the request includes it. Otherwise, it is omitted: + +``` +"description": "one or more of the attributes in the request already exist: E2 - [ A, B ]" +``` ### Query operation diff --git a/src/lib/mongoBackend/mongoUpdateContext.cpp b/src/lib/mongoBackend/mongoUpdateContext.cpp index 80ebfad8b3..4a19eaa863 100644 --- a/src/lib/mongoBackend/mongoUpdateContext.cpp +++ b/src/lib/mongoBackend/mongoUpdateContext.cpp @@ -184,9 +184,9 @@ HttpStatusCode mongoUpdateContext } break; case UC_FULL_ATTRS_FAIL: - // If global UC is full attrs fail, we need also full attrs fail in the processed entity to keep global full attrs fail + // If global UC is full attrs fail, we need also full attrs fail or not found entity in the processed entity to keep global full attrs fail // Otherwise (success, partial, not found entity), the global UC changes to partial - if (entityUpdateCoverage != UC_FULL_ATTRS_FAIL) + if ((entityUpdateCoverage != UC_FULL_ATTRS_FAIL) && (entityUpdateCoverage != UC_ENTITY_NOT_FOUND)) { updateCoverage = UC_PARTIAL; } diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_09a_batch_update_replace.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_09a_batch_update_replace.test new file mode 100644 index 0000000000..afa86653a0 --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_09a_batch_update_replace.test @@ -0,0 +1,182 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Batch update (POST /v2/op/update) with replace action error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found if none of the entitys in the `entities` field exists in `update`, `delete` or `replace` cases +# * 422 Unprocessable Content for other cases +# +# Response payload: +# +# For action type `replace`: +# +# * If *none* of the entities in `entities` exist: +# +# { +# "description": "do not exist: F/T - [entity itself], G/T [entity itself]", +# "error": "NotFound" +# } +# +# * If *any (but not all)* of the entities in `entities` does not exist (partial update): +# +# { +# "description": "do not exist: G/T - [entity itself]", +# "error": "PartialUpdate" +# } +# + +# +# 01. POST /v2/op/update replace with entities F/G, see error (NotFound) +# 02. Create entity E/A-B +# 03. POST /v2/op/update replace with entities E/G, see error (PartialUpdate) +# + +echo "01. POST /v2/op/update replace with entities F/G, see error (NotFound)" +echo "======================================================================" +payload='{ + "actionType": "replace", + "entities": [ + { + "id": "F", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + }, + { + "id": "G", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. Create entity E/A-B" +echo "=======================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. POST /v2/op/update replace with entities E/G, see error (PartialUpdate)" +echo "===========================================================================" +payload='{ + "actionType": "replace", + "entities": [ + { + "id": "E", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + }, + { + "id": "G", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. POST /v2/op/update replace with entities F/G, see error (NotFound) +====================================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 93 + +{ + "description": "do not exist: F/T - [entity itself], G/T [entity itself]", + "error": "NotFound" +} + + +02. Create entity E/A-B +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. POST /v2/op/update replace with entities E/G, see error (PartialUpdate) +=========================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 77 + +{ + "description": "do not exist: G/T - [entity itself]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_09b_batch_update_update.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_09b_batch_update_update.test new file mode 100644 index 0000000000..da4ba04e0e --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_09b_batch_update_update.test @@ -0,0 +1,241 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Batch update (POST /v2/op/update) with update action error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found if none of the entitys in the `entities` field exists in `update`, `delete` or `replace` cases +# * 422 Unprocessable Content for other cases +# +# Response payload: +# +# For action type `update` or `delete`: +# +# * If *none* of the entities in `entities` exist: +# +# { +# "description": "do not exist: F/T - [entity itself], G/T [entity itself]", +# "error": "NotFound" +# } +# +# * If at least one entity in `entities` exists and in *all* of existing entities there was a *full fail* due to missing attributes: +# +# { +# "description": "do not exist: E/T - [ C, D ], G/T [entity itself]", +# "error": "Unprocessable" +# } +# +# * If at least one entity in `entities` exists and in *at least one* of the existing entities *at least* one attribute exists +# but not all entities exists or all entities exist but in at least one entity there is at least one missing attribute (partial update): +# +# { +# "description": "do not exist: E/T - [ D ], G/T [entity itself]", +# "error": "PartialUpdate" +# } +# + +# +# 01. POST /v2/op/update update with entities F/G, see error (NotFound) +# 02. Create entity E/A-B +# 03. POST /v2/op/update update with entities E{C,D}/G, see error (Unprocessable) +# 04. POST /v2/op/update update with entities E{A,D}/G, see error (PartialUpdate) +# + +echo "01. POST /v2/op/update update with entities F/G, see error (NotFound)" +echo "=====================================================================" +payload='{ + "actionType": "update", + "entities": [ + { + "id": "F", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + }, + { + "id": "G", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. Create entity E/A-B" +echo "=======================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. POST /v2/op/update update with entities E{C,D}/G, see error (Unprocessable)" +echo "===============================================================================" +payload='{ + "actionType": "update", + "entities": [ + { + "id": "E", + "type": "T", + "C": { + "type": "Number", + "value": 1 + }, + "D": { + "type": "Number", + "value": 1 + } + }, + { + "id": "G", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "04. POST /v2/op/update update with entities E{A,D}/G, see error (PartialUpdate)" +echo "===============================================================================" +payload='{ + "actionType": "update", + "entities": [ + { + "id": "E", + "type": "T", + "A": { + "type": "Number", + "value": 1 + }, + "D": { + "type": "Number", + "value": 1 + } + }, + { + "id": "G", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. POST /v2/op/update update with entities F/G, see error (NotFound) +===================================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 93 + +{ + "description": "do not exist: F/T - [entity itself], G/T [entity itself]", + "error": "NotFound" +} + + +02. Create entity E/A-B +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. POST /v2/op/update update with entities E{C,D}/G, see error (Unprocessable) +=============================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 91 + +{ + "description": "do not exist: E/T - [ C, D ], G/T [entity itself]", + "error": "Unprocessable" +} + + +04. POST /v2/op/update update with entities E{A,D}/G, see error (PartialUpdate) +=============================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 88 + +{ + "description": "do not exist: E/T - [ D ], G/T [entity itself]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_09c_batch_update_delete.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_09c_batch_update_delete.test new file mode 100644 index 0000000000..b7467902c5 --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_09c_batch_update_delete.test @@ -0,0 +1,241 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Batch update (POST /v2/op/update) with delete action error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# * 404 Not Found if none of the entitys in the `entities` field exists in `update`, `delete` or `replace` cases +# * 422 Unprocessable Content for other cases +# +# Response payload: +# +# For action type `update` or `delete`: +# +# * If *none* of the entities in `entities` exist: +# +# { +# "description": "do not exist: F/T - [entity itself], G/T [entity itself]", +# "error": "NotFound" +# } +# +# * If at least one entity in `entities` exists and in *all* of existing entities there was a *full fail* due to missing attributes: +# +# { +# "description": "do not exist: E/T - [ C, D ], G/T [entity itself]", +# "error": "Unprocessable" +# } +# +# * If at least one entity in `entities` exists and in *at least one* of the existing entities *at least* one attribute exists +# but not all entities exists or all entities exist but in at least one entity there is at least one missing attribute (partial update): +# +# { +# "description": "do not exist: E/T - [ D ], G/T [entity itself]", +# "error": "PartialUpdate" +# } +# + +# +# 01. POST /v2/op/update delete with entities F/G, see error (NotFound) +# 02. Create entity E/A-B +# 03. POST /v2/op/update delete with entities E{C,D}/G, see error (Unprocessable) +# 04. POST /v2/op/update delete with entities E{A,D}/G, see error (PartialUpdate) +# + +echo "01. POST /v2/op/update delete with entities F/G, see error (NotFound)" +echo "=====================================================================" +payload='{ + "actionType": "delete", + "entities": [ + { + "id": "F", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + }, + { + "id": "G", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "02. Create entity E/A-B" +echo "=======================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. POST /v2/op/update delete with entities E{C,D}/G, see error (Unprocessable)" +echo "===============================================================================" +payload='{ + "actionType": "delete", + "entities": [ + { + "id": "E", + "type": "T", + "C": { + "type": "Number", + "value": 1 + }, + "D": { + "type": "Number", + "value": 1 + } + }, + { + "id": "G", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "04. POST /v2/op/update delete with entities E{A,D}/G, see error (PartialUpdate)" +echo "===============================================================================" +payload='{ + "actionType": "delete", + "entities": [ + { + "id": "E", + "type": "T", + "A": { + "type": "Number", + "value": 1 + }, + "D": { + "type": "Number", + "value": 1 + } + }, + { + "id": "G", + "type": "T", + "A": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. POST /v2/op/update delete with entities F/G, see error (NotFound) +===================================================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 93 + +{ + "description": "do not exist: F/T - [entity itself], G/T [entity itself]", + "error": "NotFound" +} + + +02. Create entity E/A-B +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. POST /v2/op/update delete with entities E{C,D}/G, see error (Unprocessable) +=============================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 91 + +{ + "description": "do not exist: E/T - [ C, D ], G/T [entity itself]", + "error": "Unprocessable" +} + + +04. POST /v2/op/update delete with entities E{A,D}/G, see error (PartialUpdate) +=============================================================================== +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 88 + +{ + "description": "do not exist: E/T - [ D ], G/T [entity itself]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_09d_batch_update_append_strict.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_09d_batch_update_append_strict.test new file mode 100644 index 0000000000..199614d018 --- /dev/null +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_09d_batch_update_append_strict.test @@ -0,0 +1,230 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Batch update (POST /v2/op/update) with replace appendStrict error responses + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# From documentation: +# +# Response code: +# +# ... +# * Errors use a non-2xx code and error payload (see next subsection): +# ... +# * 422 Unprocessable Content for other cases +# +# Response payload: +# +# For action type `appendStrict`:` +# +# * If in *all* entities in `entities` there was a *full fail* due to existing attributes: +# +# { +# "description": "one or more of the attributes in the request already exist: E1/T - [ A, B ], E2/T - [ A, B ]", +# "error": "Unprocessable" +# } +# +# * If in *at least one entity* in `entities` in *at least* one attribute there was a success but not all entities in `entities` have +# a full success (partial update): +# +# { +# "description": "one or more of the attributes in the request already exist: E2/T - [ A, B ]", +# "error": "PartialUpdate" +# } +# + +# +# 01. Create entity E1{A-B} +# 02. Create entity E2{A-B} +# 03. POST /v2/op/update appendStrict with entities E1{A,B}/E2{A,B}, see error (Unprocessable) +# 04. POST /v2/op/update appendStrict with entities E1{C,D}/E2{A,B}, see error (PartialUpdate) +# + + +echo "01. Create entity E1{A-B}" +echo "=========================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "02. Create entity E2{A-B}" +echo "=========================" +payload='{ + "id": "E2", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. POST /v2/op/update appendStrict with entities E1{A,B}/E2{A,B}, see error (Unprocessable)" +echo "============================================================================================" +payload='{ + "actionType": "appendStrict", + "entities": [ + { + "id": "E1", + "type": "T", + "A": { + "type": "Number", + "value": 1 + }, + "B": { + "type": "Number", + "value": 1 + } + }, + { + "id": "E2", + "type": "T", + "A": { + "type": "Number", + "value": 1 + }, + "B": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +echo "04. POST /v2/op/update appendStrict with entities E1{C,D}/E2{A,B}, see error (PartialUpdate)" +echo "============================================================================================" +payload='{ + "actionType": "appendStrict", + "entities": [ + { + "id": "E1", + "type": "T", + "C": { + "type": "Number", + "value": 1 + }, + "D": { + "type": "Number", + "value": 1 + } + }, + { + "id": "E2", + "type": "T", + "A": { + "type": "Number", + "value": 1 + }, + "B": { + "type": "Number", + "value": 1 + } + } + ] +}' +orionCurl --url /v2/op/update --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. Create entity E1{A-B} +========================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +02. Create entity E2{A-B} +========================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +03. POST /v2/op/update appendStrict with entities E1{A,B}/E2{A,B}, see error (Unprocessable) +============================================================================================ +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 134 + +{ + "description": "one or more of the attributes in the request already exist: E1/T - [ A, B ], E2/T - [ A, B ]", + "error": "Unprocessable" +} + + +04. POST /v2/op/update appendStrict with entities E1{C,D}/E2{A,B}, see error (PartialUpdate) +============================================================================================ +HTTP/1.1 422 Unprocessable Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 117 + +{ + "description": "one or more of the attributes in the request already exist: E2/T - [ A, B ]", + "error": "PartialUpdate" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file From b4baaac2b80c4369d4809f7c9e94f1bf34970e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 13 Dec 2023 14:22:18 +0100 Subject: [PATCH 033/390] FIX typos --- doc/manuals/orion-api.md | 2 -- src/lib/mongoBackend/mongoUpdateContext.cpp | 2 +- .../error_responses_09a_batch_update_replace.test | 2 +- .../error_responses_09b_batch_update_update.test | 2 +- .../error_responses_09c_batch_update_delete.test | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 967a185cc3..a524ca2eed 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -4701,8 +4701,6 @@ For action type `appendStrict`:` "error": "Unprocessable" } -* If *in some (but not in all)* entities in `entities` there was a fail due to existing attributes (partial update): - * If in *at least one entity* in `entities` in *at least* one attribute there was a success but not all entities in `entities` have a full success (partial update): diff --git a/src/lib/mongoBackend/mongoUpdateContext.cpp b/src/lib/mongoBackend/mongoUpdateContext.cpp index 4a19eaa863..a6e29ea78b 100644 --- a/src/lib/mongoBackend/mongoUpdateContext.cpp +++ b/src/lib/mongoBackend/mongoUpdateContext.cpp @@ -185,7 +185,7 @@ HttpStatusCode mongoUpdateContext break; case UC_FULL_ATTRS_FAIL: // If global UC is full attrs fail, we need also full attrs fail or not found entity in the processed entity to keep global full attrs fail - // Otherwise (success, partial, not found entity), the global UC changes to partial + // Otherwise (success, partial), the global UC changes to partial if ((entityUpdateCoverage != UC_FULL_ATTRS_FAIL) && (entityUpdateCoverage != UC_ENTITY_NOT_FOUND)) { updateCoverage = UC_PARTIAL; diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_09a_batch_update_replace.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_09a_batch_update_replace.test index afa86653a0..0bd2d948dd 100644 --- a/test/functionalTest/cases/0000_update_error_responses/error_responses_09a_batch_update_replace.test +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_09a_batch_update_replace.test @@ -35,7 +35,7 @@ brokerStart CB # # ... # * Errors use a non-2xx code and error payload (see next subsection): -# * 404 Not Found if none of the entitys in the `entities` field exists in `update`, `delete` or `replace` cases +# * 404 Not Found if none of the entities in the `entities` field exists in `update`, `delete` or `replace` cases # * 422 Unprocessable Content for other cases # # Response payload: diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_09b_batch_update_update.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_09b_batch_update_update.test index da4ba04e0e..65df37a7e9 100644 --- a/test/functionalTest/cases/0000_update_error_responses/error_responses_09b_batch_update_update.test +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_09b_batch_update_update.test @@ -35,7 +35,7 @@ brokerStart CB # # ... # * Errors use a non-2xx code and error payload (see next subsection): -# * 404 Not Found if none of the entitys in the `entities` field exists in `update`, `delete` or `replace` cases +# * 404 Not Found if none of the entities in the `entities` field exists in `update`, `delete` or `replace` cases # * 422 Unprocessable Content for other cases # # Response payload: diff --git a/test/functionalTest/cases/0000_update_error_responses/error_responses_09c_batch_update_delete.test b/test/functionalTest/cases/0000_update_error_responses/error_responses_09c_batch_update_delete.test index b7467902c5..4a99978e6b 100644 --- a/test/functionalTest/cases/0000_update_error_responses/error_responses_09c_batch_update_delete.test +++ b/test/functionalTest/cases/0000_update_error_responses/error_responses_09c_batch_update_delete.test @@ -35,7 +35,7 @@ brokerStart CB # # ... # * Errors use a non-2xx code and error payload (see next subsection): -# * 404 Not Found if none of the entitys in the `entities` field exists in `update`, `delete` or `replace` cases +# * 404 Not Found if none of the entities in the `entities` field exists in `update`, `delete` or `replace` cases # * 422 Unprocessable Content for other cases # # Response payload: From 5ff4b26e09528ced9ee7fe67e36cfb3933e5fad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 13 Dec 2023 14:36:51 +0100 Subject: [PATCH 034/390] FIX cleanup --- CHANGES_NEXT_RELEASE | 3 +++ src/lib/mongoBackend/MongoCommonUpdate.cpp | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index ef5fd3affb..6756a51fbd 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -6,6 +6,9 @@ - Fix: improve error traces (#4387) - Fix: on delete attributes operation, the lack of attribute in entity doesn't preclude the deletion of attributes which do exist (this way behaving same way as update attributes operation) - Fix: POST /v2/entities/E/attrs?options=append was wrongly creating entities +- Fix: provide more inforamtive error description in some error responses in update/delete operations +- Fix: proper use of "PartialUpdate" (instead of "Unprocessed") in responses in the case of partial updates/deletions (#3499) +- Fix: response 404 Not Found "NotFound" errors instead of 422 Unprocessable Content "Unprocessed" in the case of missing attribute in existing entity in attribute update operations - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) - Fix: improve logs in MongoDB query logic - Upgrade Debian version from 11.6 to 12.1 in Dockerfile diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index cea1c10471..273f613896 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -2448,7 +2448,6 @@ static bool updateContextAttributeItem " - offending attribute: " + targetAttr->getName(); cerP->statusCode.fill(SccInvalidParameter, details); - //oe->fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ATTRIBUTE, ERROR_NOT_FOUND); /* Although 'ca' has been already pushed into cerP, the pointer is still valid, of course */ ca->found = false; @@ -2613,12 +2612,9 @@ static bool deleteContextAttributeItem " - attribute not found"; cerP->statusCode.fill(SccInvalidParameter, details); - //oe->fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ATTRIBUTE, ERROR_NOT_FOUND); alarmMgr.badInput(clientIp, "attribute to be deleted is not found", targetAttr->getName()); ca->found = false; - - //return false; } return true; From a1cd4c5d1ebdc6d576324b07b800140957e5aa35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 13 Dec 2023 15:00:05 +0100 Subject: [PATCH 035/390] FIX simplify implementation --- src/lib/common/globals.h | 3 +-- src/lib/mongoBackend/MongoCommonUpdate.cpp | 4 ++-- src/lib/serviceRoutinesV2/postEntity.cpp | 6 +----- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/lib/common/globals.h b/src/lib/common/globals.h index 2d385d9092..66cae65f1c 100644 --- a/src/lib/common/globals.h +++ b/src/lib/common/globals.h @@ -149,8 +149,7 @@ typedef enum Ngsiv2Flavour { NGSIV2_NO_FLAVOUR = 0, NGSIV2_FLAVOUR_ONCREATE = 1, - NGSIV2_FLAVOUR_ONAPPEND = 2, - NGSIV2_FLAVOUR_ONPUREAPPEND = 3 + NGSIV2_FLAVOUR_ONAPPEND = 2 } Ngsiv2Flavour; diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 273f613896..9fb1f84c8f 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4184,8 +4184,8 @@ unsigned int processContextElement } // This is the case of POST /v2/entities/, in order to check that entity previously exist - // both for regular case (NGSIV2_FLAVOUR_ONAPPEND) and when ?options=append is used (NGSIV2_FLAVOUR_ONPUREAPPEND) - if ((entitiesNumber == 0) && ((ngsiv2Flavour == NGSIV2_FLAVOUR_ONAPPEND)||(ngsiv2Flavour == NGSIV2_FLAVOUR_ONPUREAPPEND))) + // both for regular case and when ?options=append is used + if ((entitiesNumber == 0) && (ngsiv2Flavour == NGSIV2_FLAVOUR_ONAPPEND)) { buildGeneralErrorResponse(eP, NULL, responseP, SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY); responseP->oe.fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ERROR_NOT_FOUND); diff --git a/src/lib/serviceRoutinesV2/postEntity.cpp b/src/lib/serviceRoutinesV2/postEntity.cpp index c81b095cc9..1403cd93ea 100644 --- a/src/lib/serviceRoutinesV2/postEntity.cpp +++ b/src/lib/serviceRoutinesV2/postEntity.cpp @@ -64,7 +64,6 @@ std::string postEntity { Entity* eP = &parseDataP->ent.res; ActionType op; - Ngsiv2Flavour flavor; eP->id = compV[2]; eP->type = ciP->uriParam["type"]; @@ -76,23 +75,20 @@ std::string postEntity return oe.toJson(); } - // FIXME PR: maybe NGSIV2_FLAVOUR_ONPUREAPPEND and NGSIV2_FLAVOUR_ONAPPEND can be merged into a single one if (ciP->uriParamOptions[OPT_APPEND] == true) // pure-append { op = ActionTypeAppendStrict; - flavor = NGSIV2_FLAVOUR_ONPUREAPPEND; } else { op = ActionTypeAppend; // append or update - flavor = NGSIV2_FLAVOUR_ONAPPEND; } // Fill in UpdateContextRequest parseDataP->upcr.res.fill(eP, op); // Call standard op postUpdateContext - postUpdateContext(ciP, components, compV, parseDataP, flavor); + postUpdateContext(ciP, components, compV, parseDataP, NGSIV2_FLAVOUR_ONAPPEND); // Any error in the response? std::string answer = ""; From 151e41ae43f66accd1dd684863ac942e361dfe2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 19 Dec 2023 16:17:40 +0100 Subject: [PATCH 036/390] FIX false positive in log deprecation logic when entity name (or other literal) includes the token v1 --- CHANGES_NEXT_RELEASE | 1 + src/lib/common/logTracing.cpp | 2 +- ...lse_positive_in_log_deprecation_logic.test | 98 +++++++++++++++++++ 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0c8dcd255b..f65bd57a35 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -6,6 +6,7 @@ - Fix: improve error traces (#4387) - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) - Fix: improve logs in MongoDB query logic +- Fix: false positive in log deprecation logic when entity name (or other literal) includes the token "v1" (#4454) - Upgrade Debian version from 11.6 to 12.1 in Dockerfile - Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3 - Reference MongoDB version changed from 4.4 to 6.0 diff --git a/src/lib/common/logTracing.cpp b/src/lib/common/logTracing.cpp index 26904d5628..41539e2907 100644 --- a/src/lib/common/logTracing.cpp +++ b/src/lib/common/logTracing.cpp @@ -37,7 +37,7 @@ */ inline bool isNgsiV1Url(const char* url) { - return (strstr(url, "v1") || strcasestr(url, "ngsi10") || strcasestr(url, "ngsi9")); + return (strstr(url, "/v1/") || strcasestr(url, "/ngsi10/") || strcasestr(url, "/ngsi9/")); } diff --git a/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test b/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test new file mode 100644 index 0000000000..8a6212d562 --- /dev/null +++ b/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test @@ -0,0 +1,98 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Disable NGSIv1 CLI + +--SHELL-INIT-- +dbInit CB +brokerStart CB 0 IPV4 -logDeprecate + +--SHELL-- + +# +# 01. GET /v2/entities/glory_ev1 +# 02. GET /v1/contextEntities/glory_ev2/attributes/A +# 03. Get WARNING trace in logs corresponding only to the second case +# + +echo "01. GET /v2/entities/glory_ev1" +echo "==============================" +orionCurl --url /v2/entities/glory_ev1 +echo +echo + + +echo "02. GET /v1/contextEntities/glory_ev2/attributes/A" +echo "==================================================" +orionCurl --url /v1/contextEntities/glory_ev2/attributes/A +echo +echo + + +echo "03. Get WARNING trace in logs corresponding only to the second case" +echo "===================================================================" +cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' +echo +echo + + +--REGEXPECT-- +01. GET /v2/entities/glory_ev1 +============================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +02. GET /v1/contextEntities/glory_ev2/attributes/A +================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 106 + +{ + "statusCode": { + "code": "404", + "details": "Entity id: /glory_ev2/", + "reasonPhrase": "No context element found" + } +} + + +03. Get WARNING trace in logs corresponding only to the second case +=================================================================== +Deprecated NGSIv1 request received: GET /v1/contextEntities/glory_ev2/attributes/A, response code: 200 + + +--TEARDOWN-- +brokerStop CB +dbDrop CB From 314abdf97b6eaa70174e0678a41c3adfa79eea47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 19 Dec 2023 17:11:21 +0100 Subject: [PATCH 037/390] ADD notification payload in INFO log traces --- CHANGES_NEXT_RELEASE | 1 + src/lib/common/logTracing.cpp | 91 +++++++++++++------ src/lib/common/logTracing.h | 9 +- src/lib/ngsiNotify/doNotify.cpp | 6 +- .../log_deprecate_warning.test | 2 +- .../3001_mqtt_alarms/mqtt_alarms_none.test | 9 +- .../mqtt_alarms_raise_and_release.test | 6 +- .../mqtt_alarms_raise_repeat_and_release.test | 6 +- .../log_notifications.test | 31 +++---- 9 files changed, 101 insertions(+), 60 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0c8dcd255b..d478714491 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,5 +1,6 @@ - Add: servicePath field to builtin attributes (#2877) - Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388) +- Add: notification payload in INFO log traces (#4449) - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) diff --git a/src/lib/common/logTracing.cpp b/src/lib/common/logTracing.cpp index 26904d5628..3ecd573bf8 100644 --- a/src/lib/common/logTracing.cpp +++ b/src/lib/common/logTracing.cpp @@ -42,6 +42,28 @@ inline bool isNgsiV1Url(const char* url) +/* **************************************************************************** +* +* truncatePayload - +* +* NOTE: this function allocated dynamic memory, be careful with memory leaks! +*/ +static char* truncatePayload(const char* payload) +{ + // +5 due to "(...)" + // +1 due to '\0' + unsigned int truncatedPayloadLengh = logInfoPayloadMaxSize + 5 + 1; + + char* truncatedPayload = (char*) malloc(logInfoPayloadMaxSize + 5 + 1); + strncpy(truncatedPayload, payload, logInfoPayloadMaxSize); + strncpy(truncatedPayload + logInfoPayloadMaxSize, "(...)", 5); + truncatedPayload[truncatedPayloadLengh - 1] = '\0'; + + return truncatedPayload; +} + + + /* **************************************************************************** * * logInfoNotification - rc as int @@ -52,12 +74,13 @@ void logInfoHttpNotification const char* endpoint, const char* verb, const char* resource, + const char* payload, int rc ) { char buffer[STRING_SIZE_FOR_INT]; snprintf(buffer, sizeof(buffer), "%d", rc); - logInfoHttpNotification(subId, endpoint, verb, resource, buffer); + logInfoHttpNotification(subId, endpoint, verb, resource, payload, buffer); } @@ -72,10 +95,29 @@ void logInfoHttpNotification const char* endpoint, const char* verb, const char* resource, + const char* payload, const char* rc ) { - LM_I(("Notif delivered (subId: %s): %s %s%s, response code: %s", subId, verb, endpoint, resource, rc)); + bool cleanAfterUse = false; + char* effectivePayload; + + if (strlen(payload) > logInfoPayloadMaxSize) + { + effectivePayload = truncatePayload(payload); + cleanAfterUse = true; + } + else + { + effectivePayload = (char*) payload; + } + + LM_I(("Notif delivered (subId: %s): %s %s%s, payload (%d bytes): %s, response code: %s", subId, verb, endpoint, resource, strlen(payload), effectivePayload, rc)); + + if (cleanAfterUse) + { + free(effectivePayload); + } } @@ -88,10 +130,29 @@ void logInfoMqttNotification ( const char* subId, const char* endpoint, - const char* resource + const char* resource, + const char* payload ) { - LM_I(("MQTT Notif delivered (subId: %s): broker: %s, topic: %s", subId, endpoint, resource)); + bool cleanAfterUse = false; + char* effectivePayload; + + if (strlen(payload) > logInfoPayloadMaxSize) + { + effectivePayload = truncatePayload(payload); + cleanAfterUse = true; + } + else + { + effectivePayload = (char*) payload; + } + + LM_I(("MQTT Notif delivered (subId: %s): broker: %s, topic: %s, payload (%d bytes): %s", subId, endpoint, resource, strlen(payload), effectivePayload)); + + if (cleanAfterUse) + { + free(effectivePayload); + } } @@ -117,28 +178,6 @@ void logInfoRequestWithoutPayload -/* **************************************************************************** -* -* truncatePayload - -* -* NOTE: this function allocated dynamic memory, be careful with memory leaks! -*/ -static char* truncatePayload(const char* payload) -{ - // +5 due to "(...)" - // +1 due to '\0' - unsigned int truncatedPayloadLengh = logInfoPayloadMaxSize + 5 + 1; - - char* truncatedPayload = (char*) malloc(logInfoPayloadMaxSize + 5 + 1); - strncpy(truncatedPayload, payload, logInfoPayloadMaxSize); - strncpy(truncatedPayload + logInfoPayloadMaxSize, "(...)", 5); - truncatedPayload[truncatedPayloadLengh - 1] = '\0'; - - return truncatedPayload; -} - - - /* **************************************************************************** * * logInfoRequestWithPayload - diff --git a/src/lib/common/logTracing.h b/src/lib/common/logTracing.h index 4e3c7aac8c..939e2beb66 100644 --- a/src/lib/common/logTracing.h +++ b/src/lib/common/logTracing.h @@ -32,7 +32,7 @@ /* **************************************************************************** * -* logInfoNotification - rc as int +* logInfoHttpNotification - rc as int */ extern void logInfoHttpNotification ( @@ -40,6 +40,7 @@ extern void logInfoHttpNotification const char* endpoint, const char* verb, const char* resource, + const char* payload, int rc ); @@ -47,7 +48,7 @@ extern void logInfoHttpNotification /* **************************************************************************** * -* logInfoNotification - rc as string +* logInfoHttpNotification - rc as string */ extern void logInfoHttpNotification ( @@ -55,6 +56,7 @@ extern void logInfoHttpNotification const char* endpoint, const char* verb, const char* resource, + const char* payload, const char* rc ); @@ -68,7 +70,8 @@ extern void logInfoMqttNotification ( const char* subId, const char* endpoint, - const char* resource + const char* resource, + const char* payload ); diff --git a/src/lib/ngsiNotify/doNotify.cpp b/src/lib/ngsiNotify/doNotify.cpp index 7598643047..25f25b384f 100644 --- a/src/lib/ngsiNotify/doNotify.cpp +++ b/src/lib/ngsiNotify/doNotify.cpp @@ -127,11 +127,11 @@ static void doNotifyHttp(SenderThreadParams* params, CURL* curl, SyncQOverflowsubscriptionId.c_str(), endpoint.c_str(), params->verb.c_str(), params->resource.c_str(), statusCode); + logInfoHttpNotification(params->subscriptionId.c_str(), endpoint.c_str(), params->verb.c_str(), params->resource.c_str(), params->content.c_str(), statusCode); } else { - logInfoHttpNotification(params->subscriptionId.c_str(), endpoint.c_str(), params->verb.c_str(), params->resource.c_str(), out.c_str()); + logInfoHttpNotification(params->subscriptionId.c_str(), endpoint.c_str(), params->verb.c_str(), params->resource.c_str(), params->content.c_str(), out.c_str()); } } @@ -164,7 +164,7 @@ static void doNotifyMqtt(SenderThreadParams* params) // mqttOnPublishCallback is called (by the moment we are not doing nothing there, just printing in // DEBUG log level). Note however that even if mqttOnPublishCallback() is called there is no actual // guarantee if MQTT QoS is 0 - logInfoMqttNotification(params->subscriptionId.c_str(), endpoint.c_str(), params->resource.c_str()); + logInfoMqttNotification(params->subscriptionId.c_str(), endpoint.c_str(), params->resource.c_str(), params->content.c_str()); subNotificationErrorStatus(params->tenant, params->subscriptionId, false, -1, ""); } else diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index baf5d43988..a0a0704263 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -31,7 +31,7 @@ brokerStart CB 0 IPV4 -logDeprecate # # 01. Query E1-T1 -# 02. GET /v1/contextEntities/E +# 02. GET /v1/contextEntities/E/attributes/A # 03. Create entity using NGSIv1 metadata location # 04. Create entity using NGSIv1 and geo:point # 05. Create entity using NGSIv2 and geo:point diff --git a/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_none.test b/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_none.test index 00ad02a854..35ee20c7c3 100644 --- a/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_none.test +++ b/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_none.test @@ -61,6 +61,7 @@ orionCurl --url /v2/subscriptions --payload "$payload" echo echo +SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") echo "02. Upsert three times to trigger three MQTT notifications" echo "==========================================================" @@ -85,7 +86,7 @@ echo echo "03. Check logs and see three MQTT notifications and no alarms" echo "=============================================================" -cat /tmp/contextBroker.log | grep -v 'corr=N/A' | awk -F 'msg=' '{print $2}' +cat /tmp/contextBroker.log | grep -v 'corr=N/A' | awk -F 'msg=' '{print $2}' | sed -e "s/$SUB_ID/SUB_ID/g" echo echo @@ -129,11 +130,11 @@ Location: /v2/entities/E?type=T #SORT_START Request received: POST /v2/subscriptions, request payload (142 bytes): { "subject": { "entities": [ { "id": "E", "type": "T" } ] }, "notification": { "mqtt": { "url": "mqtt://localhost:1883", "topic": "sub1" } } }, response code: 201 Request received: POST /v2/entities?options=forcedUpdate,upsert, request payload (64 bytes): { "id": "E", "type": "T", "A": { "value": 1, "type": "Float" } }, response code: 204 -MQTT Notif delivered REGEX(.*): broker: localhost:1883, topic: sub1 +MQTT Notif delivered (subId: SUB_ID): broker: localhost:1883, topic: sub1, payload (121 bytes): {"subscriptionId":"SUB_ID","data":[{"id":"E","type":"T","A":{"type":"Float","value":1,"metadata":{}}}]} Request received: POST /v2/entities?options=forcedUpdate,upsert, request payload (64 bytes): { "id": "E", "type": "T", "A": { "value": 1, "type": "Float" } }, response code: 204 -MQTT Notif delivered REGEX(.*): broker: localhost:1883, topic: sub1 +MQTT Notif delivered (subId: SUB_ID): broker: localhost:1883, topic: sub1, payload (121 bytes): {"subscriptionId":"SUB_ID","data":[{"id":"E","type":"T","A":{"type":"Float","value":1,"metadata":{}}}]} Request received: POST /v2/entities?options=forcedUpdate,upsert, request payload (64 bytes): { "id": "E", "type": "T", "A": { "value": 1, "type": "Float" } }, response code: 204 -MQTT Notif delivered REGEX(.*): broker: localhost:1883, topic: sub1 +MQTT Notif delivered (subId: SUB_ID): broker: localhost:1883, topic: sub1, payload (121 bytes): {"subscriptionId":"SUB_ID","data":[{"id":"E","type":"T","A":{"type":"Float","value":1,"metadata":{}}}]} #SORT_END diff --git a/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_raise_and_release.test b/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_raise_and_release.test index 0877eb5d53..f531b4247f 100644 --- a/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_raise_and_release.test +++ b/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_raise_and_release.test @@ -162,7 +162,7 @@ echo echo "09. Check logs and see 1 raised alarm and 1 released alarm" echo "==========================================================" -cat /tmp/contextBroker.log | grep -v 'corr=N/A' | awk -F 'msg=' '{print $2}' +cat /tmp/contextBroker.log | grep -v 'corr=N/A' | awk -F 'msg=' '{print $2}' | sed -e "s/$SUB_ID/SUB_ID/g" echo echo @@ -360,10 +360,10 @@ Raising alarm MqttConnectionError mqtt.flespi.io:1883: Connection Refused: bad u Request received: GET /v2/subscriptions, response code: 200 Request received: POST /v2/entities?options=forcedUpdate,upsert, request payload (64 bytes): { "id": "E", "type": "T", "A": { "value": 1, "type": "Float" } }, response code: 204 Request received: GET /v2/subscriptions, response code: 200 -Request received: PATCH /v2/subscriptions/REGEX(.*): { "notification": { "mqtt": { "url": "mqtt://mqtt.flespi.io:1883", "user": "SeY7oD5XPa1UENBiOLPHqWXmj4r4OZHu4tsgWn1AmTkQuMW6lCDCmqMvi1oURVfJ", "passwd": "xxxx", "topic": "sub1" } } }, response code: 204 +Request received: PATCH /v2/subscriptions/SUB_ID, request payload (182 bytes): { "notification": { "mqtt": { "url": "mqtt://mqtt.flespi.io:1883", "user": "SeY7oD5XPa1UENBiOLPHqWXmj4r4OZHu4tsgWn1AmTkQuMW6lCDCmqMvi1oURVfJ", "passwd": "xxxx", "topic": "sub1" } } }, response code: 204 Request received: POST /v2/entities?options=forcedUpdate,upsert, request payload (64 bytes): { "id": "E", "type": "T", "A": { "value": 1, "type": "Float" } }, response code: 204 Releasing alarm MqttConnectionError mqtt.flespi.io:1883 -MQTT Notif delivered REGEX(.*): broker: mqtt.flespi.io:1883, topic: sub1 +MQTT Notif delivered (subId: SUB_ID): broker: mqtt.flespi.io:1883, topic: sub1, payload (121 bytes): {"subscriptionId":"SUB_ID","data":[{"id":"E","type":"T","A":{"type":"Float","value":1,"metadata":{}}}]} Request received: GET /v2/subscriptions, response code: 200 #SORT_END diff --git a/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_raise_repeat_and_release.test b/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_raise_repeat_and_release.test index caf1d9af57..c60bf280d5 100644 --- a/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_raise_repeat_and_release.test +++ b/test/functionalTest/cases/3001_mqtt_alarms/mqtt_alarms_raise_repeat_and_release.test @@ -162,7 +162,7 @@ echo echo "09. Check logs and see 1 raised alarm, 1 repeat alarm and 1 released alarm" echo "==========================================================================" -cat /tmp/contextBroker.log | grep -v 'corr=N/A' | awk -F 'msg=' '{print $2}' +cat /tmp/contextBroker.log | grep -v 'corr=N/A' | awk -F 'msg=' '{print $2}' | sed -e "s/$SUB_ID/SUB_ID/g" echo echo @@ -361,10 +361,10 @@ Request received: GET /v2/subscriptions, response code: 200 Request received: POST /v2/entities?options=forcedUpdate,upsert, request payload (64 bytes): { "id": "E", "type": "T", "A": { "value": 1, "type": "Float" } }, response code: 204 Request received: GET /v2/subscriptions, response code: 200 Repeated MqttConnectionError mqtt.flespi.io:1883: Connection Refused: bad user name or password. -Request received: PATCH /v2/subscriptions/REGEX(.*): { "notification": { "mqtt": { "url": "mqtt://mqtt.flespi.io:1883", "user": "SeY7oD5XPa1UENBiOLPHqWXmj4r4OZHu4tsgWn1AmTkQuMW6lCDCmqMvi1oURVfJ", "passwd": "xxxx", "topic": "sub1" } } }, response code: 204 +Request received: PATCH /v2/subscriptions/SUB_ID, request payload (182 bytes): { "notification": { "mqtt": { "url": "mqtt://mqtt.flespi.io:1883", "user": "SeY7oD5XPa1UENBiOLPHqWXmj4r4OZHu4tsgWn1AmTkQuMW6lCDCmqMvi1oURVfJ", "passwd": "xxxx", "topic": "sub1" } } }, response code: 204 Request received: POST /v2/entities?options=forcedUpdate,upsert, request payload (64 bytes): { "id": "E", "type": "T", "A": { "value": 1, "type": "Float" } }, response code: 204 Releasing alarm MqttConnectionError mqtt.flespi.io:1883 -MQTT Notif delivered REGEX(.*): broker: mqtt.flespi.io:1883, topic: sub1 +MQTT Notif delivered (subId: SUB_ID): broker: mqtt.flespi.io:1883, topic: sub1, payload (121 bytes): {"subscriptionId":"SUB_ID","data":[{"id":"E","type":"T","A":{"type":"Float","value":1,"metadata":{}}}]} Request received: GET /v2/subscriptions, response code: 200 #SORT_END diff --git a/test/functionalTest/cases/3694_logs_improvements/log_notifications.test b/test/functionalTest/cases/3694_logs_improvements/log_notifications.test index 8483b3f495..81a922c5bc 100644 --- a/test/functionalTest/cases/3694_logs_improvements/log_notifications.test +++ b/test/functionalTest/cases/3694_logs_improvements/log_notifications.test @@ -35,7 +35,7 @@ accumulatorStart --pretty-print localhost $LISTENER_PORT # 02. Create subscription SUB2 for E (to nowhere) # 03. Create E entity (triggering 2 notifications) # 04. POST /v2/op/update updating twice E (triggering 4 notifications) -# 05. Check notification logs (sorted by url) +# 05. Check notification logs (sorted by sub id) # 06. Check notification correlators (two blocks, corresponding to each update) # @@ -61,6 +61,7 @@ orionCurl --url /v2/subscriptions --payload "$payload" echo echo +SUB_ID1=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") echo "02. Create subscription SUB2 for E (to nowhere)" echo "===============================================" @@ -83,6 +84,7 @@ orionCurl --url /v2/subscriptions --payload "$payload" echo echo +SUB_ID2=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") echo "03. Create E entity (triggering 2 notifications)" echo "================================================" @@ -130,14 +132,9 @@ echo echo -# Example log trace for notifications (to help understanding the awk/cut stuff :) -# -# msg=Notif delivered (subId: 5f904fb3e970f217bde75f98): POST localhost:9997/notify, response code: 200 - - -echo "05. Check notification logs (sorted by url)" -echo "===========================================" -cat /tmp/contextBroker.log | grep INFO | grep 'Notif' | awk -F '|' '{print $10}' | cut -c 57- | sort +echo "05. Check notification logs (sorted by sub id)" +echo "==============================================" +cat /tmp/contextBroker.log | grep INFO | grep 'Notif' | awk -F '|' '{print $10}' | sed -e "s/$SUB_ID1/SUB_ID1/g" | sed -e "s/$SUB_ID2/SUB_ID2/g" | sort echo echo @@ -188,14 +185,14 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -05. Check notification logs (sorted by url) -=========================================== -POST localhost:12345/notify, response code: Couldn't connect to server -POST localhost:12345/notify, response code: Couldn't connect to server -POST localhost:12345/notify, response code: Couldn't connect to server -POST localhost:9997/notify, response code: 200 -POST localhost:9997/notify, response code: 200 -POST localhost:9997/notify, response code: 200 +05. Check notification logs (sorted by sub id) +============================================== + msg=Notif delivered (subId: SUB_ID1): POST localhost:9997/notify, payload (125 bytes): {"subscriptionId":"SUB_ID1","data":[{"id":"E","type":"T","A1":{"type":"Text","value":"foo","metadata":{}}}]}, response code: 200 + msg=Notif delivered (subId: SUB_ID1): POST localhost:9997/notify, payload (173 bytes): {"subscriptionId":"SUB_ID1","data":[{"id":"E","type":"T","A":{"type":"Text","value":"foo","metadata":{}},"A1":{"type":"Text","value":"foo","metadata":{}}}]}, response code: 200 + msg=Notif delivered (subId: SUB_ID1): POST localhost:9997/notify, payload (173 bytes): {"subscriptionId":"SUB_ID1","data":[{"id":"E","type":"T","A1":{"type":"Text","value":"foo","metadata":{}},"A":{"type":"Text","value":"foo","metadata":{}}}]}, response code: 200 + msg=Notif delivered (subId: SUB_ID2): POST localhost:12345/notify, payload (125 bytes): {"subscriptionId":"SUB_ID2","data":[{"id":"E","type":"T","A1":{"type":"Text","value":"foo","metadata":{}}}]}, response code: Couldn't connect to server + msg=Notif delivered (subId: SUB_ID2): POST localhost:12345/notify, payload (173 bytes): {"subscriptionId":"SUB_ID2","data":[{"id":"E","type":"T","A":{"type":"Text","value":"foo","metadata":{}},"A1":{"type":"Text","value":"foo","metadata":{}}}]}, response code: Couldn't connect to server + msg=Notif delivered (subId: SUB_ID2): POST localhost:12345/notify, payload (173 bytes): {"subscriptionId":"SUB_ID2","data":[{"id":"E","type":"T","A1":{"type":"Text","value":"foo","metadata":{}},"A":{"type":"Text","value":"foo","metadata":{}}}]}, response code: Couldn't connect to server 06. Check notification correlators (two blocks, corresponding to each update) From ca088da5c962f9568bb357168e7234d9925049b6 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Wed, 20 Dec 2023 20:55:08 +0900 Subject: [PATCH 038/390] Fix typo --- doc/manuals/admin/build_source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index b534e0b2db..64064030e4 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -107,7 +107,7 @@ The Orion Context Broker comes with a suite of unit, valgrind and end-to-end tes In the case of the aarch64 architecture, install libxslt using apt-get, and run `./configure` with `--build=arm-linux` option. -* Install MongoDB (tests rely on mongod running in localhost). Check [the official MongoDB documentation](hhttps://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-debian/) for details. Recommended version is 6.0 (it may work with previous versions, but we don't recommend it). +* Install MongoDB (tests rely on mongod running in localhost). Check [the official MongoDB documentation](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-debian/) for details. Recommended version is 6.0 (it may work with previous versions, but we don't recommend it). * Run unit test From 1125baea1761406f4bc5ff127e476f048795101f Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Wed, 20 Dec 2023 21:00:59 +0900 Subject: [PATCH 039/390] (JP) Add documentation about MongoDB 6.0 (#4332) --- doc/manuals.jp/admin/build_source.md | 8 +------- doc/manuals.jp/admin/install.md | 4 ++-- doc/manuals.jp/admin/perf_tuning.md | 2 +- docker/README.jp.md | 6 +++--- docker/docker_swarm.jp.md | 2 +- docker/raspberry_pi.jp.md | 2 +- 6 files changed, 9 insertions(+), 15 deletions(-) diff --git a/doc/manuals.jp/admin/build_source.md b/doc/manuals.jp/admin/build_source.md index 8b6936c127..2797e3f5b3 100644 --- a/doc/manuals.jp/admin/build_source.md +++ b/doc/manuals.jp/admin/build_source.md @@ -108,13 +108,7 @@ Orion Context Broker には、次の手順に従って実行できる一連の aarch64 アーキテクチャの場合、apt-get を使用して libxslt をインストールし、`--build=arm-linux` オプションを指定して `/configure` を実行します。 -* MongoDB をインストールします (テストはローカル・ホストで実行されている mongod に依存します)。詳細については、[MongoDB の公式ドキュメント](hhttps://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-debian/) を確認してください。推奨バージョンは 4.4 です (以前のバージョンでも動作する可能性がありますが、お勧めしません)。 - * mongo レガシー・シェル (`mongo` コマンド) は MongoDB 5 で非推奨となり、MongoDB 6 では新しいシェル (`mongosh` コマンド) が優先されて削除されたことに注意してください。一部の機能テスト (ftest) は、`mongosh` ではなく `mongo` を使用しているため、MongoDB 6 以降を使用している場合、これが原因で失敗します。 - * Debian 12 は libssl3 に移行しましたが、一部の MongoDB バージョンでは libssl1 が必要な場合があります。`Depends: libssl1.1 (>= 1.1.1) but it is not installable` エラーが発生した場合は、次のことをテストできます ([こちら](https://askubuntu.com/a/1421959) を参照)) - - wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb - sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb - rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb # optional, for cleanness +* MongoDB をインストールします (テストはローカルホストで実行されている mongod に依存します)。 詳細については、[MongoDB の公式ドキュメント](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-debian/)を確認してください。推奨バージョンは 6.0 です (以前のバージョンでも動作する可能性がありますが、お勧めしません)。 * ユニット・テストを実行します diff --git a/doc/manuals.jp/admin/install.md b/doc/manuals.jp/admin/install.md index cd73b7ec67..eee8edb8f6 100644 --- a/doc/manuals.jp/admin/install.md +++ b/doc/manuals.jp/admin/install.md @@ -29,10 +29,10 @@ Docker hub で公式の Orion docker コンテナを使用してインストー * オペレーティングシステム: Debian。リファレンス・オペレーティングシステムは Debian 12.1 ですが、それ以降の Debian 12 バージョンでも動作するはずです * データベース: MongoDB は、Orion Context Broker がインストールされるのと同じホストで実行するか、ネットワーク経由で - アクセスできる別のホストで実行する必要があります。推奨される MongoDB バージョンは 4.4 です (Orion は古いバージョンで + アクセスできる別のホストで実行する必要があります。推奨される MongoDB バージョンは 6.0 です (Orion は古いバージョンで 動作する可能性がありますが、まったくお勧めしません!) -システムリソース (CPU, RAMなど) については、[これらの推奨事項](diagnosis.md＃resource-availability) を参照してください。 +システムリソース (CPU, RAMなど) については、[これらの推奨事項](diagnosis.md#resource-availability) を参照してください。 ## 以前のバージョンからのアップグレード diff --git a/doc/manuals.jp/admin/perf_tuning.md b/doc/manuals.jp/admin/perf_tuning.md index 0ee6e55d15..bd5197dcf5 100644 --- a/doc/manuals.jp/admin/perf_tuning.md +++ b/doc/manuals.jp/admin/perf_tuning.md @@ -21,7 +21,7 @@ ## MongoDB の設定 -パフォーマンスの観点から、特に Update-Intensive シナリオでは、WireTiger で MongoDB 4.4 を使用することをお勧めします。 +パフォーマンスの観点から、特に Update-Intensive シナリオでは、WireTiger で MongoDB 6.0 を使用することをお勧めします。 さらに、パフォーマンスに影響を与える可能性があるため、公式の MongoDB のドキュメントから次の情報を考慮してください : diff --git a/docker/README.jp.md b/docker/README.jp.md index 72f32ac440..c36ae73140 100644 --- a/docker/README.jp.md +++ b/docker/README.jp.md @@ -36,7 +36,7 @@ Orion Context Broker を試してみたいし、データベースについて command: -dbhost mongo mongo: - image: mongo:4.4 + image: mongo:6.0 command: --nojournal 3. コマンドラインを使用して作成したディレクトリで、`sudo docker-compose up` を実行します @@ -75,7 +75,7 @@ Orion Context Broker を試してみたいし、データベースについて ### 2B. MongoDB が別の Docker コンテナで動作している場合 他のコンテナで MongoDB を実行したい場合は、次のように起動することができます - sudo docker run --name mongodb -d mongo:4.4 + sudo docker run --name mongodb -d mongo:6.0 そして、このコマンドで Orion を実行します @@ -109,7 +109,7 @@ Orion Context Broker を試してみたいし、データベースについて 4. Orion を実行 ... * docker-compose で自動化されたシナリオを使用し、新しいイメージを構築する : `sudo docker-compose up`。必要に応じて、提供されている `docker-compose.yml` ファイルを変更することもできます * 手動で MongoDB を別のコンテナで実行します : - 1. `sudo docker run --name mongodb -d mongo:4.4` + 1. `sudo docker run --name mongodb -d mongo:6.0` 2. `sudo docker build -t orion .` 3. `sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 orion -dbhost mongodb`. * 手動で MongoDB ホストを見つける場所を指定します : diff --git a/docker/docker_swarm.jp.md b/docker/docker_swarm.jp.md index 9ef927b9d5..6b35f7920d 100644 --- a/docker/docker_swarm.jp.md +++ b/docker/docker_swarm.jp.md @@ -77,7 +77,7 @@ MongoDB ReplicaSet を Docker Swarm にデプロイする方法の詳細につ services: mongo: - image: mongo:4.4 + image: mongo:6.0 entrypoint: [ "/usr/bin/mongod", "--replSet", "rs", "--journal", "--smallfiles", "--bind_ip", "0.0.0.0"] volumes: - mongodata:/data/db diff --git a/docker/raspberry_pi.jp.md b/docker/raspberry_pi.jp.md index 2769483c76..6a2e961cc7 100644 --- a/docker/raspberry_pi.jp.md +++ b/docker/raspberry_pi.jp.md @@ -50,7 +50,7 @@ services: command: -dbhost mongo mongo: - image: mongo:4.4 + image: mongo:6.0 command: --nojournal ``` From 39aa16208648f433f88c6d6ed77b26dcd89c679a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 20 Dec 2023 14:07:15 +0100 Subject: [PATCH 040/390] FIX typos in ftest names --- .../log_deprecate_warning_dynamic.test | 2 +- .../false_positive_in_log_deprecation_logic.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test index b7f1dfa95d..a71f1a27e1 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -Disable NGSIv1 CLI with dynamic changes done by management REST API +Log deprecate warnings with dynamic changes done by management REST API --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test b/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test index 8a6212d562..1ee335d350 100644 --- a/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test +++ b/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -Disable NGSIv1 CLI +False positive in log deprecation --SHELL-INIT-- dbInit CB From 8f9fac0d3be4c9a414b6873f52f1e063052a0399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 20 Dec 2023 14:38:42 +0100 Subject: [PATCH 041/390] FIX additional typos --- .../log_deprecate_warning_dynamic.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test index a71f1a27e1..df0025e539 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test @@ -32,15 +32,15 @@ brokerStart CB 0 IPV4 -logDeprecate # # 00. Check deprecate is true # 01. Query E1-T1 -# 02. GET /v1/contextEntities/E +# 02. GET /v1/contextEntities/E/attributes/A # 03. Disable logDeprecated by API # 04. Check deprecate is false # 05. Query E1-T1 (not logged) -# 06. GET /v1/contextEntities/E (not logged) +# 06. GET /v1/contextEntities/E/attributes/A (not logged) # 07. Enable logDeprecated by API # 08. Check deprecate is false # 09. Query E1-T1 -# 10. GET /v1/contextEntities/E +# 10. GET /v1/contextEntities/E/attributes/A # 11. Get WARNING trace in logs (see 4 traces) # From eb230405d3724ca33e9ec146580a4d29d3ad3bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 20 Dec 2023 17:15:26 +0100 Subject: [PATCH 042/390] FIX additional ftest for 4449 case --- .../notification_payload_in_logs.test | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 test/functionalTest/cases/4449_notification_payload_in_logs/notification_payload_in_logs.test diff --git a/test/functionalTest/cases/4449_notification_payload_in_logs/notification_payload_in_logs.test b/test/functionalTest/cases/4449_notification_payload_in_logs/notification_payload_in_logs.test new file mode 100644 index 0000000000..d9f7f692d3 --- /dev/null +++ b/test/functionalTest/cases/4449_notification_payload_in_logs/notification_payload_in_logs.test @@ -0,0 +1,231 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Notification payload included in logs in several subscriptions types + +--SHELL-INIT-- +dbInit CB +brokerStart CB 0-255 IPV4 -multiservice +accumulatorStart --pretty-print localhost $LISTENER_PORT + +--SHELL-- + +# +# 01. Create subscription SUB1 (normalized) +# 02. Create subscription SUB2 (simplifiedKeyValues) +# 03. Create subscription SUB3 (custom text) +# 04. Create subscription SUB4 (empty payload) +# 05. Create E entity (triggering 4 notifications) +# 06. Check notification logs (sorted by sub id) +# + + +echo "01. Create subscription SUB1 (normalized)" +echo "=========================================" +payload=' +{ + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": {"url": "http://localhost:'$LISTENER_PORT'/notify"} + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID1=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "02. Create subscription SUB2 (simplifiedKeyValues)" +echo "==================================================" +payload=' +{ + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": {"url": "http://localhost:'$LISTENER_PORT'/notify"}, + "attrsFormat": "simplifiedKeyValues" + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID2=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "03. Create subscription SUB3 (custom text)" +echo "==========================================" +payload=' +{ + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://localhost:'$LISTENER_PORT'/notify", + "payload": "this is ${A}" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID3=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "04. Create subscription SUB4 (empty payload)" +echo "============================================" +payload=' +{ + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://localhost:'$LISTENER_PORT'/notify", + "payload": null + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID4=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "05. Create E entity (triggering 4 notifications)" +echo "================================================" +payload=' +{ + "id":"E", + "type":"T", + "A":{ + "value": 42, + "type": "Number" + } +}' +orionCurl --url '/v2/entities' --payload "$payload" +echo +echo + + +echo "06. Check notification logs (sorted by sub id)" +echo "==============================================" +cat /tmp/contextBroker.log | grep INFO | grep 'Notif' | awk -F '|' '{print $10}' | sed -e "s/$SUB_ID1/SUB_ID1/g" | sed -e "s/$SUB_ID2/SUB_ID2/g" | sed -e "s/$SUB_ID3/SUB_ID3/g" | sed -e "s/$SUB_ID4/SUB_ID4/g" | sort +echo +echo + + +--REGEXPECT-- +01. Create subscription SUB1 (normalized) +========================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f\-]{24}) +Content-Length: 0 + + + +02. Create subscription SUB2 (simplifiedKeyValues) +================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f\-]{24}) +Content-Length: 0 + + + +03. Create subscription SUB3 (custom text) +========================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f\-]{24}) +Content-Length: 0 + + + +04. Create subscription SUB4 (empty payload) +============================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f\-]{24}) +Content-Length: 0 + + + +05. Create E entity (triggering 4 notifications) +================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +06. Check notification logs (sorted by sub id) +============================================== + msg=Notif delivered (subId: SUB_ID1): POST localhost:9997/notify, payload (123 bytes): {"subscriptionId":"SUB_ID1","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 200 + msg=Notif delivered (subId: SUB_ID2): POST localhost:9997/notify, payload (28 bytes): {"id":"E","type":"T","A":42}, response code: 200 + msg=Notif delivered (subId: SUB_ID3): POST localhost:9997/notify, payload (10 bytes): this is 42, response code: 200 + msg=Notif delivered (subId: SUB_ID4): POST localhost:9997/notify, payload (0 bytes): , response code: 200 + + +--TEARDOWN-- +accumulatorStop +brokerStop CB +dbDrop CB From 2d032e35d9775559acf89735feba0400091743bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 20 Dec 2023 17:19:52 +0100 Subject: [PATCH 043/390] FIX logs improvements --- doc/manuals/admin/logs.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index 192e835dbe..cffa9bb934 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -170,13 +170,13 @@ time=2020-10-26T10:32:41.724Z | lvl=INFO | corr=93bdc5b4-1776-11eb-954d-000c29df different value for the correlator. ``` -time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f914177334436ea590f6edb): POST localhost:1028/accumulate, response code: 200 +time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f914177334436ea590f6edb): POST localhost:1028/accumulate, payload (123 bytes): {"subscriptionId":"5f914177334436ea590f6edb","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 200 ``` * In the case of MQTT notifications the `msg` field of the above trace is slightly different: ``` -time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoMqttNotification | msg=MQTT Notif delivered (subId: 60ffea6c1bca454f9a64c96c): broker: localhost:1883, topic: sub2 +time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoMqttNotification | msg=MQTT Notif delivered (subId: 60ffea6c1bca454f9a64c96c): broker: localhost:1883, topic: sub2, payload (121 bytes): {"subscriptionId":"60ffea6c1bca454f9a64c96c","data":[{"id":"E","type":"T","A":{"type":"Float","value":1,"metadata":{}}}]} ``` * For each forwarded request to a [Context Provider](../user/context_providers.md) (either queries or updates), @@ -200,7 +200,7 @@ Some additional considerations: occurs. For instance: ``` -time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f914177334436ea590f6edb): POST localhost:1028/accumulate, response code: Couldn't connect to server +time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f914177334436ea590f6edb): POST localhost:1028/accumulate, payload (123 bytes): {"subscriptionId":"5f914177334436ea590f6edb","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Couldn't connect to server ``` * When a client request triggers forwarding to Context Providers, a `Starting forwarding for ` @@ -346,56 +346,56 @@ contextBroker -fg -httpTimeout 10000 -logLevel INFO -notificationMode threadpool Successful sent (response code 200): ``` -time=2020-10-26T14:48:37.192Z | lvl=INFO | corr=54393a44-179a-11eb-bb87-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000006 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e174b14e7532482ac794): POST localhost:1028/accumulate, response code: 200 +time=2020-10-26T14:48:37.192Z | lvl=INFO | corr=54393a44-179a-11eb-bb87-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000006 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e174b14e7532482ac794): POST localhost:1028/accumulate, payload (123 bytes): {"subscriptionId":"5f96e174b14e7532482ac794","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 200 ``` Notification endpoint response with 400 (a WARN trace is printed): ``` time=2020-10-26T14:49:34.619Z | lvl=WARN | corr=7689f6ba-179a-11eb-ac4c-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000009 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=httpRequestSend.cpp[583]:httpRequestSend | msg=Notification (subId: 5f96e1fdb14e7532482ac795) response NOT OK, http code: 400 -time=2020-10-26T14:49:34.619Z | lvl=INFO | corr=7689f6ba-179a-11eb-ac4c-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000009 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e1fdb14e7532482ac795): POST localhost:1028/giveme400, response code: 400 +time=2020-10-26T14:49:34.619Z | lvl=INFO | corr=7689f6ba-179a-11eb-ac4c-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000009 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e1fdb14e7532482ac795): POST localhost:1028/giveme400, payload (123 bytes): {"subscriptionId":"5f96e1fdb14e7532482ac795","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 400 ``` Notification endpoint response with 404 (a WARN trace is printed): ``` time=2020-10-26T14:51:40.764Z | lvl=WARN | corr=c1b8e9c0-179a-11eb-9edc-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000012 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=httpRequestSend.cpp[583]:httpRequestSend | msg=Notification (subId: 5f96e27cb14e7532482ac796) response NOT OK, http code: 404 -time=2020-10-26T14:51:40.764Z | lvl=INFO | corr=c1b8e9c0-179a-11eb-9edc-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000012 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e27cb14e7532482ac796): POST localhost:1028/giveme404, response code: 404 +time=2020-10-26T14:51:40.764Z | lvl=INFO | corr=c1b8e9c0-179a-11eb-9edc-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000012 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e27cb14e7532482ac796): POST localhost:1028/giveme404, payload (123 bytes): {"subscriptionId":"5f96e27cb14e7532482ac796","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 404 ``` Notification endpoint response with 500 (a WARN trace is printed) ``` time=2020-10-26T14:53:04.246Z | lvl=WARN | corr=f37b5024-179a-11eb-9ce6-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000015 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=httpRequestSend.cpp[583]:httpRequestSend | msg=Notification (subId: 5f96e2cfb14e7532482ac797) response NOT OK, http code: 500 -time=2020-10-26T14:53:04.247Z | lvl=INFO | corr=f37b5024-179a-11eb-9ce6-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000015 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e2cfb14e7532482ac797): POST localhost:1028/giveme500, response code: 500 +time=2020-10-26T14:53:04.247Z | lvl=INFO | corr=f37b5024-179a-11eb-9ce6-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000015 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e2cfb14e7532482ac797): POST localhost:1028/giveme500, payload (123 bytes): {"subscriptionId":"5f96e2cfb14e7532482ac797","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 500 ``` Endpoint not responding within 10 seconds timeout or some other connection error (alarm is raised in WARN level): ``` time=2020-10-26T14:54:15.996Z | lvl=WARN | corr=184b8b80-179b-11eb-9c52-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000018 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=AlarmManager.cpp[328]:notificationError | msg=Raising alarm NotificationError localhost:1028/givemeDelay: notification failure for queue worker: Timeout was reached -time=2020-10-26T14:54:15.996Z | lvl=INFO | corr=184b8b80-179b-11eb-9c52-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000018 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e30db14e7532482ac798): POST localhost:1028/givemeDelay, response code: Timeout was reached +time=2020-10-26T14:54:15.996Z | lvl=INFO | corr=184b8b80-179b-11eb-9c52-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000018 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e30db14e7532482ac798): POST localhost:1028/givemeDelay, payload (123 bytes): {"subscriptionId":"5f96e30db14e7532482ac798","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Timeout was reached ``` Endpoint in not responding port, e.g. localhost:9999 (alarm is raised in WARN log level): ``` time=2020-10-26T15:01:50.659Z | lvl=WARN | corr=2d3e4cfc-179c-11eb-b667-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000030 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=AlarmManager.cpp[328]:notificationError | msg=Raising alarm NotificationError localhost:9999/giveme: notification failure for queue worker: Couldn't connect to server -time=2020-10-26T15:01:50.659Z | lvl=INFO | corr=2d3e4cfc-179c-11eb-b667-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000030 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e4deb14e7532482ac79c): POST localhost:9999/giveme, response code: Couldn't connect to server +time=2020-10-26T15:01:50.659Z | lvl=INFO | corr=2d3e4cfc-179c-11eb-b667-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000030 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e4deb14e7532482ac79c): POST localhost:9999/giveme, payload (123 bytes): {"subscriptionId":"5f96e4deb14e7532482ac79c","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Couldn't connect to server ``` Endpoint in unresolvable name, e.g. foo.bar.bar.com (alarm is raised in WARN log level): ``` time=2020-10-26T15:03:54.258Z | lvl=WARN | corr=769f8d8e-179c-11eb-960f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000033 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=AlarmManager.cpp[328]:notificationError | msg=Raising alarm NotificationError foo.bar.bar.com:9999/giveme: notification failure for queue worker: Couldn't resolve host name -time=2020-10-26T15:03:54.258Z | lvl=INFO | corr=769f8d8e-179c-11eb-960f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000033 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e559b14e7532482ac79d): POST foo.bar.bar.com:9999/giveme, response code: Couldn't resolve host name +time=2020-10-26T15:03:54.258Z | lvl=INFO | corr=769f8d8e-179c-11eb-960f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000033 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e559b14e7532482ac79d): POST foo.bar.bar.com:9999/giveme, payload (123 bytes): {"subscriptionId":"5f96e559b14e7532482ac79d","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Couldn't resolve host name ``` Endpoint in unreachable IP, e.g. 12.34.56.87 (alarm is raised in WARN log level): ``` time=2020-10-26T15:06:14.642Z | lvl=WARN | corr=c4a3192e-179c-11eb-ac8f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000036 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=AlarmManager.cpp[328]:notificationError | msg=Raising alarm NotificationError 12.34.56.78:9999/giveme: notification failure for queue worker: Timeout was reached -time=2020-10-26T15:06:14.642Z | lvl=INFO | corr=c4a3192e-179c-11eb-ac8f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000036 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e5dbb14e7532482ac79e): POST 12.34.56.78:9999/giveme, response code: Timeout was reached +time=2020-10-26T15:06:14.642Z | lvl=INFO | corr=c4a3192e-179c-11eb-ac8f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000036 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e5dbb14e7532482ac79e): POST 12.34.56.78:9999/giveme, payload (123 bytes): {"subscriptionId":"5f96e5dbb14e7532482ac79e","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Timeout was reached ``` [Top](#top) From e0a3b4d664f12eaf7557ca7afb3303e6f8f1ac0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 20 Dec 2023 17:27:40 +0100 Subject: [PATCH 044/390] Update doc/manuals/admin/logs.md --- doc/manuals/admin/logs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index cffa9bb934..337016aea1 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -176,7 +176,7 @@ time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df * In the case of MQTT notifications the `msg` field of the above trace is slightly different: ``` -time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoMqttNotification | msg=MQTT Notif delivered (subId: 60ffea6c1bca454f9a64c96c): broker: localhost:1883, topic: sub2, payload (121 bytes): {"subscriptionId":"60ffea6c1bca454f9a64c96c","data":[{"id":"E","type":"T","A":{"type":"Float","value":1,"metadata":{}}}]} +time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoMqttNotification | msg=MQTT Notif delivered (subId: 60ffea6c1bca454f9a64c96c): broker: localhost:1883, topic: sub2, payload (123 bytes): {"subscriptionId":"5f914177334436ea590f6edb","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]} ``` * For each forwarded request to a [Context Provider](../user/context_providers.md) (either queries or updates), From 031ddf6d50267fa1fa04559623eb27b41b18266d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 20 Dec 2023 17:28:31 +0100 Subject: [PATCH 045/390] Update doc/manuals/admin/logs.md --- doc/manuals/admin/logs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index 337016aea1..2f4020b79f 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -176,7 +176,7 @@ time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df * In the case of MQTT notifications the `msg` field of the above trace is slightly different: ``` -time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoMqttNotification | msg=MQTT Notif delivered (subId: 60ffea6c1bca454f9a64c96c): broker: localhost:1883, topic: sub2, payload (123 bytes): {"subscriptionId":"5f914177334436ea590f6edb","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]} +time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoMqttNotification | msg=MQTT Notif delivered (subId: 60ffea6c1bca454f9a64c96c): broker: localhost:1883, topic: sub2, payload (123 bytes): {"subscriptionId":"60ffea6c1bca454f9a64c96c","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]} ``` * For each forwarded request to a [Context Provider](../user/context_providers.md) (either queries or updates), From 0bedb0e6301c7d9f122eaa983bd2bd3865074ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 21 Dec 2023 08:47:56 +0100 Subject: [PATCH 046/390] FIX broken ftest --- src/lib/mongoBackend/location.cpp | 5 + .../log_deprecate_warning.test | 91 ++----------------- 2 files changed, 14 insertions(+), 82 deletions(-) diff --git a/src/lib/mongoBackend/location.cpp b/src/lib/mongoBackend/location.cpp index d524de1e74..b737e7264e 100644 --- a/src/lib/mongoBackend/location.cpp +++ b/src/lib/mongoBackend/location.cpp @@ -320,6 +320,11 @@ static bool getGeoJson std::vector coordLong; orion::BSONArrayBuilder ba; + if ((logDeprecate) && ((caP->type == GEO_POINT) || (caP->type == GEO_LINE) || (caP->type == GEO_BOX) || (caP->type == GEO_POLYGON))) + { + LM_W(("Deprecated usage of %s detected in attribute %s at entity update, please use geo:json instead", caP->type.c_str(), caP->name.c_str())); + } + if (caP->type == GEO_POINT) { double aLat; diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index baf5d43988..663b3c833f 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -32,10 +32,9 @@ brokerStart CB 0 IPV4 -logDeprecate # # 01. Query E1-T1 # 02. GET /v1/contextEntities/E -# 03. Create entity using NGSIv1 metadata location -# 04. Create entity using NGSIv1 and geo:point -# 05. Create entity using NGSIv2 and geo:point -# 06. Get WARNING trace in logs +# 03. Create entity using NGSIv1 and geo:point +# 04. Create entity using NGSIv2 and geo:point +# 05. Get WARNING trace in logs # echo "01. Query E1-T1" @@ -60,38 +59,7 @@ echo echo -echo "03. Create entity using NGSIv1 metadata location" -echo "================================================" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Madrid", - "attributes": [ - { - "name": "location", - "type": "coords", - "value": "40.418889, -3.691944", - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WSG84" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url /v1/updateContext --payload "$payload" -echo -echo - - -echo "04. Create entity using NGSIv1 and geo:point" +echo "03. Create entity using NGSIv1 and geo:point" echo "============================================" payload='{ "contextElements": [ @@ -115,7 +83,7 @@ echo echo -echo "05. Create entity using NGSIv2 and geo:point" +echo "04. Create entity using NGSIv2 and geo:point" echo "============================================" payload='{ "id": "Sevilla", @@ -130,7 +98,7 @@ echo echo -echo "06. Get WARNING trace in logs" +echo "05. Get WARNING trace in logs" echo "=============================" cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' echo @@ -171,46 +139,7 @@ Content-Length: 98 } -03. Create entity using NGSIv1 metadata location -================================================ -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 267 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "location", - "type": "string", - "value": "WSG84" - } - ], - "name": "location", - "type": "coords", - "value": "" - } - ], - "id": "Madrid", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -04. Create entity using NGSIv1 and geo:point +03. Create entity using NGSIv1 and geo:point ============================================ HTTP/1.1 200 OK Date: REGEX(.*) @@ -242,7 +171,7 @@ Content-Length: 207 } -05. Create entity using NGSIv2 and geo:point +04. Create entity using NGSIv2 and geo:point ============================================ HTTP/1.1 201 Created Date: REGEX(.*) @@ -252,12 +181,10 @@ Content-Length: 0 -06. Get WARNING trace in logs +05. Get WARNING trace in logs ============================= Deprecated NGSIv1 request received: POST /v1/queryContext, request payload (48 bytes): { "entities": [ { "type": "T1", "id": "E1" } ] }, response code: 200 Deprecated NGSIv1 request received: GET /v1/contextEntities/E/attributes/A, response code: 200 -Deprecated usage of metadata location coords detected in attribute location at entity update, please use geo:json instead -Deprecated NGSIv1 request received: POST /v1/updateContext, request payload (279 bytes): { "contextElements": [ { "type": "City", "isPattern": "false", "id": "Madrid", "attributes": [ { "name": "location", "type": "coords", "value": "40.418889, -3.691944", "metadatas": [ { "name": "location", "type": "string", "value": "WSG84" } ] } ] } ], "updateAction": "APPEND" }, response code: 200 Deprecated usage of geo:point detected in attribute location at entity update, please use geo:json instead Deprecated NGSIv1 request received: POST /v1/updateContext, request payload (208 bytes): { "contextElements": [ { "type": "City", "isPattern": "false", "id": "Barcelona", "attributes": [ { "name": "location", "type": "geo:point", "value": "40.418889, -3.691944" } ] } ], "updateAction": "APPEND" }, response code: 200 Deprecated usage of geo:point detected in attribute location at entity update, please use geo:json instead From 50e35a8f8447f244fec7b4cb9de8416d983d9d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 21 Dec 2023 10:19:55 +0100 Subject: [PATCH 047/390] FIX upgrade from Debian 12.1 to 12.4 --- ci/deb/Dockerfile | 2 +- doc/manuals.jp/admin/install.md | 2 +- doc/manuals/admin/install.md | 2 +- docker/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/deb/Dockerfile b/ci/deb/Dockerfile index 42a635fd3e..93c38a5cf9 100644 --- a/ci/deb/Dockerfile +++ b/ci/deb/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:12.1-slim +FROM debian:12.4-slim ADD build.sh /opt/bin/ ADD build-dep.sh /opt/bin/ diff --git a/doc/manuals.jp/admin/install.md b/doc/manuals.jp/admin/install.md index eee8edb8f6..07cfba626f 100644 --- a/doc/manuals.jp/admin/install.md +++ b/doc/manuals.jp/admin/install.md @@ -26,7 +26,7 @@ Docker hub で公式の Orion docker コンテナを使用してインストー 必要なソースから ビルドした Orion をインストールする場合： -* オペレーティングシステム: Debian。リファレンス・オペレーティングシステムは Debian 12.1 ですが、それ以降の +* オペレーティングシステム: Debian。リファレンス・オペレーティングシステムは Debian 12.4 ですが、それ以降の Debian 12 バージョンでも動作するはずです * データベース: MongoDB は、Orion Context Broker がインストールされるのと同じホストで実行するか、ネットワーク経由で アクセスできる別のホストで実行する必要があります。推奨される MongoDB バージョンは 6.0 です (Orion は古いバージョンで diff --git a/doc/manuals/admin/install.md b/doc/manuals/admin/install.md index 13da6cc513..5efbe8d65c 100644 --- a/doc/manuals/admin/install.md +++ b/doc/manuals/admin/install.md @@ -23,7 +23,7 @@ In the case you install using the official Orion docker container at Dockerhub, In the case you are installing Orion building from sources you need: -* Operating system: Debian. The reference operating system is Debian 12.1 +* Operating system: Debian. The reference operating system is Debian 12.4 but it should work also in any later Debian 12 version. * Database: MongoDB is required to run either in the same host where Orion Context Broker is to be installed or in a different host accessible through the network. The recommended MongoDB version is 6.0 (Orion may work with older versions but we don't recommend it at all!). diff --git a/docker/Dockerfile b/docker/Dockerfile index 17c2c70b01..b5b80157d4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ # ARG IMAGE_NAME=debian -ARG IMAGE_TAG=12.1-slim +ARG IMAGE_TAG=12.4-slim FROM ${IMAGE_NAME}:${IMAGE_TAG} ARG GITHUB_ACCOUNT=telefonicaid From ff3302f5ed28fd7a37ff5d41478535170b976032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 21 Dec 2023 10:57:06 +0100 Subject: [PATCH 048/390] FIX CNR --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 56f21bf989..fe97d8c1f3 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -8,7 +8,7 @@ - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) - Fix: improve logs in MongoDB query logic - Fix: false positive in log deprecation logic when entity name (or other literal) includes the token "v1" (#4454) -- Upgrade Debian version from 11.6 to 12.1 in Dockerfile +- Upgrade Debian version from 11.6 to 12.4 in Dockerfile - Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3 - Reference MongoDB version changed from 4.4 to 6.0 - Reference distribution changed from Debian 11 to Debian 12 From cb4b1cd7a2ed32744c36203edc6187c93b6ee461 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Thu, 21 Dec 2023 22:00:29 +0900 Subject: [PATCH 049/390] (JP) ADD notification payload in INFO log traces (#4456) --- doc/manuals.jp/admin/logs.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manuals.jp/admin/logs.md b/doc/manuals.jp/admin/logs.md index 65898ed564..9697b65ac5 100644 --- a/doc/manuals.jp/admin/logs.md +++ b/doc/manuals.jp/admin/logs.md @@ -125,13 +125,13 @@ time=2020-10-26T10:32:41.724Z | lvl=INFO | corr=93bdc5b4-1776-11eb-954d-000c29df 異なる値を持ちます ``` -time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f914177334436ea590f6edb): POST localhost:1028/accumulate, response code: 200 +time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f914177334436ea590f6edb): POST localhost:1028/accumulate, payload (123 bytes): {"subscriptionId":"5f914177334436ea590f6edb","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 200 ``` * MQTT 通知の場合、上記のトレースの `msg` フィールドはわずかに異なります: ``` -time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoMqttNotification | msg=MQTT Notif delivered (subId: 60ffea6c1bca454f9a64c96c): broker: localhost:1883, topic: sub2 +time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoMqttNotification | msg=MQTT Notif delivered (subId: 60ffea6c1bca454f9a64c96c): broker: localhost:1883, topic: sub2, payload (123 bytes): {"subscriptionId":"60ffea6c1bca454f9a64c96c","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]} ``` * [コンテキスト・プロバイダ](../user/context_providers.md) (クエリまたは更新) にフォワードされたリクエスト @@ -156,7 +156,7 @@ time=2020-10-22T19:51:03.565Z | lvl=INFO | corr=eabce3e2-149f-11eb-a2e8-000c29df または接続の問題が発生した場合の文字列のいずれかになります。 例えば: ``` -time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f914177334436ea590f6edb): POST localhost:1028/accumulate, response code: Couldn't connect to server +time=2020-10-26T10:32:22.145Z | lvl=INFO | corr=87f708a8-1776-11eb-b327-000c29df7908; cbnotif=1 | trans=1603707992-318-00000000003 | from=0.0.0.0 | srv=s1| subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f914177334436ea590f6edb): POST localhost:1028/accumulate, payload (123 bytes): {"subscriptionId":"5f914177334436ea590f6edb","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Couldn't connect to server ``` * クライアント・リクエストがコンテキスト・プロバイダへのフォワーディングをトリガーすると、最初に転送された @@ -286,56 +286,56 @@ contextBroker -fg -httpTimeout 10000 -logLevel INFO -notificationMode threadpool 送信成功 (レスポンス・コード 200) : ``` -time=2020-10-26T14:48:37.192Z | lvl=INFO | corr=54393a44-179a-11eb-bb87-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000006 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e174b14e7532482ac794): POST localhost:1028/accumulate, response code: 200 +time=2020-10-26T14:48:37.192Z | lvl=INFO | corr=54393a44-179a-11eb-bb87-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000006 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e174b14e7532482ac794): POST localhost:1028/accumulate, payload (123 bytes): {"subscriptionId":"5f96e174b14e7532482ac794","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 200 ``` 400 での通知エンドポイントのレスポンス (WARN トレースがプリントされます) : ``` time=2020-10-26T14:49:34.619Z | lvl=WARN | corr=7689f6ba-179a-11eb-ac4c-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000009 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=httpRequestSend.cpp[583]:httpRequestSend | msg=Notification (subId: 5f96e1fdb14e7532482ac795) response NOT OK, http code: 400 -time=2020-10-26T14:49:34.619Z | lvl=INFO | corr=7689f6ba-179a-11eb-ac4c-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000009 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e1fdb14e7532482ac795): POST localhost:1028/giveme400, response code: 400 +time=2020-10-26T14:49:34.619Z | lvl=INFO | corr=7689f6ba-179a-11eb-ac4c-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000009 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e1fdb14e7532482ac795): POST localhost:1028/giveme400, payload (123 bytes): {"subscriptionId":"5f96e1fdb14e7532482ac795","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 400 ``` 404 での通知エンドポイントのレスポンス (WARN トレースがプリントされます) : ``` time=2020-10-26T14:51:40.764Z | lvl=WARN | corr=c1b8e9c0-179a-11eb-9edc-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000012 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=httpRequestSend.cpp[583]:httpRequestSend | msg=Notification (subId: 5f96e27cb14e7532482ac796) response NOT OK, http code: 404 -time=2020-10-26T14:51:40.764Z | lvl=INFO | corr=c1b8e9c0-179a-11eb-9edc-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000012 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e27cb14e7532482ac796): POST localhost:1028/giveme404, response code: 404 +time=2020-10-26T14:51:40.764Z | lvl=INFO | corr=c1b8e9c0-179a-11eb-9edc-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000012 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e27cb14e7532482ac796): POST localhost:1028/giveme404, payload (123 bytes): {"subscriptionId":"5f96e27cb14e7532482ac796","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 404 ``` 500 での通知エンドポイントのレスポンス (WARN トレースがプリントされます) : ``` time=2020-10-26T14:53:04.246Z | lvl=WARN | corr=f37b5024-179a-11eb-9ce6-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000015 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=httpRequestSend.cpp[583]:httpRequestSend | msg=Notification (subId: 5f96e2cfb14e7532482ac797) response NOT OK, http code: 500 -time=2020-10-26T14:53:04.247Z | lvl=INFO | corr=f37b5024-179a-11eb-9ce6-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000015 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e2cfb14e7532482ac797): POST localhost:1028/giveme500, response code: 500 +time=2020-10-26T14:53:04.247Z | lvl=INFO | corr=f37b5024-179a-11eb-9ce6-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000015 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e2cfb14e7532482ac797): POST localhost:1028/giveme500, payload (123 bytes): {"subscriptionId":"5f96e2cfb14e7532482ac797","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: 500 ``` 10 秒以内にエンドポイントが応答しない、またはその他の何らかの接続エラーが発生しました (アラームは WARN レベルで発生します) : ``` time=2020-10-26T14:54:15.996Z | lvl=WARN | corr=184b8b80-179b-11eb-9c52-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000018 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=AlarmManager.cpp[328]:notificationError | msg=Raising alarm NotificationError localhost:1028/givemeDelay: notification failure for queue worker: Timeout was reached -time=2020-10-26T14:54:15.996Z | lvl=INFO | corr=184b8b80-179b-11eb-9c52-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000018 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e30db14e7532482ac798): POST localhost:1028/givemeDelay, response code: Timeout was reached +time=2020-10-26T14:54:15.996Z | lvl=INFO | corr=184b8b80-179b-11eb-9c52-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000018 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e30db14e7532482ac798): POST localhost:1028/givemeDelay, payload (123 bytes): {"subscriptionId":"5f96e30db14e7532482ac798","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Timeout was reached ``` 応答しないポートのエンドポイント。例えば、localhost：9999 (アラームは WARN ログ・レベルで発生します) : ``` time=2020-10-26T15:01:50.659Z | lvl=WARN | corr=2d3e4cfc-179c-11eb-b667-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000030 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=AlarmManager.cpp[328]:notificationError | msg=Raising alarm NotificationError localhost:9999/giveme: notification failure for queue worker: Couldn't connect to server -time=2020-10-26T15:01:50.659Z | lvl=INFO | corr=2d3e4cfc-179c-11eb-b667-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000030 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e4deb14e7532482ac79c): POST localhost:9999/giveme, response code: Couldn't connect to server +time=2020-10-26T15:01:50.659Z | lvl=INFO | corr=2d3e4cfc-179c-11eb-b667-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000030 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e4deb14e7532482ac79c): POST localhost:9999/giveme, payload (123 bytes): {"subscriptionId":"5f96e4deb14e7532482ac79c","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Couldn't connect to server ``` 解決できない名前のエンドポイント。例えば、foo.bar.bar.com (アラームは WARN ログ・レベルで発生します) : ``` time=2020-10-26T15:03:54.258Z | lvl=WARN | corr=769f8d8e-179c-11eb-960f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000033 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=AlarmManager.cpp[328]:notificationError | msg=Raising alarm NotificationError foo.bar.bar.com:9999/giveme: notification failure for queue worker: Couldn't resolve host name -time=2020-10-26T15:03:54.258Z | lvl=INFO | corr=769f8d8e-179c-11eb-960f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000033 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e559b14e7532482ac79d): POST foo.bar.bar.com:9999/giveme, response code: Couldn't resolve host name +time=2020-10-26T15:03:54.258Z | lvl=INFO | corr=769f8d8e-179c-11eb-960f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000033 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e559b14e7532482ac79d): POST foo.bar.bar.com:9999/giveme, payload (123 bytes): {"subscriptionId":"5f96e559b14e7532482ac79d","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Couldn't resolve host name ``` 到達不能な IP のエンドポイント。例えば、12.34.56.87 (アラームは WARN ログ・レベルで発生します) : ``` time=2020-10-26T15:06:14.642Z | lvl=WARN | corr=c4a3192e-179c-11eb-ac8f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000036 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=AlarmManager.cpp[328]:notificationError | msg=Raising alarm NotificationError 12.34.56.78:9999/giveme: notification failure for queue worker: Timeout was reached -time=2020-10-26T15:06:14.642Z | lvl=INFO | corr=c4a3192e-179c-11eb-ac8f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000036 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e5dbb14e7532482ac79e): POST 12.34.56.78:9999/giveme, response code: Timeout was reached +time=2020-10-26T15:06:14.642Z | lvl=INFO | corr=c4a3192e-179c-11eb-ac8f-000c29df7908; cbnotif=1 | trans=1603722272-416-00000000036 | from=0.0.0.0 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[63]:logInfoHttpNotification | msg=Notif delivered (subId: 5f96e5dbb14e7532482ac79e): POST 12.34.56.78:9999/giveme, payload (123 bytes): {"subscriptionId":"5f96e5dbb14e7532482ac79e","data":[{"id":"E","type":"T","A":{"type":"Number","value":42,"metadata":{}}}]}, response code: Timeout was reached ``` [トップ](#top) From 18136f6de321567f640bc7cedfcd99de9a32b5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 22 Dec 2023 11:28:44 +0100 Subject: [PATCH 050/390] ADD test case for 4460 working aligned with the current faulty behaviour --- .../overloging_runtime_error_fwd_update.test | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test diff --git a/test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test b/test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test new file mode 100644 index 0000000000..783003bac7 --- /dev/null +++ b/test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test @@ -0,0 +1,189 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Overloging Runtime Error on forwarding update request case + +--SHELL-INIT-- +dbInit CB +dbInit CP1 +brokerStart CB +brokerStart CP1 + +--SHELL-- + +# +# 01. Create device:wt:13 attribute profile_config in CP1 +# 02. Create device:wt:13 attribute profile in CB +# 03. Register device:wt:13 attribute profile_config in CB, provApp: CP1 +# 04. Update device:wt:13 attribute profile_config in CB +# 05. Check no Runtime Error occur +# 06. Query device:wt:13 attribute profile_config in CP1 +# + +echo "01. Create device:wt:13 attribute profile_config in CP1" +echo "=======================================================" +payload='{ + "id": "device:wt:13", + "type": "Device", + "profile_config": { + "type": "command", + "value": "init" + } +}' +orionCurl --url /v2/entities --payload "$payload" --port $CP1_PORT +echo +echo + + +echo "02. Create device:wt:13 attribute profile in CB" +echo "===============================================" +payload='{ + "id": "device:wt:13", + "type": "Device", + "profile": { + "type": "Text", + "value": "prof01" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Register device:wt:13 attribute profile_config in CB, provApp: CP1" +echo "======================================================================" +payload='{ + "dataProvided": { + "entities": [ + { + "type": "Device", + "id": "device:wt:13" + } + ], + "attrs": [ "profile_config" ] + }, + "provider": { + "http": { + "url": "http://localhost:'${CP1_PORT}'/v2" + } + } +}' +orionCurl --url /v2/registrations --payload "$payload" +echo +echo + + +echo "04. Update device:wt:13 attribute profile_config in CB" +echo "======================================================" +payload='{ + "type": "command", + "value": "foobar" +}' +orionCurl --url /v2/entities/device:wt:13/attrs/profile_config --payload "$payload" -X PUT +echo +echo + + +echo "05. Check no Runtime Error occur" +echo "================================" +cat /tmp/contextBroker.log | grep "Runtime Error" | wc -l +echo +echo + + +echo "06. Query device:wt:13 attribute profile_config in CP1 and see foobar" +echo "=====================================================================" +orionCurl --url /v2/entities/device:wt:13 --port $CP1_PORT +echo +echo + + + +--REGEXPECT-- +01. Create device:wt:13 attribute profile_config in CP1 +======================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/device:wt:13?type=Device +Content-Length: 0 + + + +02. Create device:wt:13 attribute profile in CB +=============================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/device:wt:13?type=Device +Content-Length: 0 + + + +03. Register device:wt:13 attribute profile_config in CB, provApp: CP1 +====================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/registrations/REGEX([0-9a-f\-]{24}) +Content-Length: 0 + + + +04. Update device:wt:13 attribute profile_config in CB +====================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Check no Runtime Error occur +================================ +1 + + +06. Query device:wt:13 attribute profile_config in CP1 and see foobar +===================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 98 + +{ + "id": "device:wt:13", + "profile_config": { + "metadata": {}, + "type": "command", + "value": "" + }, + "type": "Device" +} + + +--TEARDOWN-- +brokerStop CB +brokerStop CP1 +dbDrop CB +dbDrop CP1 From 0512f650719822452d831b0cdfd264b564f3701d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 22 Dec 2023 11:33:58 +0100 Subject: [PATCH 051/390] FIX overloging runtime error problem and forwarding update case --- src/lib/ngsi10/UpdateContextRequest.cpp | 4 +++- .../overloging_runtime_error_fwd_update.test | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib/ngsi10/UpdateContextRequest.cpp b/src/lib/ngsi10/UpdateContextRequest.cpp index 2959a8e743..ca55df1790 100644 --- a/src/lib/ngsi10/UpdateContextRequest.cpp +++ b/src/lib/ngsi10/UpdateContextRequest.cpp @@ -332,11 +332,13 @@ ContextAttribute* UpdateContextRequest::attributeLookup(Entity* eP, const std::s { Entity* enP = entityVector[ceIx]; - if ((enP->id != eP->id) || (enP->type != eP->type)) + // empty type in request (enP) is always a match + if ((enP->id != eP->id) || ((enP->type != "") && (enP->type != eP->type))) { continue; } + // FIXME PR: eP is overriden here. Remove eP from function signature Entity* eP = entityVector[ceIx]; for (unsigned int aIx = 0; aIx < eP->attributeVector.size(); ++aIx) diff --git a/test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test b/test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test index 783003bac7..a7331870c4 100644 --- a/test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test +++ b/test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test @@ -160,7 +160,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) 05. Check no Runtime Error occur ================================ -1 +0 06. Query device:wt:13 attribute profile_config in CP1 and see foobar @@ -169,14 +169,14 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 98 +Content-Length: 104 { "id": "device:wt:13", "profile_config": { "metadata": {}, "type": "command", - "value": "" + "value": "foobar" }, "type": "Device" } From a885d5627bbef211ff9d6ab1a31eed5bf50398e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 22 Dec 2023 11:51:08 +0100 Subject: [PATCH 052/390] FIX cleanup and CNR --- CHANGES_NEXT_RELEASE | 1 + src/lib/ngsi10/UpdateContextRequest.cpp | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index fe97d8c1f3..0cfc39e90e 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -2,6 +2,7 @@ - Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388) - Add: notification payload in INFO log traces (#4449) - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) +- Fix: update forwarding was not working when entity type is not included in the request (#4460) - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) - Fix: improve error traces (#4387) diff --git a/src/lib/ngsi10/UpdateContextRequest.cpp b/src/lib/ngsi10/UpdateContextRequest.cpp index ca55df1790..db3a910e75 100644 --- a/src/lib/ngsi10/UpdateContextRequest.cpp +++ b/src/lib/ngsi10/UpdateContextRequest.cpp @@ -338,12 +338,11 @@ ContextAttribute* UpdateContextRequest::attributeLookup(Entity* eP, const std::s continue; } - // FIXME PR: eP is overriden here. Remove eP from function signature - Entity* eP = entityVector[ceIx]; + Entity* eVItemP = entityVector[ceIx]; - for (unsigned int aIx = 0; aIx < eP->attributeVector.size(); ++aIx) + for (unsigned int aIx = 0; aIx < eVItemP->attributeVector.size(); ++aIx) { - ContextAttribute* aP = eP->attributeVector[aIx]; + ContextAttribute* aP = eVItemP->attributeVector[aIx]; if (aP->name == attributeName) { From 7e3d43bb8b0f27a12e4dbc15c70117b0db5693c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 22 Dec 2023 12:09:20 +0100 Subject: [PATCH 053/390] RENAME case 4460 ftest --- ...d_not_working_when_entity_type_not_included_in_request.test} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename test/functionalTest/cases/{4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test => 4460_update_fwd_not_working_when_entity_type_not_included_in_request/update_fwd_not_working_when_entity_type_not_included_in_request.test} (98%) diff --git a/test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test b/test/functionalTest/cases/4460_update_fwd_not_working_when_entity_type_not_included_in_request/update_fwd_not_working_when_entity_type_not_included_in_request.test similarity index 98% rename from test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test rename to test/functionalTest/cases/4460_update_fwd_not_working_when_entity_type_not_included_in_request/update_fwd_not_working_when_entity_type_not_included_in_request.test index a7331870c4..c1dba621db 100644 --- a/test/functionalTest/cases/4460_overloging_runtime_error_fwd_update/overloging_runtime_error_fwd_update.test +++ b/test/functionalTest/cases/4460_update_fwd_not_working_when_entity_type_not_included_in_request/update_fwd_not_working_when_entity_type_not_included_in_request.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -Overloging Runtime Error on forwarding update request case +Update forwarding was not working when entity type is not included in the request --SHELL-INIT-- dbInit CB From a4b8cc1407366e41ca6f54e5d594207beeb6b2d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 8 Jan 2024 09:23:47 +0100 Subject: [PATCH 054/390] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 048209e15d..4e7879928f 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -6,7 +6,7 @@ - Fix: improve error traces (#4387) - Fix: on delete attributes operation, the lack of attribute in entity doesn't preclude the deletion of attributes which do exist (this way behaving same way as update attributes operation) - Fix: POST /v2/entities/E/attrs?options=append was wrongly creating entities -- Fix: provide more inforamtive error description in some error responses in update/delete operations +- Fix: provide more informative error description in some error responses in update/delete operations - Fix: proper use of "PartialUpdate" (instead of "Unprocessed") in responses in the case of partial updates/deletions (#3499) - Fix: response 404 Not Found "NotFound" errors instead of 422 Unprocessable Content "Unprocessed" in the case of missing attribute in existing entity in attribute update operations - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) From c16bdc980fc0f5e86ee512e644ef79d92f41f814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 8 Jan 2024 14:47:14 +0100 Subject: [PATCH 055/390] FIX documentation --- doc/manuals/orion-api.md | 63 +++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index a524ca2eed..242895a2d2 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2816,8 +2816,9 @@ _**Request headers**_ _**Response code**_ * Successful operation uses 200 OK -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response headers**_ @@ -2909,8 +2910,9 @@ _**Request headers**_ _**Response code**_ * Successful operation uses 200 OK -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response headers**_ @@ -3024,9 +3026,10 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity - * 422 Unprocessable Content for existing attributes when `append` options is used +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity (see next subsection) + * 422 Unprocessable Content for existing attributes when `append` options is used (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3123,9 +3126,10 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity - * 422 Unprocessable Content for non existing attributes +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity (see next subsection) + * 422 Unprocessable Content for non existing attributes (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3221,8 +3225,9 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3263,8 +3268,9 @@ _**Request headers**_ _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3400,8 +3406,9 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity or not found attribute +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity or not found attribute (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3452,8 +3459,9 @@ _**Request headers**_ _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity or not found attribute +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity or not found attribute (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3513,9 +3521,10 @@ _**Request headers**_ _**Response code**_ * Successful operation uses 200 OK -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity or not found attribute - * 406 Not Acceptable in the case of not acceptable content +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity or not found attribute (see next subsection) + * 406 Not Acceptable in the case of not acceptable content (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response headers**_ @@ -3631,8 +3640,9 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found for not found entity or not found attribute +* Errors use a non-2xx code and error payload: + * 404 Not Found for not found entity or not found attribute (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -4636,9 +4646,10 @@ Example: _**Response code**_ * Successful operation uses 204 No Content -* Errors use a non-2xx code and error payload (see next subsection): - * 404 Not Found if none of the entities in the `entities` field exists in `update`, `delete` or `replace` cases - * 422 Unprocessable Content for other cases +* Errors use a non-2xx code and error payload: + * 404 Not Found if none of the entities in the `entities` field exists in `update`, `delete` or `replace` cases (see next subsection) + * 422 Unprocessable Content for other cases (see next subsection) + * Check addicional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ From b46c269bbdd40efb2c25091bae88ab89fc9fb97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 8 Jan 2024 14:52:12 +0100 Subject: [PATCH 056/390] FIX typo in doc --- doc/manuals/orion-api.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 242895a2d2..671606686b 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2818,7 +2818,7 @@ _**Response code**_ * Successful operation uses 200 OK * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response headers**_ @@ -2912,7 +2912,7 @@ _**Response code**_ * Successful operation uses 200 OK * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response headers**_ @@ -3029,7 +3029,7 @@ _**Response code**_ * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity (see next subsection) * 422 Unprocessable Content for existing attributes when `append` options is used (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3129,7 +3129,7 @@ _**Response code**_ * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity (see next subsection) * 422 Unprocessable Content for non existing attributes (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3227,7 +3227,7 @@ _**Response code**_ * Successful operation uses 204 No Content * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3270,7 +3270,7 @@ _**Response code**_ * Successful operation uses 204 No Content * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3408,7 +3408,7 @@ _**Response code**_ * Successful operation uses 204 No Content * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity or not found attribute (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3461,7 +3461,7 @@ _**Response code**_ * Successful operation uses 204 No Content * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity or not found attribute (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -3524,7 +3524,7 @@ _**Response code**_ * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity or not found attribute (see next subsection) * 406 Not Acceptable in the case of not acceptable content (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response headers**_ @@ -3642,7 +3642,7 @@ _**Response code**_ * Successful operation uses 204 No Content * Errors use a non-2xx code and error payload: * 404 Not Found for not found entity or not found attribute (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ @@ -4649,7 +4649,7 @@ _**Response code**_ * Errors use a non-2xx code and error payload: * 404 Not Found if none of the entities in the `entities` field exists in `update`, `delete` or `replace` cases (see next subsection) * 422 Unprocessable Content for other cases (see next subsection) - * Check addicional cases in [Error Responses](#error-responses) general documentation + * Check additional cases in [Error Responses](#error-responses) general documentation _**Response payload**_ From 370cde9704d1d366a9068f0d8687ffe0f00fd1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 8 Jan 2024 15:40:45 +0100 Subject: [PATCH 057/390] FIX steping some tokens to year 2024 --- README.md | 2 +- src/lib/parseArgs/paConfig.h | 2 +- test/functionalTest/cases/0000_cli/version.test | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 71526c8105..0c30a069bd 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ tag `fiware-orion` Orion Context Broker is licensed under [Affero General Public License (GPL) version 3](./LICENSE). -© 2023 Telefonica Investigación y Desarrollo, S.A.U +© 2024 Telefonica Investigación y Desarrollo, S.A.U
Further information on the use of the AGPL open source license diff --git a/src/lib/parseArgs/paConfig.h b/src/lib/parseArgs/paConfig.h index 54f33f407b..679974a0ac 100644 --- a/src/lib/parseArgs/paConfig.h +++ b/src/lib/parseArgs/paConfig.h @@ -39,7 +39,7 @@ #endif #ifndef DEFAULT_COPYRIGHT - #define DEFAULT_COPYRIGHT "Copyright 2013-2023 Telefonica Investigacion y Desarrollo, S.A.U\n" \ + #define DEFAULT_COPYRIGHT "Copyright 2013-2024 Telefonica Investigacion y Desarrollo, S.A.U\n" \ "Orion Context Broker is free software: you can redistribute it and/or\n" \ "modify it under the terms of the GNU Affero General Public License as\n" \ "published by the Free Software Foundation, either version 3 of the\n" \ diff --git a/test/functionalTest/cases/0000_cli/version.test b/test/functionalTest/cases/0000_cli/version.test index 73e65f85b1..bac1621329 100644 --- a/test/functionalTest/cases/0000_cli/version.test +++ b/test/functionalTest/cases/0000_cli/version.test @@ -31,7 +31,7 @@ contextBroker --version --REGEXPECT-- REGEX(\d+\.\d+\.\d+.* \(git version: .*\)) -Copyright 2013-2023 Telefonica Investigacion y Desarrollo, S.A.U +Copyright 2013-2024 Telefonica Investigacion y Desarrollo, S.A.U Orion Context Broker is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the From 0395f1e89a2a2646479dc57cb0f9631c583d08d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 8 Jan 2024 16:25:17 +0100 Subject: [PATCH 058/390] FIX OrionError field names refactor --- src/lib/jsonParseV2/parseNotification.cpp | 2 +- src/lib/jsonParseV2/parseRegistration.cpp | 24 +++--- .../mongoBackend/mongoSubscribeContext.cpp | 2 +- src/lib/mongoBackend/mongoUpdateContext.cpp | 2 +- .../mongoUpdateContextSubscription.cpp | 2 +- src/lib/rest/OrionError.cpp | 80 +++++++++---------- src/lib/rest/OrionError.h | 12 +-- src/lib/rest/RestService.cpp | 2 +- src/lib/rest/rest.cpp | 12 +-- src/lib/serviceRoutines/postQueryContext.cpp | 2 +- .../serviceRoutinesV2/getEntityAttribute.cpp | 4 +- .../serviceRoutinesV2/putEntityAttribute.cpp | 2 +- .../serviceRoutinesCommon.cpp | 10 +-- .../mongoCreateSubscription_test.cpp | 8 +- .../mongoGetSubscriptions_test.cpp | 8 +- test/unittests/rest/OrionError_test.cpp | 16 ++-- 16 files changed, 94 insertions(+), 94 deletions(-) diff --git a/src/lib/jsonParseV2/parseNotification.cpp b/src/lib/jsonParseV2/parseNotification.cpp index 6e6ed9e6ae..dd975f2e2f 100644 --- a/src/lib/jsonParseV2/parseNotification.cpp +++ b/src/lib/jsonParseV2/parseNotification.cpp @@ -212,7 +212,7 @@ static bool parseNotificationNormalized(ConnectionInfo* ciP, NotifyContextReques if (parseNotificationData(ciP, iter, ncrP, oeP) == false) { - alarmMgr.badInput(clientIp, oeP->details); + alarmMgr.badInput(clientIp, oeP->description); ciP->httpStatusCode = SccBadRequest; return false; diff --git a/src/lib/jsonParseV2/parseRegistration.cpp b/src/lib/jsonParseV2/parseRegistration.cpp index ac84af15bc..af3261e537 100644 --- a/src/lib/jsonParseV2/parseRegistration.cpp +++ b/src/lib/jsonParseV2/parseRegistration.cpp @@ -58,8 +58,8 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccBadRequest; oeP->code = SccBadRequest; - oeP->reasonPhrase = ERROR_BAD_REQUEST; - oeP->details = "/dataProvided/ must be a JSON object"; + oeP->error = ERROR_BAD_REQUEST; + oeP->description = "/dataProvided/ must be a JSON object"; return false; } @@ -68,20 +68,20 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccBadRequest; oeP->code = SccBadRequest; - oeP->reasonPhrase = ERROR_BAD_REQUEST; - oeP->details = "/dataProvided/ is empty"; + oeP->error = ERROR_BAD_REQUEST; + oeP->description = "/dataProvided/ is empty"; return false; } if (dataProvided.HasMember("entities")) { - bool b = parseEntitiesVector(ciP, &dataProvidedP->entities, dataProvided["entities"], &oeP->details); + bool b = parseEntitiesVector(ciP, &dataProvidedP->entities, dataProvided["entities"], &oeP->description); if (b == false) { ciP->httpStatusCode = SccBadRequest; oeP->code = SccBadRequest; - oeP->reasonPhrase = ERROR_BAD_REQUEST; + oeP->error = ERROR_BAD_REQUEST; return false; } @@ -99,8 +99,8 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccNotImplemented; oeP->code = SccNotImplemented; - oeP->reasonPhrase = ERROR_NOTIMPLEMENTED; - oeP->details = ERROR_DESC_IDPATTERN_NOTSUPPORTED; + oeP->error = ERROR_NOTIMPLEMENTED; + oeP->description = ERROR_DESC_IDPATTERN_NOTSUPPORTED; return false; } @@ -109,8 +109,8 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccNotImplemented; oeP->code = SccNotImplemented; - oeP->reasonPhrase = ERROR_NOTIMPLEMENTED; - oeP->details = ERROR_DESC_TYPEPATTERN_NOTIMPLEMENTED; + oeP->error = ERROR_NOTIMPLEMENTED; + oeP->description = ERROR_DESC_TYPEPATTERN_NOTIMPLEMENTED; return false; } @@ -118,11 +118,11 @@ static bool dataProvidedParse if (dataProvided.HasMember("attrs")) { - if (!parseStringVector(&dataProvidedP->attributes, dataProvided["attrs"], "attrs", true, true, &oeP->details)) + if (!parseStringVector(&dataProvidedP->attributes, dataProvided["attrs"], "attrs", true, true, &oeP->description)) { ciP->httpStatusCode = SccBadRequest; oeP->code = SccBadRequest; - oeP->reasonPhrase = ERROR_BAD_REQUEST; + oeP->error = ERROR_BAD_REQUEST; return false; } diff --git a/src/lib/mongoBackend/mongoSubscribeContext.cpp b/src/lib/mongoBackend/mongoSubscribeContext.cpp index 9064dece44..966d72d156 100644 --- a/src/lib/mongoBackend/mongoSubscribeContext.cpp +++ b/src/lib/mongoBackend/mongoSubscribeContext.cpp @@ -70,7 +70,7 @@ HttpStatusCode mongoSubscribeContext else { // Check OrionError - responseP->subscribeError.errorCode.fill(oe.code, oe.details); + responseP->subscribeError.errorCode.fill(oe.code, oe.description); } // free sub memory associated to subscription diff --git a/src/lib/mongoBackend/mongoUpdateContext.cpp b/src/lib/mongoBackend/mongoUpdateContext.cpp index a6e29ea78b..f3cea44ee8 100644 --- a/src/lib/mongoBackend/mongoUpdateContext.cpp +++ b/src/lib/mongoBackend/mongoUpdateContext.cpp @@ -222,7 +222,7 @@ HttpStatusCode mongoUpdateContext if (updateCoverage == UC_PARTIAL) { responseP->oe.code = SccInvalidModification; - responseP->oe.reasonPhrase = ERROR_PARTIAL_UPDATE; + responseP->oe.error = ERROR_PARTIAL_UPDATE; } LM_T(LmtNotifier, ("total notifications sent during update: %d", notifSent)); diff --git a/src/lib/mongoBackend/mongoUpdateContextSubscription.cpp b/src/lib/mongoBackend/mongoUpdateContextSubscription.cpp index 7a14172a15..c27317b76f 100644 --- a/src/lib/mongoBackend/mongoUpdateContextSubscription.cpp +++ b/src/lib/mongoBackend/mongoUpdateContextSubscription.cpp @@ -83,7 +83,7 @@ HttpStatusCode mongoUpdateContextSubscription } else { - responseP->subscribeError.errorCode.fill(oe.code, oe.details); + responseP->subscribeError.errorCode.fill(oe.code, oe.description); } } diff --git a/src/lib/rest/OrionError.cpp b/src/lib/rest/OrionError.cpp index 3f178f690e..dfcb40af71 100644 --- a/src/lib/rest/OrionError.cpp +++ b/src/lib/rest/OrionError.cpp @@ -40,8 +40,8 @@ OrionError::OrionError() { code = SccNone; - reasonPhrase = ""; - details = ""; + error = ""; + description = ""; filled = false; } @@ -51,12 +51,12 @@ OrionError::OrionError() * * OrionError::OrionError - */ -OrionError::OrionError(HttpStatusCode _code, const std::string& _details, const std::string& _reasonPhrase) +OrionError::OrionError(HttpStatusCode _code, const std::string& _description, const std::string& _error) { - code = _code; - reasonPhrase = _reasonPhrase.empty() ? httpStatusCodeString(code) : _reasonPhrase; - details = _details; - filled = true; + code = _code; + error = _error.empty() ? httpStatusCodeString(code) : _error; + description = _description; + filled = true; } @@ -67,10 +67,10 @@ OrionError::OrionError(HttpStatusCode _code, const std::string& _details, const */ OrionError::OrionError(StatusCode& sc) { - code = sc.code; - reasonPhrase = httpStatusCodeString(code); - details = sc.details; - filled = true; + code = sc.code; + error = httpStatusCodeString(code); + description = sc.details; + filled = true; } @@ -79,12 +79,12 @@ OrionError::OrionError(StatusCode& sc) * * OrionError::fill - */ -void OrionError::fill(HttpStatusCode _code, const std::string& _details, const std::string& _reasonPhrase) +void OrionError::fill(HttpStatusCode _code, const std::string& _description, const std::string& _error) { - code = _code; - reasonPhrase = _reasonPhrase.empty()? httpStatusCodeString(code) : _reasonPhrase; - details = _details; - filled = true; + code = _code; + error = _error.empty()? httpStatusCodeString(code) : _error; + description = _description; + filled = true; } @@ -95,10 +95,10 @@ void OrionError::fill(HttpStatusCode _code, const std::string& _details, const s */ void OrionError::fill(const StatusCode& sc) { - code = sc.code; - reasonPhrase = (sc.reasonPhrase.empty())? httpStatusCodeString(code) : sc.reasonPhrase; - details = sc.details; - filled = true; + code = sc.code; + error = (sc.reasonPhrase.empty())? httpStatusCodeString(code) : sc.reasonPhrase; + description = sc.details; + filled = true; } @@ -107,19 +107,19 @@ void OrionError::fill(const StatusCode& sc) * * OrionError::fillOrAppend - */ -void OrionError::fillOrAppend(HttpStatusCode _code, const std::string& fullDetails, const std::string& appendDetail, const std::string& _reasonPhrase) +void OrionError::fillOrAppend(HttpStatusCode _code, const std::string& fullDetails, const std::string& appendDetail, const std::string& _error) { if (filled) { // Already filled by a previous operation. This can happen in batch update processing - details += appendDetail; + description += appendDetail; } else { - code = _code; - reasonPhrase = _reasonPhrase.empty()? httpStatusCodeString(code) : _reasonPhrase; - details = fullDetails; - filled = true; + code = _code; + error = _error.empty()? httpStatusCodeString(code) : _error; + description = fullDetails; + filled = true; } } @@ -137,7 +137,7 @@ std::string OrionError::smartRender(ApiVersion apiVersion) } else // admin or v2 { - shrinkReasonPhrase(); + shrinkError(); return toJson(); } } @@ -166,8 +166,8 @@ std::string OrionError::setStatusCodeAndSmartRender(ApiVersion apiVersion, HttpS */ std::string OrionError::toJson(void) { - char* reasonPhraseEscaped = htmlEscape(reasonPhrase.c_str()); - char* detailsEscaped = htmlEscape(details.c_str()); + char* reasonPhraseEscaped = htmlEscape(error.c_str()); + char* detailsEscaped = htmlEscape(description.c_str()); JsonObjectHelper jh; @@ -195,12 +195,12 @@ std::string OrionError::toJsonV1(void) // OrionError is NEVER part of any other payload, so the JSON start/end braces must be added here // out += startTag("orionError", false); - out += valueTag("code", code, true); - out += valueTag("reasonPhrase", reasonPhrase, !details.empty()); + out += valueTag("code", code, true); + out += valueTag("reasonPhrase", error, !description.empty()); - if (!details.empty()) + if (!description.empty()) { - out += valueTag("details", details); + out += valueTag("details", description); } out += endTag(); @@ -214,9 +214,9 @@ std::string OrionError::toJsonV1(void) /* **************************************************************************** * -* OrionError::shrinkReasonPhrase - +* OrionError::shrinkError - * -* This method removes any whitespace in the reasonPhrase field, i.e. +* This method removes any whitespace in the error field, i.e. * transforms "Not Found" to "NotFound". * * It is used by smartRender method, in order to prepare to render in API v2 case @@ -228,19 +228,19 @@ std::string OrionError::toJsonV1(void) * * ... * -* reasonPhrase.erase(std::remove_if(reasonPhrase.begin(), reasonPhrase.end(), std::isspace), reasonPhrase.end()); +* reasonPhrase.erase(std::remove_if(error.begin(), error.end(), std::isspace), error.end()); * * However, 'std::isspace' doesn't directly work. We have been able to make it work with * 'static_cast(isspace)'. However, that is obscure so until we can find * a way of using just 'std::isspace', the current implementation stills. * */ -void OrionError::shrinkReasonPhrase(void) +void OrionError::shrinkError(void) { char buf[80]; // 80 should be enough to hold any reason phrase #if 0 - strncpy(buf, reasonPhrase.c_str(), sizeof(buf)); + strncpy(buf, error.c_str(), sizeof(buf)); // See: http://stackoverflow.com/questions/1726302/removing-spaces-from-a-string-in-c if (*j != ' ') @@ -263,7 +263,7 @@ void OrionError::shrinkReasonPhrase(void) *i = 0; #endif - char* fromP = (char*) reasonPhrase.c_str(); + char* fromP = (char*) error.c_str(); char* toP = buf; unsigned int toLen = 0; @@ -281,5 +281,5 @@ void OrionError::shrinkReasonPhrase(void) } *toP = 0; // End-of string - reasonPhrase = std::string(buf); + error = std::string(buf); } diff --git a/src/lib/rest/OrionError.h b/src/lib/rest/OrionError.h index ab6110c944..2418ce8e64 100644 --- a/src/lib/rest/OrionError.h +++ b/src/lib/rest/OrionError.h @@ -42,25 +42,25 @@ typedef struct OrionError { public: HttpStatusCode code; - std::string reasonPhrase; - std::string details; + std::string error; + std::string description; bool filled; OrionError(); OrionError(StatusCode& statusCode); - OrionError(HttpStatusCode _code, const std::string& _details = "", const std::string& _reasonPhrase = ""); + OrionError(HttpStatusCode _code, const std::string& _description = "", const std::string& _error = ""); std::string smartRender(ApiVersion apiVersion); std::string setStatusCodeAndSmartRender(ApiVersion apiVersion, HttpStatusCode* scP); std::string toJson(void); std::string toJsonV1(void); - void fill(HttpStatusCode _code, const std::string& _details, const std::string& _reasonPhrase = ""); + void fill(HttpStatusCode _code, const std::string& _description, const std::string& _error = ""); void fill(const StatusCode& sc); - void fillOrAppend(HttpStatusCode _code, const std::string& fullDetails, const std::string& appendDetail, const std::string& _reasonPhrase); + void fillOrAppend(HttpStatusCode _code, const std::string& fullDetails, const std::string& appendDetail, const std::string& _error); private: - void shrinkReasonPhrase(void); + void shrinkError(void); } OrionError; diff --git a/src/lib/rest/RestService.cpp b/src/lib/rest/RestService.cpp index fe7d3564b0..ec3341910c 100644 --- a/src/lib/rest/RestService.cpp +++ b/src/lib/rest/RestService.cpp @@ -556,7 +556,7 @@ static std::string restService(ConnectionInfo* ciP, RestService* serviceV) if (compErrorDetect(ciP->apiVersion, components, compV, &oe)) { - alarmMgr.badInput(clientIp, oe.details); + alarmMgr.badInput(clientIp, oe.description); ciP->httpStatusCode = SccBadRequest; restReply(ciP, oe.smartRender(ciP->apiVersion)); return "URL PATH component error"; diff --git a/src/lib/rest/rest.cpp b/src/lib/rest/rest.cpp index c5503a1b2d..c3a4a2ef71 100644 --- a/src/lib/rest/rest.cpp +++ b/src/lib/rest/rest.cpp @@ -1555,15 +1555,15 @@ static MHD_Result connectionTreat if (textAccepted) { - oe.details = "acceptable MIME types: application/json, text/plain"; + oe.description = "acceptable MIME types: application/json, text/plain"; } else { - oe.details = "acceptable MIME types: application/json"; + oe.description = "acceptable MIME types: application/json"; } ciP->httpStatusCode = oe.code; - alarmMgr.badInput(clientIp, oe.details); + alarmMgr.badInput(clientIp, oe.description); restReply(ciP, oe.smartRender(ciP->apiVersion)); return MHD_YES; } @@ -1581,15 +1581,15 @@ static MHD_Result connectionTreat if (textAccepted) { - oe.details = "acceptable MIME types: application/json, text/plain"; + oe.description = "acceptable MIME types: application/json, text/plain"; } else { - oe.details = "acceptable MIME types: application/json"; + oe.description = "acceptable MIME types: application/json"; } ciP->httpStatusCode = oe.code; - alarmMgr.badInput(clientIp, oe.details); + alarmMgr.badInput(clientIp, oe.description); restReply(ciP, oe.smartRender(ciP->apiVersion)); return MHD_YES; } diff --git a/src/lib/serviceRoutines/postQueryContext.cpp b/src/lib/serviceRoutines/postQueryContext.cpp index c92c029af3..10f5ab4fd5 100644 --- a/src/lib/serviceRoutines/postQueryContext.cpp +++ b/src/lib/serviceRoutines/postQueryContext.cpp @@ -424,7 +424,7 @@ static bool queryForward if (b == false) { - alarmMgr.forwardingError(url, "error parsing reply from context provider: " + oe.details); + alarmMgr.forwardingError(url, "error parsing reply from context provider: " + oe.description); parseData.qcr.res.release(); parseData.qcrs.res.release(); return false; diff --git a/src/lib/serviceRoutinesV2/getEntityAttribute.cpp b/src/lib/serviceRoutinesV2/getEntityAttribute.cpp index f1f1e7fb17..ae9145b4ed 100644 --- a/src/lib/serviceRoutinesV2/getEntityAttribute.cpp +++ b/src/lib/serviceRoutinesV2/getEntityAttribute.cpp @@ -105,11 +105,11 @@ std::string getEntityAttribute TIMED_RENDER(answer = oe.toJson()); } - if (oe.reasonPhrase == ERROR_TOO_MANY) + if (oe.error == ERROR_TOO_MANY) { ciP->httpStatusCode = SccConflict; } - else if (oe.reasonPhrase == ERROR_NOT_FOUND) + else if (oe.error == ERROR_NOT_FOUND) { ciP->httpStatusCode = SccContextElementNotFound; // Attribute to be precise! } diff --git a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp index 25713cb229..e49c262f95 100644 --- a/src/lib/serviceRoutinesV2/putEntityAttribute.cpp +++ b/src/lib/serviceRoutinesV2/putEntityAttribute.cpp @@ -93,7 +93,7 @@ std::string putEntityAttribute if (parseDataP->upcrs.res.oe.code != SccNone ) { TIMED_RENDER(answer = parseDataP->upcrs.res.oe.toJson()); - alarmMgr.badInput(clientIp, parseDataP->upcrs.res.oe.details); + alarmMgr.badInput(clientIp, parseDataP->upcrs.res.oe.description); ciP->httpStatusCode = parseDataP->upcrs.res.oe.code; } else diff --git a/src/lib/serviceRoutinesV2/serviceRoutinesCommon.cpp b/src/lib/serviceRoutinesV2/serviceRoutinesCommon.cpp index 083072e0cf..6aa2cdbc96 100644 --- a/src/lib/serviceRoutinesV2/serviceRoutinesCommon.cpp +++ b/src/lib/serviceRoutinesV2/serviceRoutinesCommon.cpp @@ -135,16 +135,16 @@ RenderFormat getRenderFormat(std::map& uriParamOptions) */ void adaptErrorCodeForSingleEntityOperation(OrionError* oeP, bool singleAttributeCheck) { - if ((oeP->code == SccContextElementNotFound) & (oeP->reasonPhrase == ERROR_NOT_FOUND)) + if ((oeP->code == SccContextElementNotFound) & (oeP->error == ERROR_NOT_FOUND)) { // In single entity attribute operations (e.g. DELETE /v2/entities/E/attrs/A) when the entity doesn't exist - oeP->details = ERROR_DESC_NOT_FOUND_ENTITY; + oeP->description = ERROR_DESC_NOT_FOUND_ENTITY; } - else if (singleAttributeCheck && (oeP->code == SccInvalidModification) & (oeP->reasonPhrase == ERROR_UNPROCESSABLE)) + else if (singleAttributeCheck && (oeP->code == SccInvalidModification) & (oeP->error == ERROR_UNPROCESSABLE)) { // In single entity attribute operations (e.g. DELETE /v2/entities/E/attrs/A) when the entity exists but the attribute doesn't oeP->code = SccContextElementNotFound; - oeP->reasonPhrase = ERROR_NOT_FOUND; - oeP->details = ERROR_DESC_NOT_FOUND_ATTRIBUTE; + oeP->error = ERROR_NOT_FOUND; + oeP->description = ERROR_DESC_NOT_FOUND_ATTRIBUTE; } } \ No newline at end of file diff --git a/test/unittests/mongoBackend/mongoCreateSubscription_test.cpp b/test/unittests/mongoBackend/mongoCreateSubscription_test.cpp index 4cef7e95b2..688b1e1b3c 100644 --- a/test/unittests/mongoBackend/mongoCreateSubscription_test.cpp +++ b/test/unittests/mongoBackend/mongoCreateSubscription_test.cpp @@ -96,8 +96,8 @@ TEST(mongoCreateSubscriptions, createSubscriptionNotCustomOK) /* Check response is as expected */ EXPECT_EQ(SccNone, oe.code); - EXPECT_EQ("", oe.reasonPhrase); - EXPECT_EQ("", oe.details); + EXPECT_EQ("", oe.error); + EXPECT_EQ("", oe.description); DBClientBase* connection = getMongoConnection(); @@ -193,8 +193,8 @@ TEST(mongoCreateSubscriptions, createSubscriptionCustomOK) /* Check response is as expected */ EXPECT_EQ(SccNone, oe.code); - EXPECT_EQ("", oe.reasonPhrase); - EXPECT_EQ("", oe.details); + EXPECT_EQ("", oe.error); + EXPECT_EQ("", oe.description); DBClientBase* connection = getMongoConnection(); diff --git a/test/unittests/mongoBackend/mongoGetSubscriptions_test.cpp b/test/unittests/mongoBackend/mongoGetSubscriptions_test.cpp index 90ef9de3a8..07c0a6cbc0 100644 --- a/test/unittests/mongoBackend/mongoGetSubscriptions_test.cpp +++ b/test/unittests/mongoBackend/mongoGetSubscriptions_test.cpp @@ -140,8 +140,8 @@ TEST(mongoListSubscriptions, getAllSubscriptionsV1Info) /* Check response is as expected */ EXPECT_EQ(SccOk, oe.code); - EXPECT_EQ("OK", oe.reasonPhrase); - EXPECT_EQ("", oe.details); + EXPECT_EQ("OK", oe.error); + EXPECT_EQ("", oe.description); ASSERT_EQ(3, subs.size()); Subscription s; @@ -244,8 +244,8 @@ TEST(mongoGetSubscription, getSubscription) /* Check response is as expected */ EXPECT_EQ(SccOk, oe.code); - EXPECT_EQ("OK", oe.reasonPhrase); - EXPECT_EQ("", oe.details); + EXPECT_EQ("OK", oe.error); + EXPECT_EQ("", oe.description); std::vector ents; std::vector attrs; diff --git a/test/unittests/rest/OrionError_test.cpp b/test/unittests/rest/OrionError_test.cpp index 245d651a11..83a697d112 100644 --- a/test/unittests/rest/OrionError_test.cpp +++ b/test/unittests/rest/OrionError_test.cpp @@ -53,20 +53,20 @@ TEST(OrionError, all) ci.outMimeType = JSON; EXPECT_EQ(SccNone, e0.code); - EXPECT_EQ("", e0.reasonPhrase); - EXPECT_EQ("", e0.details); + EXPECT_EQ("", e0.error); + EXPECT_EQ("", e0.description); EXPECT_EQ(SccOk, e1.code); - EXPECT_EQ("OK", e1.reasonPhrase); - EXPECT_EQ("no details 3", e1.details); + EXPECT_EQ("OK", e1.error); + EXPECT_EQ("no details 3", e1.description); EXPECT_EQ(sc.code, e3.code); - EXPECT_EQ(sc.reasonPhrase, e3.reasonPhrase); - EXPECT_EQ(sc.details, e3.details); + EXPECT_EQ(sc.reasonPhrase, e3.error); + EXPECT_EQ(sc.details, e3.description); EXPECT_EQ(SccOk, e4.code); - EXPECT_EQ("OK", e4.reasonPhrase); - EXPECT_EQ("Good Request", e4.details); + EXPECT_EQ("OK", e4.error); + EXPECT_EQ("Good Request", e4.description); ci.outMimeType = JSON; From 82ed507f8e409c6e92ee00a3b2bb8a06affda811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 8 Jan 2024 16:27:22 +0100 Subject: [PATCH 059/390] FIX indent --- src/lib/jsonParseV2/parseRegistration.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/lib/jsonParseV2/parseRegistration.cpp b/src/lib/jsonParseV2/parseRegistration.cpp index af3261e537..c72d9df72a 100644 --- a/src/lib/jsonParseV2/parseRegistration.cpp +++ b/src/lib/jsonParseV2/parseRegistration.cpp @@ -58,8 +58,8 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccBadRequest; oeP->code = SccBadRequest; - oeP->error = ERROR_BAD_REQUEST; - oeP->description = "/dataProvided/ must be a JSON object"; + oeP->error = ERROR_BAD_REQUEST; + oeP->description = "/dataProvided/ must be a JSON object"; return false; } @@ -68,8 +68,8 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccBadRequest; oeP->code = SccBadRequest; - oeP->error = ERROR_BAD_REQUEST; - oeP->description = "/dataProvided/ is empty"; + oeP->error = ERROR_BAD_REQUEST; + oeP->description = "/dataProvided/ is empty"; return false; } @@ -81,7 +81,7 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccBadRequest; oeP->code = SccBadRequest; - oeP->error = ERROR_BAD_REQUEST; + oeP->error = ERROR_BAD_REQUEST; return false; } @@ -99,8 +99,8 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccNotImplemented; oeP->code = SccNotImplemented; - oeP->error = ERROR_NOTIMPLEMENTED; - oeP->description = ERROR_DESC_IDPATTERN_NOTSUPPORTED; + oeP->error = ERROR_NOTIMPLEMENTED; + oeP->description = ERROR_DESC_IDPATTERN_NOTSUPPORTED; return false; } @@ -109,8 +109,8 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccNotImplemented; oeP->code = SccNotImplemented; - oeP->error = ERROR_NOTIMPLEMENTED; - oeP->description = ERROR_DESC_TYPEPATTERN_NOTIMPLEMENTED; + oeP->error = ERROR_NOTIMPLEMENTED; + oeP->description = ERROR_DESC_TYPEPATTERN_NOTIMPLEMENTED; return false; } @@ -122,7 +122,7 @@ static bool dataProvidedParse { ciP->httpStatusCode = SccBadRequest; oeP->code = SccBadRequest; - oeP->error = ERROR_BAD_REQUEST; + oeP->error = ERROR_BAD_REQUEST; return false; } From 96035b85141d0f2a570ca4fb8ca9fc6e9519faaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 8 Jan 2024 16:29:55 +0100 Subject: [PATCH 060/390] FIX indent --- src/lib/mongoBackend/mongoUpdateContext.cpp | 2 +- src/lib/rest/OrionError.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/mongoBackend/mongoUpdateContext.cpp b/src/lib/mongoBackend/mongoUpdateContext.cpp index f3cea44ee8..c280ee6ca9 100644 --- a/src/lib/mongoBackend/mongoUpdateContext.cpp +++ b/src/lib/mongoBackend/mongoUpdateContext.cpp @@ -221,7 +221,7 @@ HttpStatusCode mongoUpdateContext // Other cases follow the usual response processing flow (whatever it is :) if (updateCoverage == UC_PARTIAL) { - responseP->oe.code = SccInvalidModification; + responseP->oe.code = SccInvalidModification; responseP->oe.error = ERROR_PARTIAL_UPDATE; } diff --git a/src/lib/rest/OrionError.cpp b/src/lib/rest/OrionError.cpp index dfcb40af71..710e1fa810 100644 --- a/src/lib/rest/OrionError.cpp +++ b/src/lib/rest/OrionError.cpp @@ -39,10 +39,10 @@ */ OrionError::OrionError() { - code = SccNone; - error = ""; - description = ""; - filled = false; + code = SccNone; + error = ""; + description = ""; + filled = false; } @@ -194,13 +194,13 @@ std::string OrionError::toJsonV1(void) // // OrionError is NEVER part of any other payload, so the JSON start/end braces must be added here // - out += startTag("orionError", false); - out += valueTag("code", code, true); - out += valueTag("reasonPhrase", error, !description.empty()); + out += startTag("orionError", false); + out += valueTag("code", code, true); + out += valueTag("reasonPhrase", error, !description.empty()); if (!description.empty()) { - out += valueTag("details", description); + out += valueTag("details", description); } out += endTag(); From 4cb6515b7e243a8bbdc0f96e5e0c0c9a13e18c34 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Thu, 11 Jan 2024 03:10:21 +0900 Subject: [PATCH 061/390] Fix typo --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 671606686b..a3aaaf557c 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -4703,7 +4703,7 @@ For action type `update` or `delete`: } ``` -For action type `appendStrict`:` +For action type `appendStrict`: * If in *all* entities in `entities` there was a *full fail* due to existing attributes: From a2e280d6efc181e8ef57a90fa8b63ee075fa0375 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Thu, 11 Jan 2024 03:12:14 +0900 Subject: [PATCH 062/390] (JP) ADD doc about PartialUpdates reporting in error responses (#4447) --- doc/manuals.jp/orion-api.md | 328 +++++++++++++++++++++++++++++++++--- 1 file changed, 305 insertions(+), 23 deletions(-) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index bc4a4f0c7d..07055c136a 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -2822,6 +2822,9 @@ _**レスポンス・コード**_ - 成功したオペレーションでは、200 OK を使用します - エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティが見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください _**レスポンス・ヘッダ**_ @@ -2829,8 +2832,17 @@ _**レスポンス・ヘッダ**_ _**レスポンス・ペイロード**_ -レスポンスは、id で識別されるエンティティを表すオブジェクトです。オブジェクトは、JSON エンティティ表現形式 -([JSON エンティティ表現](#json-entity-representation)のセクションと、 +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +エンティティが見つかった場合、レスポンスは ID で識別されるエンティティを表すオブジェクトになります。 +オブジェクトは、JSON エンティティ表現形式 ([JSON エンティティ表現](#json-entity-representation)のセクションと、 [簡略化されたエンティティ表現](#simplified-entity-representation)および、[部分的表現](#partial-representations) のセクションで説明されています) に従います。 @@ -2906,8 +2918,9 @@ _**リクエスト・ヘッダ**_ _**レスポンス・コード**_ - 成功したオペレーションでは、200 OK を使用します -- エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 - [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティが見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください _**レスポンス・ヘッダ**_ @@ -2915,8 +2928,17 @@ _**レスポンス・ヘッダ**_ _**レスポンス・ペイロード**_ -ペイロードは、URL パラメータの id で識別されるエンティティを表すオブジェクトです。オブジェクトは JSON -エンティティ表現形式 ([JSON エンティティ表現](#json-entity-representation)のセクションと、 +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +エンティティが見つかった場合、ペイロードは、URL パラメータの ID によって識別されるエンティティを表すオブジェクトです。 +オブジェクトは JSON エンティティ表現形式 ([JSON エンティティ表現](#json-entity-representation)のセクションと、 [簡略化されたエンティティ表現](#simplified-entity-representation)および、[部分表現](#partial-representations) のセクションで説明されています) に従いますが、`id` フィールドと `type` フィールドは省略されます。 @@ -3014,8 +3036,45 @@ JSON エンティティ表現形式 ([JSON エンティティ表現](#json-entit _**レスポンス・コード**_ - 成功したオペレーションでは、204 No Content を使用します -- エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 - [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティが見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - `append` オプションが使用されたときに、属性が存在していて、422 Unprocessable Content になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください + +_**レスポンス・ペイロード**_ + +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +`append` オプションが使用されたときに、*すべて* の属性が存在する場合: + +``` +{ + "description": "one or more of the attributes in the request already exist: E/T - [ A, B ]", + "error": "Unprocessable" +} +``` + +`append` オプションが使用されたときに、*一部* (すべてではない) 属性が存在する場合 (部分更新): + +``` +{ + "description": "one or more of the attributes in the request already exist: E/T - [ B ]", + "error": "PartialUpdate" +} +``` + +`description` 内のエンティティ型は、リクエストにエンティティ型が含まれている場合にのみ表示されます。それ以外の場合は省略されます: + +``` +"description": "one or more of the attributes in the request already exist: E - [ B ]", +``` @@ -3081,6 +3140,46 @@ _**レスポンス**_ - 成功したオペレーションでは、204 No Content を使用します - エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティが見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - 属性が存在しないで、422 Unprocessable Content になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください + +_**レスポンス・ペイロード**_ + +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +リクエストに属性が *何も存在しない* 場合: + +``` +{ + "description": "do not exist: E/T - [ C, D ]", + "error": "Unprocessable" +} +``` + +*一部* (すべてではない) 属性が存在しない場合 (部分更新): + +``` +{ + "description": "do not exist: E/T - [ C ]", + "error": "PartialUpdate" +} +``` + + +`description` 内のエンティティ型は、リクエストにエンティティ型が含まれている場合にのみ表示されます。それ以外の場合は省略されます: + +``` +"description": "do not exist: E - [ C ]", +``` @@ -3143,8 +3242,20 @@ _**リクエスト・ペイロード**_ _**レスポンス・コード**_ - 成功したオペレーションでは、204 No Content を使用します -- エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 - [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティが見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください + +_**レスポンス・ペイロード**_ + +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` @@ -3176,8 +3287,20 @@ _**リクエスト・ヘッダ**_ _**レスポンス・コード**_ - 成功したオペレーションでは、204 No Content を使用します -- エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 - [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティが見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください + +_**レスポンス・ペイロード**_ + +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` @@ -3308,8 +3431,29 @@ _**リクエスト・ペイロード**_ _**レスポンス・コード**_ - 成功したオペレーションでは、204 No Content を使用します -- エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 - [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティまたは属性が見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください + +_**レスポンス・ペイロード**_ + +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +属性が見つからない場合: + +``` +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} +``` @@ -3342,8 +3486,29 @@ _**リクエスト・ヘッダ**_ _**レスポンス・コード**_ - 正常なオペレーションには、204 No Content を使用します -- エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 - [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティまたは属性が見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください + +_**レスポンス・ペイロード**_ + +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +属性が見つからない場合: + +``` +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} +``` @@ -3387,8 +3552,10 @@ _**リクエスト・ヘッダ**_ _**レスポンス・コード**_ - 成功したオペレーションでは、200 OK を使用します -- エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 - [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティまたは属性が見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - コンテンツが受け入れられないときに、406 Not Acceptable になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください _**レスポンス・ヘッダ**_ @@ -3396,7 +3563,25 @@ _**レスポンス・ヘッダ**_ _**レスポンス・ペイロード**_ -レスポンス・ペイロードは、オブジェクト、配列、文字列、数値、ブール値、または属性の値を持つ null にすることができます。 +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +属性が見つからない場合: + +``` +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} +``` + +エンティティと属性の両方が見つかった場合、レスポンス・ペイロードはオブジェクト、配列、文字列、数値、ブール値、または属性の値を持つ null になります。 - 属性値が JSON 配列またはオブジェクトの場合: - `Accept` ヘッダを `application/json` または `text/plain` に展開できる場合、レスポンス・タイプが @@ -3486,8 +3671,29 @@ _**リクエスト・ペイロード**_ _**レスポンス・コード**_ - 成功したオペレーションでは、200 OK を使用します -- エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 - [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - エンティティまたは属性が見つからず、404 Not Found になる場合 (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください + +_**レスポンス・ペイロード**_ + +エンティティが見つからない場合: + +``` +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} +``` + +属性が見つからない場合: + +``` +{ + "description": "The entity does not have such an attribute", + "error": "NotFound" +} +``` @@ -4548,8 +4754,84 @@ _**リクエスト・ペイロード**_ _**レスポンス・コード**_ - 成功したオペレーションでは、204 No Content を使用します -- エラーは、2xx 以外のものと、(オプションで) エラー・ペイロードを使用します。詳細については、 - [エラー・レスポンス](#error-responses)のサブセクションを参照してください +- エラーでは、2xx 以外のコードとエラー・ペイロードが使用されます: + - `update`, `delete` または `replace` 時に `entities` フィールドにエンティティが存在しなければ、404 Not Found になります (次のサブセクションを参照) + - そのほかの場合は、422 Unprocessable Content になります (次のサブセクションを参照) + - [エラー・レスポンス](#error-responses) の一般ドキュメントで追加のケースを確認してください + +_**レスポンス・ペイロード**_ + +アクション・タイプが `replace` の場合: + +- `entities` 内にエンティティが *何も*存在しない場合: + +``` +{ + "description": "do not exist: F/T - [entity itself], G/T [entity itself]", + "error": "NotFound" +} +``` + +- `entities` 内にエンティティの *いずれか (すべてではない)* が存在しない場合 (部分更新): + +``` +{ + "description": "do not exist: G/T - [entity itself]", + "error": "PartialUpdate" +} +``` + +アクション・タイプが `update` または `delete` の場合: + +- `entities` 内にエンティティが *何も*存在しない場合: + +``` +{ + "description": "do not exist: F/T - [entity itself], G/T [entity itself]", + "error": "NotFound" +} +``` + +- `entities` に少なくとも1つのエンティティが存在し、既存のエンティティの *すべて* で属性の欠落により *完全な失敗* が発生した場合: + +``` +{ + "description": "do not exist: E/T - [ C, D ], G/T [entity itself]", + "error": "Unprocessable" +} +``` + +- `entities` 内に少なくとも1つのエンティティが存在し、既存のエンティティの *少なくとも1つ*に、*少なくとも*1つの属性が存在するが、すべてのエンティティが存在するわけではない、 + またはすべてのエンティティが存在するが、少なくとも1つのエンティティに少なくとも1つの属性が欠落している場合 (部分更新): + +``` +{ + "description": "do not exist: E/T - [ D ], G/T [entity itself]", + "error": "PartialUpdate" +} +``` + +アクション・タイプが `appendStrict` の場合: + +- `entities` 内の *すべて* エンティティで、既存の属性が原因で*完全な失敗*があった場合: + +{ + "description": "one or more of the attributes in the request already exist: E1/T - [ A, B ], E2/T - [ A, B ]", + "error": "Unprocessable" +} + +- *少なくとも*1つの属性の `entities` 内の *少なくとも 1 つのエンティティ* で成功したが、 `entities` 内のすべてのエンティティが完全に成功したわけではない場合 (部分更新): + +{ + "description": "one or more of the attributes in the request already exist: E2/T - [ A, B ]", + "error": "PartialUpdate" +} + +`description` 内のエンティティ型は、リクエストにエンティティ型が含まれている場合にのみ表示されます。それ以外の場合は省略されます: + +``` +"description": "one or more of the attributes in the request already exist: E2 - [ A, B ]" +``` From ae94b30e0205b81ced7985799b25dc757ea2e0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 11:56:15 +0100 Subject: [PATCH 063/390] FIX old stuff --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/admin/diagnosis.md | 43 +++++------------------ doc/manuals/orion-api.md | 1 + doc/manuals/user/known_limitations.md | 7 ++-- src/lib/serviceRoutinesV2/getEntities.cpp | 2 +- 5 files changed, 14 insertions(+), 40 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 40fdb20979..d3d9a8da79 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -11,6 +11,7 @@ - Fix: provide more informative error description in some error responses in update/delete operations - Fix: proper use of "PartialUpdate" (instead of "Unprocessed") in responses in the case of partial updates/deletions (#3499) - Fix: response 404 Not Found "NotFound" errors instead of 422 Unprocessable Content "Unprocessed" in the case of missing attribute in existing entity in attribute update operations +- Fix: "Internal Server Error" changed to "InternalServerError" in error responses - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) - Fix: improve logs in MongoDB query logic - Fix: false positive in log deprecation logic when entity name (or other literal) includes the token "v1" (#4454) diff --git a/doc/manuals/admin/diagnosis.md b/doc/manuals/admin/diagnosis.md index e9fd657bf6..f4a0117b77 100644 --- a/doc/manuals/admin/diagnosis.md +++ b/doc/manuals/admin/diagnosis.md @@ -271,45 +271,20 @@ The symptoms of a database connection problem are the following: - At startup time. The broker doesn't start and the following message appears in the log file: -` X@08:04:45 main[313]: MongoDB error` +``` +... msg=Database Startup Error (cannot connect to mongo - doing 100 retries with a 1000 millisecond interval) +... msg=Fatal Error (MongoDB error) + +``` - During broker operation. Error message like the following ones appear in the responses sent by the broker. ``` - - ... - "errorCode": { - "code": "500", - "reasonPhrase": "Database Error", - "details": "collection: ... - exception: Null cursor" - } - ... - - ... - "errorCode": { - "code": "500", - "reasonPhrase": "Database Error", - "details": "collection: ... - exception: socket exception [CONNECT_ERROR] for localhost:27017" - } - ... - - ... - "errorCode": { - "code": "500", - "reasonPhrase": "Database Error", - "details": "collection: ... - exception: socket exception [FAILED_STATE] for localhost:27017" - } - ... - - ... - "errorCode": { - "code": "500", - "reasonPhrase": "Database Error", - "details": "collection: ... - exception: DBClientBase::findN: transport error: localhost:27017 ns: orion.$cmd query: { .. }" - } - ... - +{ + "error": "InternalServerError", + "description": "Database Error ..." +} ``` In both cases, check that the connection to MonogDB is correctly diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 671606686b..17d4940dd6 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -458,6 +458,7 @@ The `error` reporting is as follows: + HTTP 411 Length Required corresponds to `ContentLengthRequired` (`411`) + HTTP 413 Request Entity Too Large corresponds to `RequestEntityTooLarge` (`413`) + HTTP 415 Unsupported Media Type corresponds to `UnsupportedMediaType` (`415`) ++ Internal errors use `InternalServerError` (`500`) ## Multi tenancy diff --git a/doc/manuals/user/known_limitations.md b/doc/manuals/user/known_limitations.md index 10342bd82c..6aeebeaba3 100644 --- a/doc/manuals/user/known_limitations.md +++ b/doc/manuals/user/known_limitations.md @@ -7,11 +7,8 @@ this limitation into account, you will get messages such the following ones: ``` { - "errorCode" : { - "code" : "413", - "reasonPhrase" : "Request Entity Too Large", - "details" : "payload size: 1500000, max size supported: 1048576" - } + "error": "RequestEntityTooLarge" + "description": "payload size: 1500000, max size supported: 1048576", } ``` diff --git a/src/lib/serviceRoutinesV2/getEntities.cpp b/src/lib/serviceRoutinesV2/getEntities.cpp index af112f841d..ef12312fbd 100644 --- a/src/lib/serviceRoutinesV2/getEntities.cpp +++ b/src/lib/serviceRoutinesV2/getEntities.cpp @@ -315,7 +315,7 @@ std::string getEntities { OrionError oe; entities.fill(parseDataP->qcrs.res, &oe); - TIMED_RENDER(answer = oe.toJson()); + TIMED_RENDER(answer = oe.smartRender(V2)); ciP->httpStatusCode = oe.code; } // 04. Render Entities response From 1feb7f2d51d675a494c2e1a369d059fc1f9a6916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 12:29:46 +0100 Subject: [PATCH 064/390] FIX do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations --- CHANGES_NEXT_RELEASE | 1 + src/lib/mongoBackend/mongoQueryTypes.cpp | 30 +- ...do_not_show_date_expires_in_get_types.test | 278 ++++++++++++++++++ 3 files changed, 303 insertions(+), 6 deletions(-) create mode 100644 test/functionalTest/cases/4451_do_not_show_date_expires_in_get_types/do_not_show_date_expires_in_get_types.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 40fdb20979..9980d5e3a5 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,6 +1,7 @@ - Add: servicePath field to builtin attributes (#2877) - Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388) - Add: notification payload in INFO log traces (#4449) +- Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: update forwarding was not working when entity type is not included in the request (#4460) - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) diff --git a/src/lib/mongoBackend/mongoQueryTypes.cpp b/src/lib/mongoBackend/mongoQueryTypes.cpp index 7d67598d7a..32e0fb23ce 100644 --- a/src/lib/mongoBackend/mongoQueryTypes.cpp +++ b/src/lib/mongoBackend/mongoQueryTypes.cpp @@ -597,17 +597,26 @@ HttpStatusCode mongoEntityTypes continue; } + std::string attrName = attrsArray[jx].String(); + + // dateExpires shouldn't not be included in the response, as it is a built in attribute + // (see issue #4451) + if (attrName == DATE_EXPIRES) + { + continue; + } + /* Note that we need and extra query() in the database (inside attributeType() function) to get each attribute type. * This could be inefficient, especially if the number of attributes is large */ if (!noAttrDetail) { std::vector attrTypes; - getAttributeTypes(tenant, servicePathV, entityType->type , attrsArray[jx].String(), &attrTypes); + getAttributeTypes(tenant, servicePathV, entityType->type , attrName, &attrTypes); for (unsigned int kx = 0; kx < attrTypes.size(); ++kx) { - ContextAttribute* ca = new ContextAttribute(attrsArray[jx].String(), attrTypes[kx], ""); + ContextAttribute* ca = new ContextAttribute(attrName, attrTypes[kx], ""); entityType->contextAttributeVector.push_back(ca); @@ -624,7 +633,7 @@ HttpStatusCode mongoEntityTypes // NOTE: here we add a ContextAttribute with empty type, as a marker for // this special condition of 'No Attribute Detail' // - ContextAttribute* caP = new ContextAttribute(attrsArray[jx].String(), "", ""); + ContextAttribute* caP = new ContextAttribute(attrName, "", ""); entityType->contextAttributeVector.push_back(caP); } } @@ -820,17 +829,26 @@ HttpStatusCode mongoAttributesForEntityType alarmMgr.dbErrorReset(); + std::string attrName = idField.String(); + + // dateExpires shouldn't not be included in the response, as it is a built in attribute + // (see issue #4451) + if (attrName == DATE_EXPIRES) + { + continue; + } + /* Note that we need and extra query() to the database (inside attributeType() function) to get each attribute type. * This could be unefficient, specially if the number of attributes is large */ if (!noAttrDetail) { std::vector attrTypes; - getAttributeTypes(tenant, servicePathV, entityType , idField.String(), &attrTypes); + getAttributeTypes(tenant, servicePathV, entityType , attrName, &attrTypes); for (unsigned int kx = 0; kx < attrTypes.size(); ++kx) { - ContextAttribute* ca = new ContextAttribute(idField.String(), attrTypes[kx], ""); + ContextAttribute* ca = new ContextAttribute(attrName, attrTypes[kx], ""); responseP->entityType.contextAttributeVector.push_back(ca); @@ -847,7 +865,7 @@ HttpStatusCode mongoAttributesForEntityType // NOTE: here we add a ContextAttribute with empty type, as a marker for // this special condition of 'No Attribute Detail' // - ContextAttribute* caP = new ContextAttribute(idField.String(), "", ""); + ContextAttribute* caP = new ContextAttribute(attrName, "", ""); responseP->entityType.contextAttributeVector.push_back(caP); } } diff --git a/test/functionalTest/cases/4451_do_not_show_date_expires_in_get_types/do_not_show_date_expires_in_get_types.test b/test/functionalTest/cases/4451_do_not_show_date_expires_in_get_types/do_not_show_date_expires_in_get_types.test new file mode 100644 index 0000000000..5f19118204 --- /dev/null +++ b/test/functionalTest/cases/4451_do_not_show_date_expires_in_get_types/do_not_show_date_expires_in_get_types.test @@ -0,0 +1,278 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Don't show dateExpires built-in attribute in GET /v2/types operations + +--SHELL-INIT-- +dbInit CB +brokerStart CB 100 + +--SHELL-- + +# +# 01. Create E1-T with attribute A and dateExpires +# 02. Create E2-T with attribute A, B and dateExpires +# 03. Create E3-T with attribute C and dateExpires +# 04. GET /v2/types and check dateExpires does not appear +# 05. GET /v2/types/T and check dateExpires does not appear +# 06. GET /v2/types?options=noAttrDetail and check dateExpires does not appear +# 07. GET /v2/types/T?options=noAttrDetail and check dateExpires does not appear +# + +echo "01. Create E1-T with attribute A and dateExpires" +echo "================================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "type": "Number", + "value": 1 + }, + "dateExpires":{ + "type": "DateTime", + "value": "2070-01-01T00:00:00Z" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "02. Create E2-T with attribute A, B and dateExpires" +echo "===================================================" +payload='{ + "id": "E2", + "type": "T", + "A": { + "type": "Number", + "value": 1 + }, + "B": { + "type": "Number", + "value": 1 + }, + "dateExpires":{ + "type": "DateTime", + "value": "2070-01-01T00:00:00Z" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Create E3-T with attribute C and dateExpires" +echo "================================================" +payload='{ + "id": "E3", + "type": "T", + "C": { + "type": "Number", + "value": 1 + }, + "dateExpires":{ + "type": "DateTime", + "value": "2070-01-01T00:00:00Z" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "04. GET /v2/types and check dateExpires does not appear" +echo "=======================================================" +orionCurl --url /v2/types +echo +echo + + +echo "05. GET /v2/types/T and check dateExpires does not appear" +echo "=========================================================" +orionCurl --url /v2/types/T +echo +echo + + +echo "06. GET /v2/types?options=noAttrDetail and check dateExpires does not appear" +echo "============================================================================" +orionCurl --url /v2/types?options=noAttrDetail +echo +echo + + +echo "07. GET /v2/types/T?options=noAttrDetail and check dateExpires does not appear" +echo "==============================================================================" +orionCurl --url /v2/types/T?options=noAttrDetail +echo +echo + + +--REGEXPECT-- +01. Create E1-T with attribute A and dateExpires +================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +02. Create E2-T with attribute A, B and dateExpires +=================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +03. Create E3-T with attribute C and dateExpires +================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E3?type=T +Content-Length: 0 + + + +04. GET /v2/types and check dateExpires does not appear +======================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 109 + +[ + { + "attrs": { + "A": { + "types": [ + "Number" + ] + }, + "B": { + "types": [ + "Number" + ] + }, + "C": { + "types": [ + "Number" + ] + } + }, + "count": 3, + "type": "T" + } +] + + +05. GET /v2/types/T and check dateExpires does not appear +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 96 + +{ + "attrs": { + "A": { + "types": [ + "Number" + ] + }, + "B": { + "types": [ + "Number" + ] + }, + "C": { + "types": [ + "Number" + ] + } + }, + "count": 3 +} + + +06. GET /v2/types?options=noAttrDetail and check dateExpires does not appear +============================================================================ +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 85 + +[ + { + "attrs": { + "A": { + "types": [] + }, + "B": { + "types": [] + }, + "C": { + "types": [] + } + }, + "count": 3, + "type": "T" + } +] + + +07. GET /v2/types/T?options=noAttrDetail and check dateExpires does not appear +============================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 72 + +{ + "attrs": { + "A": { + "types": [] + }, + "B": { + "types": [] + }, + "C": { + "types": [] + } + }, + "count": 3 +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB From 834379ecfb53ea912ec118847fc9811e462c7e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 14:22:31 +0100 Subject: [PATCH 065/390] FIX NGSI patching macro subscription covering completely the string always cast to string for id/type --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/orion-api.md | 4 +- src/lib/ngsiNotify/Notifier.cpp | 30 ++- ...tching_id_type_macro_always_stringfly.test | 206 ++++++++++++++++++ 4 files changed, 232 insertions(+), 9 deletions(-) create mode 100644 test/functionalTest/cases/4462_ngsi_patching_id_type_macro_always_stringfly/ngsi_patching_id_type_macro_always_stringfly.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 51c5369b83..b2cca13521 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -6,6 +6,7 @@ - Fix: update forwarding was not working when entity type is not included in the request (#4460) - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) +- Fix: in custom notification NGSI payload patching macro subscription covering completely the string where is used, always cast to string for id/type (#4462) - Fix: improve error traces (#4387) - Fix: on delete attributes operation, the lack of attribute in entity doesn't preclude the deletion of attributes which do exist (this way behaving same way as update attributes operation) - Fix: POST /v2/entities/E/attrs?options=append was wrongly creating entities diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index c23cebe0dc..11b86a3859 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2253,8 +2253,10 @@ Some notes to take into account when using `ngsi`: * It cannot be used in the key part of JSON objects, i.e. `"${key}": 10` will not work * It cannot be used in the attribute `type`. Only in the `value` macro replacements can be done. * If the macro *covers completely the string where is used*, then the JSON nature of the attribute value - is taken into account. For instance, `"value": "${temperature}"` resolves to `"value": 10` + is taken into account. For instance, `"value": "${temperature}"` resolves to `"value": 10`. if temperature attribute is a number or to `"value": "10"` if `temperature` attribute is a string. + * Exception to this is `id` and `type`. Given that entity id and type must be a string (as decribed + in [this section](#identifiers-syntax-restrictions)) the attribute value is always casted to string in this case. * If the macro *is only part of string where is used*, then the attribute value is always casted to string. For instance, `"value": "Temperature is: ${temperature}"` resolves to `"value": "Temperature is 10"` even if temperature attribute is a number. Note that if the diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 75865a4f4f..0eb136c8fa 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -209,6 +209,26 @@ static bool setJsonPayload } +/* **************************************************************************** +* +* removeQuotes - +* +* Entity id and type are special. Different from a attribute, they are always +* strings and cannot take a number, boolean, etc. as value. +*/ +inline std::string removeQuotes(std::string s) +{ + if (s[0] == '"') + { + return s.substr(1, s.size()-2); + } + else + { + return s; + } +} + + /* **************************************************************************** * @@ -247,10 +267,7 @@ static bool setNgsiPayload else { // If id is not found in the replacements macro, we use en.id. - // In addition, note we have to remove double quotes here given the - // values stored in replacements map are "raw strings" - std::string s = smartStringValue(ngsi.id, &replacements, '"' + en.id + '"'); - effectiveId = s.substr(1, s.size()-2); + effectiveId = removeQuotes(smartStringValue(ngsi.id, &replacements, '"' + en.id + '"')); } std::string effectiveType; @@ -261,10 +278,7 @@ static bool setNgsiPayload else { // If type is not found in the replacements macro, we use en.type. - // In addition, note we have to remove double quotes here given the - // values stored in replacements map are "raw strings" - std::string s = smartStringValue(ngsi.type, &replacements, '"' + en.type + '"'); - effectiveType = s.substr(1, s.size()-2); + effectiveType = removeQuotes(smartStringValue(ngsi.type, &replacements, '"' + en.type + '"')); } cer.entity.fill(effectiveId, effectiveType, en.isPattern, en.servicePath); diff --git a/test/functionalTest/cases/4462_ngsi_patching_id_type_macro_always_stringfly/ngsi_patching_id_type_macro_always_stringfly.test b/test/functionalTest/cases/4462_ngsi_patching_id_type_macro_always_stringfly/ngsi_patching_id_type_macro_always_stringfly.test new file mode 100644 index 0000000000..9795cc1ab7 --- /dev/null +++ b/test/functionalTest/cases/4462_ngsi_patching_id_type_macro_always_stringfly/ngsi_patching_id_type_macro_always_stringfly.test @@ -0,0 +1,206 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implie d warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +NGSI patching macro subscription covering completely the string always cast to string for id/type + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub1 with ngsi id and type with the value of idAtt and typeAtt attributes +# 02. Create entity E with idAtt=1 and typeAtt=2 as numbers +# 03. Update entity E with idAtt=110 and typeAtt=220 as numbers +# 04. Dump accumulator and see notifications with id "1" and type "2" and id "110" and type "220" +# + + +echo "01. Create custom sub1 with ngsi id and type with the value of idAtt and typeAtt attributes" +echo "===========================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "id": "${idAtt}", + "type": "${typeAtt}" + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E with idAtt=1 and typeAtt=2 as numbers" +echo "=========================================================" +payload='{ + "id": "E", + "type": "T", + "idAtt": { + "value": 1, + "type": "Number" + }, + "typeAtt": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E with idAtt=110 and typeAtt=220 as numbers" +echo "=============================================================" +payload='{ + "idAtt": { + "value": 110, + "type": "Number" + }, + "typeAtt": { + "value": 220, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs -X PATCH --payload "$payload" +echo +echo + + +echo '04. Dump accumulator and see notifications with id "1" and type "2" and id "110" and type "220"' +echo "===============================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub1 with ngsi id and type with the value of idAtt and typeAtt attributes +=========================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E with idAtt=1 and typeAtt=2 as numbers +========================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. Update entity E with idAtt=110 and typeAtt=220 as numbers +============================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications with id "1" and type "2" and id "110" and type "220" +=============================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 178 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "1", + "idAtt": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "type": "2", + "typeAtt": { + "metadata": {}, + "type": "Number", + "value": 2 + } + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 186 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "110", + "idAtt": { + "metadata": {}, + "type": "Number", + "value": 110 + }, + "type": "220", + "typeAtt": { + "metadata": {}, + "type": "Number", + "value": 220 + } + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 06d84b9c1e668dd3306520f2dddd73bf909973ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 14:23:42 +0100 Subject: [PATCH 066/390] Update doc/manuals/orion-api.md --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 11b86a3859..72f457e42d 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2253,7 +2253,7 @@ Some notes to take into account when using `ngsi`: * It cannot be used in the key part of JSON objects, i.e. `"${key}": 10` will not work * It cannot be used in the attribute `type`. Only in the `value` macro replacements can be done. * If the macro *covers completely the string where is used*, then the JSON nature of the attribute value - is taken into account. For instance, `"value": "${temperature}"` resolves to `"value": 10`. + is taken into account. For instance, `"value": "${temperature}"` resolves to `"value": 10` if temperature attribute is a number or to `"value": "10"` if `temperature` attribute is a string. * Exception to this is `id` and `type`. Given that entity id and type must be a string (as decribed in [this section](#identifiers-syntax-restrictions)) the attribute value is always casted to string in this case. From e57767934571517518f830a053f1bd043fc785b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 15:05:44 +0100 Subject: [PATCH 067/390] ADD log deprecation traces for usage of legacyForwarding mode in registrations --- CHANGES_NEXT_RELEASE | 1 + .../mongoBackend/mongoRegistrationCreate.cpp | 6 + src/lib/mongoBackend/mongoRegistrationGet.cpp | 4 + src/lib/serviceRoutines/postQueryContext.cpp | 6 + src/lib/serviceRoutines/postUpdateContext.cpp | 5 + .../log_deprecate_warning.test | 144 +++++++++++++++++- 6 files changed, 162 insertions(+), 4 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 51c5369b83..3850a2ae0b 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,6 +1,7 @@ - Add: servicePath field to builtin attributes (#2877) - Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388) - Add: notification payload in INFO log traces (#4449) +- Add: log deprecation traces for usage of legacyForwarding mode in registrations - Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: update forwarding was not working when entity type is not included in the request (#4460) diff --git a/src/lib/mongoBackend/mongoRegistrationCreate.cpp b/src/lib/mongoBackend/mongoRegistrationCreate.cpp index ea110161e5..3f31097cec 100644 --- a/src/lib/mongoBackend/mongoRegistrationCreate.cpp +++ b/src/lib/mongoBackend/mongoRegistrationCreate.cpp @@ -229,6 +229,12 @@ void mongoRegistrationCreate setForwardingMode(regP->provider.supportedForwardingMode, &bob); std::string format = (regP->provider.legacyForwardingMode == true)? "JSON" : "normalized"; + + if (logDeprecate && format == "JSON") + { + LM_W(("Deprecated usage of legacyForwarding mode in registration creation (regId: %s)", regIdP->c_str())); + } + setFormat(format, &bob); // diff --git a/src/lib/mongoBackend/mongoRegistrationGet.cpp b/src/lib/mongoBackend/mongoRegistrationGet.cpp index 09f93abd34..31e587e906 100644 --- a/src/lib/mongoBackend/mongoRegistrationGet.cpp +++ b/src/lib/mongoBackend/mongoRegistrationGet.cpp @@ -88,6 +88,10 @@ static void setProvider(ngsiv2::Registration* regP, const ngsiv2::ForwardingMode if (format == "JSON") { + if (logDeprecate) + { + LM_W(("Deprecated usage of legacyForwarding mode detected in existing registration (regId: %s)", regP->id.c_str())); + } regP->provider.legacyForwardingMode = true; } else diff --git a/src/lib/serviceRoutines/postQueryContext.cpp b/src/lib/serviceRoutines/postQueryContext.cpp index 10f5ab4fd5..2b7c4dcffc 100644 --- a/src/lib/serviceRoutines/postQueryContext.cpp +++ b/src/lib/serviceRoutines/postQueryContext.cpp @@ -185,6 +185,12 @@ static bool queryForward if (qcrP->providerFormat == PfJson) { op = "/queryContext"; + + if (logDeprecate) + { + LM_W(("Deprecated usage of legacyForwarding mode in query forwarding operation (regId: %s)", regId.c_str())); + } + TIMED_RENDER(payload = qcrP->toJsonV1()); } else diff --git a/src/lib/serviceRoutines/postUpdateContext.cpp b/src/lib/serviceRoutines/postUpdateContext.cpp index 54fd689774..6819e5554b 100644 --- a/src/lib/serviceRoutines/postUpdateContext.cpp +++ b/src/lib/serviceRoutines/postUpdateContext.cpp @@ -157,6 +157,11 @@ static bool updateForward TIMED_RENDER(payload = upcrP->toJsonV1(asJsonObject)); op = "/updateContext"; + + if (logDeprecate) + { + LM_W(("Deprecated usage of legacyForwarding mode in update forwarding operation (regId: %s)", regId.c_str())); + } } else { diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index a0a0704263..0a3bb88e4f 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -35,7 +35,11 @@ brokerStart CB 0 IPV4 -logDeprecate # 03. Create entity using NGSIv1 metadata location # 04. Create entity using NGSIv1 and geo:point # 05. Create entity using NGSIv2 and geo:point -# 06. Get WARNING trace in logs +# 06. Create registration using legacyForwarding mode +# 07. Retrieve registrations (which uses legacyForwarding mode) +# 08. Forwarded query using legacyForwarding mode +# 09. Update query using legacyForwarding mode +# 10. Get WARNING trace in logs # echo "01. Query E1-T1" @@ -130,9 +134,62 @@ echo echo -echo "06. Get WARNING trace in logs" +echo "06. Create registration using legacyForwarding mode" +echo "===================================================" +payload='{ + "dataProvided": { + "entities": [ + { + "id": "E1", + "type": "T1" + } + ], + "attrs": [ "A1" ] + }, + "provider": { + "http": { + "url": "http://localhost:'${CP1_PORT}'/v2" + }, + "legacyForwarding": true + }, + "status": "active" +}' +orionCurl --url /v2/registrations --payload "$payload" +echo +echo + + +REG_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "07. Retrieve registrations (which uses legacyForwarding mode)" +echo "=============================================================" +orionCurl --url /v2/registrations +echo +echo + + +echo "08. Forwarded query using legacyForwarding mode" +echo "===============================================" +orionCurl --url /v2/entities/E1/attrs/A1?type=T1 +echo +echo + + +echo "09. Update query using legacyForwarding mode" +echo "============================================" +payload='{ + "value": 1, + "type": "Number" +}' +orionCurl --url /v2/entities/E1/attrs/A1?type=T1 -X PUT --payload "$payload" +echo +echo + + +echo "10. Get WARNING trace in logs" echo "=============================" -cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' +cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' | sed -e "s/$REG_ID/REG_ID/g" echo echo @@ -252,7 +309,79 @@ Content-Length: 0 -06. Get WARNING trace in logs +06. Create registration using legacyForwarding mode +=================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/registrations/REGEX([0-9a-f\-]{24}) +Content-Length: 0 + + + +07. Retrieve registrations (which uses legacyForwarding mode) +============================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 233 + +[ + { + "dataProvided": { + "attrs": [ + "A1" + ], + "entities": [ + { + "id": "E1", + "type": "T1" + } + ] + }, + "id": "REGEX([0-9a-f\-]{24})", + "provider": { + "http": { + "url": "http://localhost:9801/v2" + }, + "legacyForwarding": true, + "supportedForwardingMode": "all" + }, + "status": "active" + } +] + + +08. Forwarded query using legacyForwarding mode +=============================================== +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +09. Update query using legacyForwarding mode +============================================ +HTTP/1.1 404 Not Found +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +10. Get WARNING trace in logs ============================= Deprecated NGSIv1 request received: POST /v1/queryContext, request payload (48 bytes): { "entities": [ { "type": "T1", "id": "E1" } ] }, response code: 200 Deprecated NGSIv1 request received: GET /v1/contextEntities/E/attributes/A, response code: 200 @@ -261,6 +390,13 @@ Deprecated NGSIv1 request received: POST /v1/updateContext, request payload (279 Deprecated usage of geo:point detected in attribute location at entity update, please use geo:json instead Deprecated NGSIv1 request received: POST /v1/updateContext, request payload (208 bytes): { "contextElements": [ { "type": "City", "isPattern": "false", "id": "Barcelona", "attributes": [ { "name": "location", "type": "geo:point", "value": "40.418889, -3.691944" } ] } ], "updateAction": "APPEND" }, response code: 200 Deprecated usage of geo:point detected in attribute location at entity update, please use geo:json instead +Deprecated usage of legacyForwarding mode in registration creation (regId: REG_ID) +Deprecated usage of legacyForwarding mode detected in existing registration (regId: REG_ID) +Deprecated usage of legacyForwarding mode in query forwarding operation (regId: REG_ID) +Raising alarm ForwardingError localhost:9801/v2/queryContext: forwarding failure for sender-thread: Couldn't connect to server +Deprecated usage of legacyForwarding mode in update forwarding operation (regId: REG_ID) +Raising alarm ForwardingError localhost:9801/v2/updateContext: forwarding failure for sender-thread: Couldn't connect to server +Raising alarm BadInput 127.0.0.1: The requested entity has not been found. Check type and id --TEARDOWN-- From 19b32f0daebcdb4d10fdb29d731330444fcb83cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 17:22:40 +0100 Subject: [PATCH 068/390] ADD log deprecation traces for usage of attrsFormat legacy in subscriptions --- CHANGES_NEXT_RELEASE | 1 + .../mongoBackend/MongoCommonSubscription.cpp | 5 + .../mongoBackend/mongoGetSubscriptions.cpp | 5 + src/lib/ngsiNotify/Notifier.cpp | 10 ++ .../log_deprecate_warning.test | 149 +++++++++++++++++- 5 files changed, 166 insertions(+), 4 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 3850a2ae0b..e680cd79c2 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -2,6 +2,7 @@ - Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388) - Add: notification payload in INFO log traces (#4449) - Add: log deprecation traces for usage of legacyForwarding mode in registrations +- Add: log deprecation traces for usage of attrsFormat legacy in subscriptions - Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: update forwarding was not working when entity type is not included in the request (#4460) diff --git a/src/lib/mongoBackend/MongoCommonSubscription.cpp b/src/lib/mongoBackend/MongoCommonSubscription.cpp index dd6bcf3101..1994c5a688 100644 --- a/src/lib/mongoBackend/MongoCommonSubscription.cpp +++ b/src/lib/mongoBackend/MongoCommonSubscription.cpp @@ -593,6 +593,11 @@ void setFormat(const Subscription& sub, orion::BSONObjBuilder* b) { std::string format = renderFormatToString(sub.attrsFormat); + if (logDeprecate && sub.attrsFormat == NGSI_V1_LEGACY) + { + LM_W(("Deprecated usage of notification legacy format in subscription creation or modification (subId: %s)", sub.id.c_str())); + } + b->append(CSUB_FORMAT, format); LM_T(LmtMongo, ("Subscription format: %s", format.c_str())); } diff --git a/src/lib/mongoBackend/mongoGetSubscriptions.cpp b/src/lib/mongoBackend/mongoGetSubscriptions.cpp index e82588166d..c6cbe38e15 100644 --- a/src/lib/mongoBackend/mongoGetSubscriptions.cpp +++ b/src/lib/mongoBackend/mongoGetSubscriptions.cpp @@ -212,6 +212,11 @@ static void setNotification(Subscription* subP, const orion::BSONObj& r, const s // Attributes format subP->attrsFormat = r.hasField(CSUB_FORMAT)? stringToRenderFormat(getStringFieldF(r, CSUB_FORMAT)) : NGSI_V1_LEGACY; + if (logDeprecate && subP->attrsFormat == NGSI_V1_LEGACY) + { + LM_W(("Deprecated usage of notification legacy format detected in existing subscription (subId: %s)", subP->id.c_str())); + } + // // Check values from subscription cache, update object from cache-values if necessary diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 75865a4f4f..e3331a1c97 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -154,6 +154,11 @@ static bool setPayload if (*renderFormatP == NGSI_V1_LEGACY) { + if (logDeprecate) + { + LM_W(("Deprecated usage of notification legacy format in notification (subId: %s)", subscriptionId.c_str())); + } + *payloadP = ncr.toJsonV1(false, attrsFilter, blacklist, metadataFilter); } else @@ -670,6 +675,11 @@ SenderThreadParams* Notifier::buildSenderParams std::string payloadString; if (renderFormat == NGSI_V1_LEGACY) { + if (logDeprecate) + { + LM_W(("Deprecated usage of notification legacy format in notification (subId: %s)", subId.c_str())); + } + bool asJsonObject = (ci.uriParam[URI_PARAM_ATTRIBUTE_FORMAT] == "object" && ci.outMimeType == JSON); payloadString = ncr.toJsonV1(asJsonObject, attrsFilter, blacklist, metadataFilter); } diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index 0a3bb88e4f..4dd5952ab1 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -29,6 +29,9 @@ brokerStart CB 0 IPV4 -logDeprecate --SHELL-- +# +# Note there is not a "Update registration using legacyForwarding mode" +# as the registrations update operation is not implemented (issue #3007) # # 01. Query E1-T1 # 02. GET /v1/contextEntities/E/attributes/A @@ -39,7 +42,11 @@ brokerStart CB 0 IPV4 -logDeprecate # 07. Retrieve registrations (which uses legacyForwarding mode) # 08. Forwarded query using legacyForwarding mode # 09. Update query using legacyForwarding mode -# 10. Get WARNING trace in logs +# 10. Create subscription using attrsFormat legacy +# 11. Retrieve subscriptions (which uses attrsFormat legacy) +# 12. Update subscription using attrsFormat legacy +# 13. Trigger notification using attrsFormat legacy +# 14. Get WARNING trace in logs # echo "01. Query E1-T1" @@ -187,9 +194,72 @@ echo echo -echo "10. Get WARNING trace in logs" +echo "10. Create subscription using attrsFormat legacy" +echo "================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "E2", + "type": "T2" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:1234" + }, + "attrsFormat": "legacy" + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "11. Retrieve subscriptions (which uses attrsFormat legacy)" +echo "==========================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "12. Update subscription using attrsFormat legacy" +echo "================================================" +payload='{ + "notification": { + "http": { + "url": "http://localhost:1234" + }, + "attrsFormat": "legacy" + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID -X PATCH --payload "$payload" +echo +echo + + +echo "13. Trigger notification using attrsFormat legacy" +echo "=================================================" +payload='{ + "id": "E2", + "type": "T2", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "14. Get WARNING trace in logs" echo "=============================" -cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' | sed -e "s/$REG_ID/REG_ID/g" +cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' | sed -e "s/$REG_ID/REG_ID/g" | sed -e "s/$SUB_ID/SUB_ID/g" echo echo @@ -381,7 +451,72 @@ Content-Length: 95 } -10. Get WARNING trace in logs +10. Create subscription using attrsFormat legacy +================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f\-]{24}) +Content-Length: 0 + + + +11. Retrieve subscriptions (which uses attrsFormat legacy) +========================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 288 + +[ + { + "id": "REGEX([0-9a-f\-]{24})", + "notification": { + "attrs": [], + "attrsFormat": "legacy", + "covered": false, + "http": { + "url": "http://localhost:1234" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "id": "E2", + "type": "T2" + } + ] + } + } +] + + +12. Update subscription using attrsFormat legacy +================================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +13. Trigger notification using attrsFormat legacy +================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T2 +Content-Length: 0 + + + +14. Get WARNING trace in logs ============================= Deprecated NGSIv1 request received: POST /v1/queryContext, request payload (48 bytes): { "entities": [ { "type": "T1", "id": "E1" } ] }, response code: 200 Deprecated NGSIv1 request received: GET /v1/contextEntities/E/attributes/A, response code: 200 @@ -397,6 +532,12 @@ Raising alarm ForwardingError localhost:9801/v2/queryContext: forwarding failure Deprecated usage of legacyForwarding mode in update forwarding operation (regId: REG_ID) Raising alarm ForwardingError localhost:9801/v2/updateContext: forwarding failure for sender-thread: Couldn't connect to server Raising alarm BadInput 127.0.0.1: The requested entity has not been found. Check type and id +Releasing alarm BadInput 127.0.0.1 +Deprecated usage of notification legacy format in subscription creation or modification (subId: ) +Deprecated usage of notification legacy format detected in existing subscription (subId: SUB_ID) +Deprecated usage of notification legacy format in subscription creation or modification (subId: SUB_ID) +Deprecated usage of notification legacy format in notification (subId: SUB_ID) +Raising alarm NotificationError localhost:1234/: notification failure for queue worker: Couldn't connect to server --TEARDOWN-- From 570abc5ed2effc508a6270a6f6d6254255dd88f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 17:32:48 +0100 Subject: [PATCH 069/390] FIX minor --- src/lib/mongoBackend/MongoCommonSubscription.cpp | 5 ----- src/lib/mongoBackend/mongoCreateSubscription.cpp | 5 +++++ src/lib/mongoBackend/mongoUpdateSubscription.cpp | 4 ++++ .../0000_deprecated_checkings/log_deprecate_warning.test | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonSubscription.cpp b/src/lib/mongoBackend/MongoCommonSubscription.cpp index 1994c5a688..dd6bcf3101 100644 --- a/src/lib/mongoBackend/MongoCommonSubscription.cpp +++ b/src/lib/mongoBackend/MongoCommonSubscription.cpp @@ -593,11 +593,6 @@ void setFormat(const Subscription& sub, orion::BSONObjBuilder* b) { std::string format = renderFormatToString(sub.attrsFormat); - if (logDeprecate && sub.attrsFormat == NGSI_V1_LEGACY) - { - LM_W(("Deprecated usage of notification legacy format in subscription creation or modification (subId: %s)", sub.id.c_str())); - } - b->append(CSUB_FORMAT, format); LM_T(LmtMongo, ("Subscription format: %s", format.c_str())); } diff --git a/src/lib/mongoBackend/mongoCreateSubscription.cpp b/src/lib/mongoBackend/mongoCreateSubscription.cpp index e926239f59..b59b76bf8b 100644 --- a/src/lib/mongoBackend/mongoCreateSubscription.cpp +++ b/src/lib/mongoBackend/mongoCreateSubscription.cpp @@ -197,6 +197,11 @@ std::string mongoCreateSubscription return ""; } + if (logDeprecate && sub.attrsFormat == NGSI_V1_LEGACY) + { + LM_W(("Deprecated usage of notification legacy format in subscription creation or modification (subId: %s)", subId.c_str())); + } + reqSemGive(__FUNCTION__, "ngsiv2 create subscription request", reqSemTaken); return subId; diff --git a/src/lib/mongoBackend/mongoUpdateSubscription.cpp b/src/lib/mongoBackend/mongoUpdateSubscription.cpp index c967b03519..658ed7a11a 100644 --- a/src/lib/mongoBackend/mongoUpdateSubscription.cpp +++ b/src/lib/mongoBackend/mongoUpdateSubscription.cpp @@ -369,6 +369,10 @@ std::string mongoUpdateSubscription if (subUp.notifyOnMetadataChangeProvided) setNotifyOnMetadataChange(subUp, &setB); if (subUp.attrsFormatProvided) setFormat(subUp, &setB); + if (logDeprecate && subUp.attrsFormat == NGSI_V1_LEGACY) + { + LM_W(("Deprecated usage of notification legacy format in subscription creation or modification (subId: %s)", subUp.id.c_str())); + } // Description is special, as "" value removes the field if (subUp.descriptionProvided) diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index 4dd5952ab1..e33253bdf6 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -533,7 +533,7 @@ Deprecated usage of legacyForwarding mode in update forwarding operation (regId: Raising alarm ForwardingError localhost:9801/v2/updateContext: forwarding failure for sender-thread: Couldn't connect to server Raising alarm BadInput 127.0.0.1: The requested entity has not been found. Check type and id Releasing alarm BadInput 127.0.0.1 -Deprecated usage of notification legacy format in subscription creation or modification (subId: ) +Deprecated usage of notification legacy format in subscription creation or modification (subId: SUB_ID) Deprecated usage of notification legacy format detected in existing subscription (subId: SUB_ID) Deprecated usage of notification legacy format in subscription creation or modification (subId: SUB_ID) Deprecated usage of notification legacy format in notification (subId: SUB_ID) From 04f7ef2a52e2ba214ef42e0bbb44d80cde497389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 17:32:53 +0100 Subject: [PATCH 070/390] FIX improve logs.md --- doc/manuals/admin/logs.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index 2f4020b79f..cc9fb104dc 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -406,15 +406,30 @@ time=2020-10-26T15:06:14.642Z | lvl=INFO | corr=c4a3192e-179c-11eb-ac8f-000c29df If `-logDeprecate` CLI setting is used (or `deprecate` parameter in the [log admin REST API](management_api.md#log-configs-and-trace-levels)) the following WARN traces are generated: -* NGSIv1 requests (both with and without payload). Note this doesn't - include notifications using [`"attrsFormat": "legacy"`](../orion-api.md#subscriptionnotification) or forward requests corresponding - to registrations using [`"legacyForwarding": true`](../orion-api.md#registrationprovider)). For instance: +* NGSIv1 requests (both with and without payload). For instance: ``` time=2023-05-25T14:27:45.958Z | lvl=WARN | corr=513bd10e-fb08-11ed-8ad7-000c29583ca5 | trans=1685024865-125-00000000001 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[171]:logInfoRequestWithPayload | msg=Deprecated NGSIv1 request received: POST /v1/queryContext, request payload (48 bytes): { "entities": [ { "type": "T1", "id": "E1" } ] }, response code: 200 time=2023-05-25T14:27:46.041Z | lvl=WARN | corr=51490536-fb08-11ed-9782-000c29583ca5 | trans=1685024865-125-00000000002 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[114]:logInfoRequestWithoutPayload | msg=Deprecated NGSIv1 request received: GET /v1/contextEntities/E, response code: 200 ``` +* Usages of [`"legacyForwarding": true`](../orion-api.md#registrationprovider)). For instance: + +``` +time=2024-01-11T13:57:13.537Z | lvl=WARN | corr=527378d8-b089-11ee-875d-080027cd35f1 | trans=1704981432-655-00000000006 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=mongoRegistrationCreate.cpp[235]:mongoRegistrationCreate | msg=Deprecated usage of legacyForwarding mode in registration creation (regId: 659ff3b9691855f16d00ec5a) +time=2024-01-11T13:57:13.565Z | lvl=WARN | corr=52778eaa-b089-11ee-861c-080027cd35f1 | trans=1704981432-655-00000000007 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=mongoRegistrationGet.cpp[93]:setProvider | msg=Deprecated usage of legacyForwarding mode detected in existing registration (regId: 659ff3b9691855f16d00ec5a) +time=2024-01-11T13:57:13.595Z | lvl=WARN | corr=527c0912-b089-11ee-bb8c-080027cd35f1 | trans=1704981432-655-00000000008 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=postQueryContext.cpp[191]:queryForward | msg=Deprecated usage of legacyForwarding mode in query forwarding operation (regId: 659ff3b9691855f16d00ec5a) +time=2024-01-11T13:57:13.624Z | lvl=WARN | corr=52808938-b089-11ee-9835-080027cd35f1 | trans=1704981432-655-00000000010 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=postUpdateContext.cpp[163]:updateForward | msg=Deprecated usage of legacyForwarding mode in update forwarding operation (regId: 659ff3b9691855f16d00ec5a) +``` + +* Usages of [`"attrsFormat": "legacy"`](../orion-api.md#subscriptionnotification). For instance: + +``` +time=2024-01-11T16:23:24.646Z | lvl=WARN | corr=be709034-b09d-11ee-b5d1-080027cd35f1 | trans=1704990203-652-00000000012 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=MongoCommonSubscription.cpp[598]:setFormat | msg=Deprecated usage of notification legacy format in subscription creation or modification (subId: 65a015fcda947708d30425eb) +time=2024-01-11T16:23:24.675Z | lvl=WARN | corr=be74dc98-b09d-11ee-b2d6-080027cd35f1 | trans=1704990203-652-00000000013 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=mongoGetSubscriptions.cpp[217]:setNotification | msg=Deprecated usage of notification legacy format detected in existing subscription (subId: 65a015fcda947708d30425eb) +time=2024-01-11T16:23:24.716Z | lvl=WARN | corr=be7ae5ac-b09d-11ee-98c8-080027cd35f1 | trans=1704990203-652-00000000015 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=Notifier.cpp[680]:buildSenderParams | msg=Deprecated usage of notification legacy format in notification (subId: 65a015fcda947708d30425eb) +``` + * Usages NGSIv1 usages of location metadata. Example: ``` From 7194d957de9d98aa4c9a0d7d2515117857dceb82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 17:38:01 +0100 Subject: [PATCH 071/390] FIX improve logDeprecation logic --- doc/manuals/admin/logs.md | 3 ++- src/lib/mongoBackend/mongoCreateSubscription.cpp | 2 +- src/lib/mongoBackend/mongoUpdateSubscription.cpp | 2 +- .../0000_deprecated_checkings/log_deprecate_warning.test | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index cc9fb104dc..f4d299ac8a 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -425,8 +425,9 @@ time=2024-01-11T13:57:13.624Z | lvl=WARN | corr=52808938-b089-11ee-9835-080027cd * Usages of [`"attrsFormat": "legacy"`](../orion-api.md#subscriptionnotification). For instance: ``` -time=2024-01-11T16:23:24.646Z | lvl=WARN | corr=be709034-b09d-11ee-b5d1-080027cd35f1 | trans=1704990203-652-00000000012 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=MongoCommonSubscription.cpp[598]:setFormat | msg=Deprecated usage of notification legacy format in subscription creation or modification (subId: 65a015fcda947708d30425eb) +time=2024-01-11T16:23:24.646Z | lvl=WARN | corr=be709034-b09d-11ee-b5d1-080027cd35f1 | trans=1704990203-652-00000000012 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=MongoCommonSubscription.cpp[598]:setFormat | msg=Deprecated usage of notification legacy format in subscription creation (subId: 65a015fcda947708d30425eb) time=2024-01-11T16:23:24.675Z | lvl=WARN | corr=be74dc98-b09d-11ee-b2d6-080027cd35f1 | trans=1704990203-652-00000000013 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=mongoGetSubscriptions.cpp[217]:setNotification | msg=Deprecated usage of notification legacy format detected in existing subscription (subId: 65a015fcda947708d30425eb) +time=2024-01-11T16:23:24.701Z | lvl=WARN | corr=be709034-b09d-11ee-b5d1-080027cd35f1 | trans=1704990203-652-00000000012 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=MongoCommonSubscription.cpp[598]:setFormat | msg=Deprecated usage of notification legacy format in subscription modification (subId: 65a015fcda947708d30425eb) time=2024-01-11T16:23:24.716Z | lvl=WARN | corr=be7ae5ac-b09d-11ee-98c8-080027cd35f1 | trans=1704990203-652-00000000015 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=Notifier.cpp[680]:buildSenderParams | msg=Deprecated usage of notification legacy format in notification (subId: 65a015fcda947708d30425eb) ``` diff --git a/src/lib/mongoBackend/mongoCreateSubscription.cpp b/src/lib/mongoBackend/mongoCreateSubscription.cpp index b59b76bf8b..19d75fbc34 100644 --- a/src/lib/mongoBackend/mongoCreateSubscription.cpp +++ b/src/lib/mongoBackend/mongoCreateSubscription.cpp @@ -199,7 +199,7 @@ std::string mongoCreateSubscription if (logDeprecate && sub.attrsFormat == NGSI_V1_LEGACY) { - LM_W(("Deprecated usage of notification legacy format in subscription creation or modification (subId: %s)", subId.c_str())); + LM_W(("Deprecated usage of notification legacy format in subscription creation (subId: %s)", subId.c_str())); } reqSemGive(__FUNCTION__, "ngsiv2 create subscription request", reqSemTaken); diff --git a/src/lib/mongoBackend/mongoUpdateSubscription.cpp b/src/lib/mongoBackend/mongoUpdateSubscription.cpp index 658ed7a11a..8a02346d12 100644 --- a/src/lib/mongoBackend/mongoUpdateSubscription.cpp +++ b/src/lib/mongoBackend/mongoUpdateSubscription.cpp @@ -371,7 +371,7 @@ std::string mongoUpdateSubscription if (logDeprecate && subUp.attrsFormat == NGSI_V1_LEGACY) { - LM_W(("Deprecated usage of notification legacy format in subscription creation or modification (subId: %s)", subUp.id.c_str())); + LM_W(("Deprecated usage of notification legacy format in subscription modification (subId: %s)", subUp.id.c_str())); } // Description is special, as "" value removes the field diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index e33253bdf6..231c08e731 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -533,9 +533,9 @@ Deprecated usage of legacyForwarding mode in update forwarding operation (regId: Raising alarm ForwardingError localhost:9801/v2/updateContext: forwarding failure for sender-thread: Couldn't connect to server Raising alarm BadInput 127.0.0.1: The requested entity has not been found. Check type and id Releasing alarm BadInput 127.0.0.1 -Deprecated usage of notification legacy format in subscription creation or modification (subId: SUB_ID) +Deprecated usage of notification legacy format in subscription creation (subId: SUB_ID) Deprecated usage of notification legacy format detected in existing subscription (subId: SUB_ID) -Deprecated usage of notification legacy format in subscription creation or modification (subId: SUB_ID) +Deprecated usage of notification legacy format in subscription modification (subId: SUB_ID) Deprecated usage of notification legacy format in notification (subId: SUB_ID) Raising alarm NotificationError localhost:1234/: notification failure for queue worker: Couldn't connect to server From bb72014138c11f5221130342a6b1ac84558ad398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 11 Jan 2024 17:42:05 +0100 Subject: [PATCH 072/390] Update test/functionalTest/cases/4462_ngsi_patching_id_type_macro_always_stringfly/ngsi_patching_id_type_macro_always_stringfly.test --- .../ngsi_patching_id_type_macro_always_stringfly.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionalTest/cases/4462_ngsi_patching_id_type_macro_always_stringfly/ngsi_patching_id_type_macro_always_stringfly.test b/test/functionalTest/cases/4462_ngsi_patching_id_type_macro_always_stringfly/ngsi_patching_id_type_macro_always_stringfly.test index 9795cc1ab7..87b08ba0d1 100644 --- a/test/functionalTest/cases/4462_ngsi_patching_id_type_macro_always_stringfly/ngsi_patching_id_type_macro_always_stringfly.test +++ b/test/functionalTest/cases/4462_ngsi_patching_id_type_macro_always_stringfly/ngsi_patching_id_type_macro_always_stringfly.test @@ -8,7 +8,7 @@ # License, or (at your option) any later version. # # Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implie d warranty of +# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero # General Public License for more details. # From 4aa3035a016e3060a4d539ac835000d131986d18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jan 2024 18:32:01 +0000 Subject: [PATCH 073/390] Bump jinja2 from 3.0.0 to 3.1.3 in /doc Bumps [jinja2](https://github.com/pallets/jinja) from 3.0.0 to 3.1.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.0.0...3.1.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index e3e604003a..1f933003cc 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.2.3 Pygments==2.15.0 Markdown==3.3.4 -jinja2==3.0.0 \ No newline at end of file +jinja2==3.1.3 \ No newline at end of file From 1db95e91d93287728dcbfd365f03490b6ad2ea8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 12 Jan 2024 11:48:42 +0100 Subject: [PATCH 074/390] FIX add test combining onlyChangedAttrs true and alterationType --- ...utes_that_change_with_alteration_type.test | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type.test diff --git a/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type.test b/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type.test new file mode 100644 index 0000000000..c0798a16e8 --- /dev/null +++ b/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type.test @@ -0,0 +1,172 @@ +# Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Notify only attributes that change get with alterationType + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + + +--SHELL-- + +# +# 01. Create entity E-A,B,C +# 02. Create a subscription with onlyChanged: true and alterationType +# 03. Update E-A +# 04. Dump notification get E-A +# + + +echo '01. Create entity E-A,B,C' +echo '=========================' +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo '02. Create a subscription with onlyChanged: true and alterationType' +echo '===================================================================' +payload='{ + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "A","B","C", "alterationType" + ], + "onlyChangedAttrs": true + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Update E-A' +echo '==============' +payload='{ + "A": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs -X PATCH --payload "$payload" +echo +echo + + +echo '04. Dump notification get E-A' +echo '=============================' +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create entity E-A,B,C +========================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +02. Create a subscription with onlyChanged: true and alterationType +=================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Update E-A +============== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump notification get E-A +============================= +POST http://localhost:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 122 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: localhost:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "id": "E", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From e6d2f4e7f181428e9be643d977e2a90d8e8b8d22 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Sun, 14 Jan 2024 20:27:18 +0900 Subject: [PATCH 075/390] (JP) Add doc about NGSI patching macro subscription (#4468) --- doc/manuals.jp/orion-api.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index 07055c136a..e7729a1f9d 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -2245,6 +2245,9 @@ the value of the "temperature" attribute (of type Number) is 23.4 - マクロが *使用されている文字列を完全にカバーしている場合*、属性値の JSON の性質が考慮されます。たとえば、 `"value": "${temperature}"` は、温度属性が数値の場合は `"value": 10` に解決され、`temperature` 属性が文字列の場合は `"value": "10"` に解決されます + - `id` と `type` は例外です。エンティティ ID と型が文字列でなければならないことを考えると + ([このセクション](#identifiers-syntax-restrictions)で説明されているように)、この場合、 + 属性値は常に文字列にキャストされます - マクロが使用されている文字列の一部のみである場合、属性値は常に文字列にキャストされます。たとえば、 `"value": "Temperature is: ${temperature}"` は、温度属性が数値であっても、`"value": "Temperature is 10"` に解決されます。属性値が JSON 配列またはオブジェクトの場合、この場合は文字列化されることに注意してください From aaf2889db449c1e64074b444648f8d30149b3404 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Sun, 14 Jan 2024 20:34:59 +0900 Subject: [PATCH 076/390] (JP) ADD doc about OrionError field names (#4465) --- doc/manuals.jp/admin/diagnosis.md | 42 +++++------------------- doc/manuals.jp/orion-api.md | 1 + doc/manuals.jp/user/known_limitations.md | 7 ++-- 3 files changed, 11 insertions(+), 39 deletions(-) diff --git a/doc/manuals.jp/admin/diagnosis.md b/doc/manuals.jp/admin/diagnosis.md index 454fd6990d..03e082444b 100644 --- a/doc/manuals.jp/admin/diagnosis.md +++ b/doc/manuals.jp/admin/diagnosis.md @@ -206,44 +206,18 @@ Orion Context Broker は、次のフローを使用します : - 起動時に、broker が起動せず、ログ・ファイルに次のメッセージが表示されます : -` X@08:04:45 main[313]: MongoDB error` +``` +... msg=Database Startup Error (cannot connect to mongo - doing 100 retries with a 1000 millisecond interval) +... msg=Fatal Error (MongoDB error) +``` - broker 操作中、broker から送信されたレスポンスには、次のようなエラーメッセージが表示されます。 ``` - - ... - "errorCode": { - "code": "500", - "reasonPhrase": "Database Error", - "details": "collection: ... - exception: Null cursor" - } - ... - - ... - "errorCode": { - "code": "500", - "reasonPhrase": "Database Error", - "details": "collection: ... - exception: socket exception [CONNECT_ERROR] for localhost:27017" - } - ... - - ... - "errorCode": { - "code": "500", - "reasonPhrase": "Database Error", - "details": "collection: ... - exception: socket exception [FAILED_STATE] for localhost:27017" - } - ... - - ... - "errorCode": { - "code": "500", - "reasonPhrase": "Database Error", - "details": "collection: ... - exception: DBClientBase::findN: transport error: localhost:27017 ns: orion.$cmd query: { .. }" - } - ... - +{ + "error": "InternalServerError", + "description": "Database Error ..." +} ``` どちらの場合も、MonogDB への接続が正しく構成されていることを確認してください。特に、[コマンドラインから実行する](cli.md)場合は "-dbhost" オプションです。また、シャーディングを使用しているかどうかによって異なりますが、mongod/mongos プロセスが起動していることです。 diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index 07055c136a..682770cbf4 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -453,6 +453,7 @@ Oron の実装では、この節で説明する HTTP ステータス・コード - HTTP 411 Length Required は `ContentLengthRequired` (`411`) に対応します - HTTP 413 Request Entity Too Large は、`RequestEntityTooLarge` (`413`) に対応します - HTTP 415 Unsupported Media Type は `UnsupportedMediaType` (`415`) に対応します +- 内部エラー (Internal errors) には `InternalServerError` (`500`) を使用します diff --git a/doc/manuals.jp/user/known_limitations.md b/doc/manuals.jp/user/known_limitations.md index 2e143cfbae..6964c92d65 100644 --- a/doc/manuals.jp/user/known_limitations.md +++ b/doc/manuals.jp/user/known_limitations.md @@ -7,11 +7,8 @@ Orion Context Broker のデフォルトの最大リクエストサイズは1MB ``` { - "errorCode" : { - "code" : "413", - "reasonPhrase" : "Request Entity Too Large", - "details" : "payload size: 1500000, max size supported: 1048576" - } + "error": "RequestEntityTooLarge" + "description": "payload size: 1500000, max size supported: 1048576", } ``` From 387571124a6ce30e7e2fe11164ce68d1e13908b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 10:31:54 +0100 Subject: [PATCH 077/390] FIX batch delete logic --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 4 +- ...ving_attrs_and_entity_in_same_request.test | 110 ++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 test/functionalTest/cases/3499_partial_update_response/batch_delete_removing_attrs_and_entity_in_same_request.test diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 9fb1f84c8f..6e3f948681 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4471,7 +4471,9 @@ unsigned int processContextElement responseP->oe.fillOrAppend(SccInvalidModification, details, ", " + enStr + " - " + attributeNotExistingList, ERROR_UNPROCESSABLE); } - if (updateCoverageP != NULL) + // The following check makes sense only when there are at least one attribute in the request + // (e.g. entity deletion, which doesn't include attributes) + if ((updateCoverageP != NULL) && (eP->attributeVector.size() > 0)) { if ((action == ActionTypeUpdate) || (action == ActionTypeDelete)) { diff --git a/test/functionalTest/cases/3499_partial_update_response/batch_delete_removing_attrs_and_entity_in_same_request.test b/test/functionalTest/cases/3499_partial_update_response/batch_delete_removing_attrs_and_entity_in_same_request.test new file mode 100644 index 0000000000..7f81f26734 --- /dev/null +++ b/test/functionalTest/cases/3499_partial_update_response/batch_delete_removing_attrs_and_entity_in_same_request.test @@ -0,0 +1,110 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Removing attributes and etity itself in same entity batch delete + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entities room1 and room2 +# 02. Delete room1 (temperature), room2 (temperature), room1 (entity itself), room2 (entity itself), get 204 +# + +echo "01. Create entities room1 and room2" +echo "===================================" +payload='{ + "entities": [ + { + "type": "house", + "id": "room1", + "temperature": 34 + }, + { + "type": "home", + "id": "room2", + "temperature": 44 + } + ], + "actionType": "append" +}' +orionCurl --url /v2/op/update?options=keyValues --payload "$payload" +echo +echo + + +echo "02. Delete room1 (temperature), room2 (temperature), room1 (entity itself), room2 (entity itself), get 204" +echo "==========================================================================================================" +payload='{ + "entities": [ + { + "type": "house", + "id": "room1", + "temperature": 34 + }, + { + "type": "home", + "id": "room2", + "temperature": 44 + }, + { + "type": "house", + "id": "room1" + }, + { + "type": "home", + "id": "room2" + } + ], + "actionType": "delete" +}' +orionCurl --url /v2/op/update?options=keyValues --payload "$payload" +echo +echo + + + + +--REGEXPECT-- +01. Create entities room1 and room2 +=================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +02. Delete room1 (temperature), room2 (temperature), room1 (entity itself), room2 (entity itself), get 204 +========================================================================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file From b85af8f7459620a919159e1dac1705169939d4f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 10:34:28 +0100 Subject: [PATCH 078/390] FIX improve code comment --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 6e3f948681..2c1ad6c507 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -4472,7 +4472,7 @@ unsigned int processContextElement } // The following check makes sense only when there are at least one attribute in the request - // (e.g. entity deletion, which doesn't include attributes) + // (e.g. not for entity deletion, which doesn't include attributes) if ((updateCoverageP != NULL) && (eP->attributeVector.size() > 0)) { if ((action == ActionTypeUpdate) || (action == ActionTypeDelete)) From b5370f44c9a2396873d142288e3138bbeae99ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 11:36:22 +0100 Subject: [PATCH 079/390] FIX mkdocs version from 1.2.3 to 1.2.4 --- doc/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 1f933003cc..4252df910c 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ -mkdocs==1.2.3 +mkdocs==1.2.4 Pygments==2.15.0 Markdown==3.3.4 -jinja2==3.1.3 \ No newline at end of file +jinja2==3.1.3 From 70e997e8dafd37ed4e05bbe9f84be35a0993564c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 16:28:57 +0100 Subject: [PATCH 080/390] ADD deprecatedFeatures counters --- CHANGES_NEXT_RELEASE | 1 + src/lib/common/logTracing.cpp | 17 +++-- src/lib/common/statistics.cpp | 7 ++ src/lib/common/statistics.h | 6 ++ src/lib/mongoBackend/location.cpp | 10 ++- .../mongoBackend/mongoCreateSubscription.cpp | 9 ++- .../mongoBackend/mongoGetSubscriptions.cpp | 8 ++- .../mongoBackend/mongoRegistrationCreate.cpp | 8 ++- src/lib/mongoBackend/mongoRegistrationGet.cpp | 1 + .../mongoBackend/mongoUpdateSubscription.cpp | 9 ++- src/lib/ngsiNotify/Notifier.cpp | 2 + src/lib/serviceRoutines/postQueryContext.cpp | 1 + src/lib/serviceRoutines/postUpdateContext.cpp | 1 + src/lib/serviceRoutines/statisticsTreat.cpp | 19 +++++ .../log_deprecate_warning.test | 69 ++++++++++++++++++- .../statistics_with_counters.test | 5 +- .../statistics_with_full_counters.test | 9 ++- 17 files changed, 165 insertions(+), 17 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 12e18dc0b8..cdc1e2e174 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -3,6 +3,7 @@ - Add: notification payload in INFO log traces (#4449) - Add: log deprecation traces for usage of legacyForwarding mode in registrations - Add: log deprecation traces for usage of attrsFormat legacy in subscriptions +- Add: deprecatedFeature counters block in GET /v2/statistics - Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: update forwarding was not working when entity type is not included in the request (#4460) diff --git a/src/lib/common/logTracing.cpp b/src/lib/common/logTracing.cpp index d7593edd02..750d9bd472 100644 --- a/src/lib/common/logTracing.cpp +++ b/src/lib/common/logTracing.cpp @@ -28,6 +28,7 @@ #include #include "logMsg/logMsg.h" +#include "common/statistics.h" @@ -170,9 +171,13 @@ void logInfoRequestWithoutPayload { LM_I(("Request received: %s %s, response code: %d", verb, url, rc)); - if (logDeprecate && isNgsiV1Url(url)) + if (isNgsiV1Url(url)) { - LM_W(("Deprecated NGSIv1 request received: %s %s, response code: %d", verb, url, rc)); + __sync_fetch_and_add(&noOfDprNgsiv1Request, 1); + if (logDeprecate) + { + LM_W(("Deprecated NGSIv1 request received: %s %s, response code: %d", verb, url, rc)); + } } } @@ -205,9 +210,13 @@ void logInfoRequestWithPayload LM_I(("Request received: %s %s, request payload (%d bytes): %s, response code: %d", verb, url, strlen(payload), effectivePayload, rc)); - if (logDeprecate && isNgsiV1Url(url)) + if (isNgsiV1Url(url)) { - LM_W(("Deprecated NGSIv1 request received: %s %s, request payload (%d bytes): %s, response code: %d", verb, url, strlen(payload), effectivePayload, rc)); + __sync_fetch_and_add(&noOfDprNgsiv1Request, 1); + if (logDeprecate) + { + LM_W(("Deprecated NGSIv1 request received: %s %s, request payload (%d bytes): %s, response code: %d", verb, url, strlen(payload), effectivePayload, rc)); + } } if (cleanAfterUse) diff --git a/src/lib/common/statistics.cpp b/src/lib/common/statistics.cpp index de902948ff..b026031382 100644 --- a/src/lib/common/statistics.cpp +++ b/src/lib/common/statistics.cpp @@ -149,6 +149,13 @@ int noOfDiscoveryErrors = -1; int noOfNotificationsSent = -1; int noOfSimulatedNotifications = -1; +// Deprecated features +int noOfDprNgsiv1Request = -1; +int noOfDprLegacyForwarding = -1; +int noOfDprLegacyNotif = -1; +int noOfDprLocationMetadata = -1; +int noOfDprGeoformat = -1; + /* **************************************************************************** * diff --git a/src/lib/common/statistics.h b/src/lib/common/statistics.h index fb1d52fcaf..bfbf8d0f2d 100644 --- a/src/lib/common/statistics.h +++ b/src/lib/common/statistics.h @@ -276,6 +276,12 @@ extern int noOfDiscoveryErrors; extern int noOfNotificationsSent; extern int noOfSimulatedNotifications; +// Deprecated features +extern int noOfDprNgsiv1Request; +extern int noOfDprLegacyForwarding; +extern int noOfDprLegacyNotif; +extern int noOfDprLocationMetadata; +extern int noOfDprGeoformat; /* **************************************************************************** * diff --git a/src/lib/mongoBackend/location.cpp b/src/lib/mongoBackend/location.cpp index e683c320ad..ed2a82a36a 100644 --- a/src/lib/mongoBackend/location.cpp +++ b/src/lib/mongoBackend/location.cpp @@ -29,6 +29,7 @@ #include "common/string.h" #include "common/globals.h" #include "common/errorMessages.h" +#include "common/statistics.h" #include "logMsg/logMsg.h" #include "ngsi/ContextAttribute.h" #include "parse/CompoundValueNode.h" @@ -324,6 +325,7 @@ static bool getGeoJson // This corresponds to the legacy way in NGSIv1 based in metadata // The block is the same that for GEO_POINT but it is clearer if we keep it separated + __sync_fetch_and_add(&noOfDprLocationMetadata, 1); if (logDeprecate) { LM_W(("Deprecated usage of metadata location %s detected in attribute %s at entity update, please use geo:json instead", caP->type.c_str(), caP->name.c_str())); @@ -348,9 +350,13 @@ static bool getGeoJson return true; } - if ((logDeprecate) && ((caP->type == GEO_POINT) || (caP->type == GEO_LINE) || (caP->type == GEO_BOX) || (caP->type == GEO_POLYGON))) + if ((caP->type == GEO_POINT) || (caP->type == GEO_LINE) || (caP->type == GEO_BOX) || (caP->type == GEO_POLYGON)) { - LM_W(("Deprecated usage of %s detected in attribute %s at entity update, please use geo:json instead", caP->type.c_str(), caP->name.c_str())); + __sync_fetch_and_add(&noOfDprGeoformat, 1); + if (logDeprecate) + { + LM_W(("Deprecated usage of %s detected in attribute %s at entity update, please use geo:json instead", caP->type.c_str(), caP->name.c_str())); + } } if (caP->type == GEO_POINT) diff --git a/src/lib/mongoBackend/mongoCreateSubscription.cpp b/src/lib/mongoBackend/mongoCreateSubscription.cpp index 19d75fbc34..91192349f5 100644 --- a/src/lib/mongoBackend/mongoCreateSubscription.cpp +++ b/src/lib/mongoBackend/mongoCreateSubscription.cpp @@ -31,6 +31,7 @@ #include "logMsg/logMsg.h" #include "logMsg/traceLevels.h" #include "common/defaultValues.h" +#include "common/statistics.h" #include "apiTypesV2/Subscription.h" #include "cache/subCache.h" #include "rest/OrionError.h" @@ -197,9 +198,13 @@ std::string mongoCreateSubscription return ""; } - if (logDeprecate && sub.attrsFormat == NGSI_V1_LEGACY) + if (sub.attrsFormat == NGSI_V1_LEGACY) { - LM_W(("Deprecated usage of notification legacy format in subscription creation (subId: %s)", subId.c_str())); + __sync_fetch_and_add(&noOfDprLegacyNotif, 1); + if (logDeprecate) + { + LM_W(("Deprecated usage of notification legacy format in subscription creation (subId: %s)", subId.c_str())); + } } reqSemGive(__FUNCTION__, "ngsiv2 create subscription request", reqSemTaken); diff --git a/src/lib/mongoBackend/mongoGetSubscriptions.cpp b/src/lib/mongoBackend/mongoGetSubscriptions.cpp index c6cbe38e15..c682955d0b 100644 --- a/src/lib/mongoBackend/mongoGetSubscriptions.cpp +++ b/src/lib/mongoBackend/mongoGetSubscriptions.cpp @@ -212,9 +212,13 @@ static void setNotification(Subscription* subP, const orion::BSONObj& r, const s // Attributes format subP->attrsFormat = r.hasField(CSUB_FORMAT)? stringToRenderFormat(getStringFieldF(r, CSUB_FORMAT)) : NGSI_V1_LEGACY; - if (logDeprecate && subP->attrsFormat == NGSI_V1_LEGACY) + if (subP->attrsFormat == NGSI_V1_LEGACY) { - LM_W(("Deprecated usage of notification legacy format detected in existing subscription (subId: %s)", subP->id.c_str())); + __sync_fetch_and_add(&noOfDprLegacyNotif, 1); + if (logDeprecate) + { + LM_W(("Deprecated usage of notification legacy format detected in existing subscription (subId: %s)", subP->id.c_str())); + } } diff --git a/src/lib/mongoBackend/mongoRegistrationCreate.cpp b/src/lib/mongoBackend/mongoRegistrationCreate.cpp index 3f31097cec..fa17c450ea 100644 --- a/src/lib/mongoBackend/mongoRegistrationCreate.cpp +++ b/src/lib/mongoBackend/mongoRegistrationCreate.cpp @@ -230,9 +230,13 @@ void mongoRegistrationCreate std::string format = (regP->provider.legacyForwardingMode == true)? "JSON" : "normalized"; - if (logDeprecate && format == "JSON") + if (format == "JSON") { - LM_W(("Deprecated usage of legacyForwarding mode in registration creation (regId: %s)", regIdP->c_str())); + __sync_fetch_and_add(&noOfDprLegacyForwarding, 1); + if (logDeprecate) + { + LM_W(("Deprecated usage of legacyForwarding mode in registration creation (regId: %s)", regIdP->c_str())); + } } setFormat(format, &bob); diff --git a/src/lib/mongoBackend/mongoRegistrationGet.cpp b/src/lib/mongoBackend/mongoRegistrationGet.cpp index 31e587e906..41e677ac77 100644 --- a/src/lib/mongoBackend/mongoRegistrationGet.cpp +++ b/src/lib/mongoBackend/mongoRegistrationGet.cpp @@ -88,6 +88,7 @@ static void setProvider(ngsiv2::Registration* regP, const ngsiv2::ForwardingMode if (format == "JSON") { + __sync_fetch_and_add(&noOfDprLegacyForwarding, 1); if (logDeprecate) { LM_W(("Deprecated usage of legacyForwarding mode detected in existing registration (regId: %s)", regP->id.c_str())); diff --git a/src/lib/mongoBackend/mongoUpdateSubscription.cpp b/src/lib/mongoBackend/mongoUpdateSubscription.cpp index 8a02346d12..93d71ccb2b 100644 --- a/src/lib/mongoBackend/mongoUpdateSubscription.cpp +++ b/src/lib/mongoBackend/mongoUpdateSubscription.cpp @@ -31,6 +31,7 @@ #include "logMsg/logMsg.h" #include "logMsg/traceLevels.h" #include "common/defaultValues.h" +#include "common/statistics.h" #include "apiTypesV2/SubscriptionUpdate.h" #include "rest/OrionError.h" #include "alarmMgr/alarmMgr.h" @@ -369,9 +370,13 @@ std::string mongoUpdateSubscription if (subUp.notifyOnMetadataChangeProvided) setNotifyOnMetadataChange(subUp, &setB); if (subUp.attrsFormatProvided) setFormat(subUp, &setB); - if (logDeprecate && subUp.attrsFormat == NGSI_V1_LEGACY) + if (subUp.attrsFormat == NGSI_V1_LEGACY) { - LM_W(("Deprecated usage of notification legacy format in subscription modification (subId: %s)", subUp.id.c_str())); + __sync_fetch_and_add(&noOfDprLegacyNotif, 1); + if (logDeprecate) + { + LM_W(("Deprecated usage of notification legacy format in subscription modification (subId: %s)", subUp.id.c_str())); + } } // Description is special, as "" value removes the field diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 6142ce36ac..f452b51ac2 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -154,6 +154,7 @@ static bool setPayload if (*renderFormatP == NGSI_V1_LEGACY) { + __sync_fetch_and_add(&noOfDprLegacyNotif, 1); if (logDeprecate) { LM_W(("Deprecated usage of notification legacy format in notification (subId: %s)", subscriptionId.c_str())); @@ -689,6 +690,7 @@ SenderThreadParams* Notifier::buildSenderParams std::string payloadString; if (renderFormat == NGSI_V1_LEGACY) { + __sync_fetch_and_add(&noOfDprLegacyNotif, 1); if (logDeprecate) { LM_W(("Deprecated usage of notification legacy format in notification (subId: %s)", subId.c_str())); diff --git a/src/lib/serviceRoutines/postQueryContext.cpp b/src/lib/serviceRoutines/postQueryContext.cpp index 2b7c4dcffc..5439fcf01a 100644 --- a/src/lib/serviceRoutines/postQueryContext.cpp +++ b/src/lib/serviceRoutines/postQueryContext.cpp @@ -186,6 +186,7 @@ static bool queryForward { op = "/queryContext"; + __sync_fetch_and_add(&noOfDprLegacyForwarding, 1); if (logDeprecate) { LM_W(("Deprecated usage of legacyForwarding mode in query forwarding operation (regId: %s)", regId.c_str())); diff --git a/src/lib/serviceRoutines/postUpdateContext.cpp b/src/lib/serviceRoutines/postUpdateContext.cpp index 6819e5554b..820e074b9d 100644 --- a/src/lib/serviceRoutines/postUpdateContext.cpp +++ b/src/lib/serviceRoutines/postUpdateContext.cpp @@ -158,6 +158,7 @@ static bool updateForward op = "/updateContext"; + __sync_fetch_and_add(&noOfDprLegacyForwarding, 1); if (logDeprecate) { LM_W(("Deprecated usage of legacyForwarding mode in update forwarding operation (regId: %s)", regId.c_str())); diff --git a/src/lib/serviceRoutines/statisticsTreat.cpp b/src/lib/serviceRoutines/statisticsTreat.cpp index 2fe34c6797..95d73d2370 100644 --- a/src/lib/serviceRoutines/statisticsTreat.cpp +++ b/src/lib/serviceRoutines/statisticsTreat.cpp @@ -91,6 +91,12 @@ static void resetStatistics(void) noOfNotificationsSent = -1; noOfSimulatedNotifications = -1; + noOfDprNgsiv1Request = -1; + noOfDprLegacyForwarding = -1; + noOfDprLegacyNotif = -1; + noOfDprLocationMetadata = -1; + noOfDprGeoformat = -1; + statisticsTime = getCurrentTime(); QueueStatistics::reset(); @@ -213,6 +219,19 @@ std::string renderCounterStats(bool fullCounters) renderUsedCounter(&js, "discoveryErrors", noOfDiscoveryErrors, fullCounters); renderUsedCounter(&js, "notificationsSent", noOfNotificationsSent, fullCounters); + JsonObjectHelper jsDeprecated; + renderUsedCounter(&jsDeprecated, "ngsiv1Requests", noOfDprNgsiv1Request, fullCounters); + renderUsedCounter(&jsDeprecated, "ngsiv1Forwarding", noOfDprLegacyForwarding, fullCounters); + renderUsedCounter(&jsDeprecated, "ngsiv1NotifFormat", noOfDprLegacyNotif, fullCounters); + renderUsedCounter(&jsDeprecated, "metadataLocation", noOfDprLocationMetadata, fullCounters); + renderUsedCounter(&jsDeprecated, "geoFormat", noOfDprGeoformat, fullCounters); + + std::string deprecation = jsDeprecated.str(); + if (deprecation != "{}") + { + js.addRaw("deprecatedFeatures", deprecation); + } + return js.str(); } diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index 231c08e731..7861ea648e 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -25,7 +25,7 @@ Disable NGSIv1 CLI --SHELL-INIT-- dbInit CB -brokerStart CB 0 IPV4 -logDeprecate +brokerStart CB 0 IPV4 -logDeprecate -statCounters --SHELL-- @@ -47,6 +47,7 @@ brokerStart CB 0 IPV4 -logDeprecate # 12. Update subscription using attrsFormat legacy # 13. Trigger notification using attrsFormat legacy # 14. Get WARNING trace in logs +# 15. Get statistics and see deprecatedFeatures counters # echo "01. Query E1-T1" @@ -264,6 +265,13 @@ echo echo +echo "15. Get statistics and see deprecatedFeatures counters" +echo "======================================================" +orionCurl --url /statistics +echo +echo + + --REGEXPECT-- 01. Query E1-T1 =============== @@ -540,6 +548,65 @@ Deprecated usage of notification legacy format in notification (subId: SUB_ID) Raising alarm NotificationError localhost:1234/: notification failure for queue worker: Couldn't connect to server +15. Get statistics and see deprecatedFeatures counters +====================================================== +HTTP/1.1 200 OK +Date: Mon, 15 Jan 2024 15:17:53 GMT +Fiware-Correlator: 40bb4638-b3b9-11ee-adfc-080027cd35f1 +Content-Type: application/json +Content-Length: 587 + +{ + "counters": { + "deprecatedFeatures": { + "geoFormat": 2, + "metadataLocation": 1, + "ngsiv1Forwarding": 4, + "ngsiv1NotifFormat": 4, + "ngsiv1Requests": 4 + }, + "jsonRequests": 9, + "noPayloadRequests": 5, + "requests": { + "/statistics": { + "GET": 1 + }, + "/v2/entities": { + "POST": 2 + }, + "/v2/entities/{id}/attrs/{name}": { + "GET": 1, + "PUT": 1 + }, + "/v2/registrations": { + "GET": 1, + "POST": 1 + }, + "/v2/subscriptions": { + "GET": 1, + "POST": 1 + }, + "/v2/subscriptions/{id}": { + "PATCH": 1 + } + }, + "requestsLegacy": { + "/v1/contextEntities/{id}/attributes/{name}": { + "GET": 1 + }, + "/v1/queryContext": { + "POST": 1 + }, + "/v1/updateContext": { + "POST": 2 + } + } + }, + "measuring_interval_in_secs": REGEX(\d+), + "uptime_in_secs": REGEX(\d+) +} + + --TEARDOWN-- brokerStop CB dbDrop CB diff --git a/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test b/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test index b89e50670f..26f899f85f 100644 --- a/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test +++ b/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test @@ -249,7 +249,7 @@ orionCurl --url /v1/updateContext -X POST --payload "$payload" > # Get statistics -orionCurl --url /statistics?options +orionCurl --url /statistics --REGEXPECT-- HTTP/1.1 200 OK @@ -260,6 +260,9 @@ Content-Length: REGEX(\d+) { "counters": { + "deprecatedFeatures": { + "ngsiv1Requests": 7 + }, "jsonRequests": 16, "noPayloadRequests": 42, "requests": { diff --git a/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test b/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test index 4497c29ef5..52ed86700d 100644 --- a/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test +++ b/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test @@ -46,10 +46,17 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1525 +Content-Length: 1645 { "counters": { + "deprecatedFeatures": { + "geoFormat": 0, + "metadataLocation": 0, + "ngsiv1Forwarding": 0, + "ngsiv1NotifFormat": 0, + "ngsiv1Requests": 0 + }, "discoveryErrors": 0, "invalidRequests": 0, "jsonRequests": 0, From d41e663c5d0416c069552f799c0bf8dd4e082f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 16:31:15 +0100 Subject: [PATCH 081/390] FIX typo --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index cdc1e2e174..0304648eee 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -3,7 +3,7 @@ - Add: notification payload in INFO log traces (#4449) - Add: log deprecation traces for usage of legacyForwarding mode in registrations - Add: log deprecation traces for usage of attrsFormat legacy in subscriptions -- Add: deprecatedFeature counters block in GET /v2/statistics +- Add: deprecatedFeatures counters block in GET /v2/statistics - Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: update forwarding was not working when entity type is not included in the request (#4460) From 1f722a0b46c29cb02e2b6d2ba86e02a6a78023a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 16:34:00 +0100 Subject: [PATCH 082/390] FIX ftest --- .../0000_deprecated_checkings/log_deprecate_warning.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index 7861ea648e..d971e1aaf2 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -551,8 +551,8 @@ Raising alarm NotificationError localhost:1234/: notification failure for queue 15. Get statistics and see deprecatedFeatures counters ====================================================== HTTP/1.1 200 OK -Date: Mon, 15 Jan 2024 15:17:53 GMT -Fiware-Correlator: 40bb4638-b3b9-11ee-adfc-080027cd35f1 +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json Content-Length: 587 From 0acb29dd263521bc8633cbecded64b76b1d06ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 16:57:10 +0100 Subject: [PATCH 083/390] ADD servicePath note in orion-api.md --- doc/manuals/orion-api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 72f457e42d..ce23725d3c 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -588,6 +588,8 @@ Some additional remarks: - `Fiware-ServicePath` header is included in notification requests sent by Orion. +- You can use the [`servicePath builtin-attribute](#builtin-attributes) to get the entity service path. + - The scopes entities can be combined orthogonally with the [multi-tenancy functionality](#multi-tenancy). In that case, each `scope tree` lives in a different service/tenant and they can From 2c65769fb7832d60a56dd5b149d6c8126356507e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 16:59:31 +0100 Subject: [PATCH 084/390] FIX docu --- doc/manuals/admin/logs.md | 4 ++++ doc/manuals/admin/statistics.md | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index f4d299ac8a..227379d78c 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -443,6 +443,10 @@ time=2023-06-08T15:14:20.999Z | lvl=WARN | corr=24fd2acc-060f-11ee-94cc-000c2958 time=2023-06-08T15:14:21.176Z | lvl=WARN | corr=2518249e-060f-11ee-9e76-000c29583ca5 | trans=1686237259-703-00000000004 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=location.cpp[353]:getGeoJson | msg=Deprecated usage of geo:point detected in attribute location at entity update, please use geo:json instead ``` +Deprecated features usage is tracked in `deprecatedFeature` object in the [counters block of the statistics API +(statistics#counter-block), even when `-logDeprecate` CLI (or equivalent `deprecate` parameter in the log admin REST API) +is not used. + Get more information about deprecated features and how to overcome them in the [deprecation documentation](../deprecated.md). [Top](#top) diff --git a/doc/manuals/admin/statistics.md b/doc/manuals/admin/statistics.md index f8a4858301..379262166f 100644 --- a/doc/manuals/admin/statistics.md +++ b/doc/manuals/admin/statistics.md @@ -54,6 +54,13 @@ The counter block provides information about counters for the times a particular { ... "counters": { + "deprecatedFeatures": { + "geoFormat": 2, + "metadataLocation": 1, + "ngsiv1Forwarding": 4, + "ngsiv1NotifFormat": 4, + "ngsiv1Requests": 4 + }, "invalidRequests": 2, "jsonRequests": 4, "noPayloadRequests": 250, From 39c76b17dc975c3118c6475c58f44255db65b4d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 17:10:56 +0100 Subject: [PATCH 085/390] FIX to reference presentation and version numbers --- README.md | 2 +- doc/manuals.jp/devel/README.md | 2 +- doc/manuals/devel/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0c30a069bd..d9de4cda24 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ recommended to have a look to the brief ### Introductory presentations - Orion Context Broker - [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-20230606) + [(en)](https://www.slideshare.net/slideshows/orion-context-broker-introduction-20240115/265436732) [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-20230606-258279602) - NGSIv2 Overview for Developers That Already Know NGSIv1 [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220523) diff --git a/doc/manuals.jp/devel/README.md b/doc/manuals.jp/devel/README.md index ffde791c49..07dfc9889e 100644 --- a/doc/manuals.jp/devel/README.md +++ b/doc/manuals.jp/devel/README.md @@ -1,6 +1,6 @@ # 開発マニュアル -*注 : このドキュメントでは、リリース 3.10.x の Orion Context Broker について説明しています。* +*注 : このドキュメントでは、リリース 3.11.x の Orion Context Broker について説明しています。* ## 対象読者 diff --git a/doc/manuals/devel/README.md b/doc/manuals/devel/README.md index 3fb3a8bbe8..34046787f8 100644 --- a/doc/manuals/devel/README.md +++ b/doc/manuals/devel/README.md @@ -1,6 +1,6 @@ # Development Manual -*Note: This document describes Orion Context Broker as of release 3.10.x.* +*Note: This document describes Orion Context Broker as of release 3.11.x.* ## Intended audience The intended audience of this manual is developers that need to understand the internals of the Orion Context Broker From b304cae1b5bb1225a2ea0494a029196aa876ad6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 17:18:37 +0100 Subject: [PATCH 086/390] FIX typo in orion-api.md --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index ce23725d3c..5e7e83550e 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -588,7 +588,7 @@ Some additional remarks: - `Fiware-ServicePath` header is included in notification requests sent by Orion. -- You can use the [`servicePath builtin-attribute](#builtin-attributes) to get the entity service path. +- You can use the [`servicePath` builtin-attribute](#builtin-attributes) to get the entity service path. - The scopes entities can be combined orthogonally with the [multi-tenancy functionality](#multi-tenancy). In that case, From 7c51a90a6f1d62351536b5f89814c21cc5e55458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Jan 2024 17:19:49 +0100 Subject: [PATCH 087/390] Update doc/manuals/orion-api.md --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 5e7e83550e..956d832686 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -588,7 +588,7 @@ Some additional remarks: - `Fiware-ServicePath` header is included in notification requests sent by Orion. -- You can use the [`servicePath` builtin-attribute](#builtin-attributes) to get the entity service path. +- You can use the [`servicePath` builtin attribute](#builtin-attributes) to get the entity service path. - The scopes entities can be combined orthogonally with the [multi-tenancy functionality](#multi-tenancy). In that case, From 028da81bb05d1dedf479f0c605a5b12acfffe8a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 16 Jan 2024 09:37:33 +0100 Subject: [PATCH 088/390] FIX ftests --- .../cases/3004_v2_reg_create/v2_reg_create.test | 5 ++++- .../delete_reg_using_v2_created_using_v2.test | 5 ++++- .../get_reg_using_v2_created_using_v2.test | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create.test b/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create.test index a00069c2ae..0770b2d40d 100644 --- a/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create.test +++ b/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create.test @@ -138,10 +138,13 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 206 +Content-Length: 250 { "counters": { + "deprecatedFeatures": { + "ngsiv1Forwarding": 2 + }, "jsonRequests": 1, "noPayloadRequests": 2, "requests": { diff --git a/test/functionalTest/cases/3006_delete_registrations/delete_reg_using_v2_created_using_v2.test b/test/functionalTest/cases/3006_delete_registrations/delete_reg_using_v2_created_using_v2.test index 3dbc2ed238..ff06bae97e 100644 --- a/test/functionalTest/cases/3006_delete_registrations/delete_reg_using_v2_created_using_v2.test +++ b/test/functionalTest/cases/3006_delete_registrations/delete_reg_using_v2_created_using_v2.test @@ -123,10 +123,13 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 209 +Content-Length: 253 { "counters": { + "deprecatedFeatures": { + "ngsiv1Forwarding": 1 + }, "jsonRequests": 1, "noPayloadRequests": 3, "requests": { diff --git a/test/functionalTest/cases/3008_get_registration/get_reg_using_v2_created_using_v2.test b/test/functionalTest/cases/3008_get_registration/get_reg_using_v2_created_using_v2.test index 08304e931c..6a15472f9a 100644 --- a/test/functionalTest/cases/3008_get_registration/get_reg_using_v2_created_using_v2.test +++ b/test/functionalTest/cases/3008_get_registration/get_reg_using_v2_created_using_v2.test @@ -148,10 +148,13 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 206 +Content-Length: 250 { "counters": { + "deprecatedFeatures": { + "ngsiv1Forwarding": 2 + }, "jsonRequests": 1, "noPayloadRequests": 3, "requests": { From 2023c557c9afc5bb8c8f671d8ec2d2dc49de7caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 16 Jan 2024 10:48:09 +0100 Subject: [PATCH 089/390] FIX typo in doc --- doc/manuals/admin/logs.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index 227379d78c..dd24846b57 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -443,9 +443,8 @@ time=2023-06-08T15:14:20.999Z | lvl=WARN | corr=24fd2acc-060f-11ee-94cc-000c2958 time=2023-06-08T15:14:21.176Z | lvl=WARN | corr=2518249e-060f-11ee-9e76-000c29583ca5 | trans=1686237259-703-00000000004 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=location.cpp[353]:getGeoJson | msg=Deprecated usage of geo:point detected in attribute location at entity update, please use geo:json instead ``` -Deprecated features usage is tracked in `deprecatedFeature` object in the [counters block of the statistics API -(statistics#counter-block), even when `-logDeprecate` CLI (or equivalent `deprecate` parameter in the log admin REST API) -is not used. +Deprecated features usage is tracked in `deprecatedFeature` object in the [counters block of the statistics API](statistics#counter-block), +even when `-logDeprecate` CLI (or equivalent `deprecate` parameter in the log admin REST API) is not used. Get more information about deprecated features and how to overcome them in the [deprecation documentation](../deprecated.md). From e80b88c6519081d82c2341b8b2aee2b6580e9207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 16 Jan 2024 11:40:49 +0100 Subject: [PATCH 090/390] FIX alterationType, dateCreated and dateModified included in notification even when onlyChangedAttrs is true --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/orion-api.md | 9 +- src/lib/mongoBackend/MongoGlobal.cpp | 5 +- ...utes_that_change_with_alteration_type.test | 27 +-- ...butes_that_change_with_dates_builtins.test | 182 ++++++++++++++++++ 5 files changed, 210 insertions(+), 14 deletions(-) create mode 100644 test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_dates_builtins.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 12e18dc0b8..56bfe44c65 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -4,6 +4,7 @@ - Add: log deprecation traces for usage of legacyForwarding mode in registrations - Add: log deprecation traces for usage of attrsFormat legacy in subscriptions - Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) +- Fix: builtin attributes alterationType, dateCreated and dateModified were not included in notifications when onlyChangedAttrs was true - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: update forwarding was not working when entity type is not included in the request (#4460) - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 956d832686..bf07439d41 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -657,7 +657,7 @@ rendered by Orion to provide extra information. From a representation point of v are just like regular attributes, with name, value and type. Builtin attributes are not rendered by default. In order to render a specific attribute, add its -name to the `attrs` parameter in URLs (or payload field in POST /v2/op/query operation) or +name to the `attrs` parameter in URLs (or payload field in `POST /v2/op/query` operation) or subscription (`attrs` sub-field within `notification`). The list of builtin attributes is as follows: @@ -683,6 +683,13 @@ the subscriptions based in alteration type features (see [Subscription based in Like regular attributes, they can be used in `q` filters and in `orderBy` (except `alterationType`). However, they cannot be used in resource URLs. +The following builtin attributes are included in notifications (if added to `attrs` sub-field within `notification`) even +when `onlyChangedAttrs` is set to `true`: + +* `alterationType` +* `dateCreated` +* `dateModified` + ## Special Metadata Types Generally speaking, user-defined metadata types are informative; they are processed by Orion diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp index 599d69af8d..b582615eb7 100644 --- a/src/lib/mongoBackend/MongoGlobal.cpp +++ b/src/lib/mongoBackend/MongoGlobal.cpp @@ -2296,9 +2296,10 @@ static void getCommonAttributes { for (unsigned int avOc = 0; avOc < sVector.size(); ++avOc) { - if (fVector[cavOc] == sVector[avOc]) + // some builtin attributes are always include (even when onlyChangedAttrs is true) + if ((sVector[avOc] == ALTERATION_TYPE) || (sVector[avOc] == DATE_CREATED) || (sVector[avOc] == DATE_MODIFIED) || (fVector[cavOc] == sVector[avOc])) { - resultVector.push_back(fVector[cavOc]); + resultVector.push_back(sVector[avOc]); } } } diff --git a/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type.test b/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type.test index c0798a16e8..ea2d5f67ea 100644 --- a/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type.test +++ b/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type.test @@ -33,9 +33,9 @@ accumulatorStart --pretty-print # # 01. Create entity E-A,B,C -# 02. Create a subscription with onlyChanged: true and alterationType +# 02. Create a subscription with onlyChangedAttrs: true and alterationType # 03. Update E-A -# 04. Dump notification get E-A +# 04. Dump notification get E-A/alterationType # @@ -62,8 +62,8 @@ echo echo -echo '02. Create a subscription with onlyChanged: true and alterationType' -echo '===================================================================' +echo '02. Create a subscription with onlyChangedAttrs: true and alterationType' +echo '========================================================================' payload='{ "subject": { "entities": [ @@ -101,8 +101,8 @@ echo echo -echo '04. Dump notification get E-A' -echo '=============================' +echo '04. Dump notification get E-A/alterationType' +echo '============================================' accumulatorDump echo echo @@ -119,8 +119,8 @@ Content-Length: 0 -02. Create a subscription with onlyChanged: true and alterationType -=================================================================== +02. Create a subscription with onlyChangedAttrs: true and alterationType +======================================================================== HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -137,11 +137,11 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -04. Dump notification get E-A -============================= +04. Dump notification get E-A/alterationType +============================================ POST http://localhost:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 122 +Content-Length: 192 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: localhost:REGEX(\d+) @@ -157,6 +157,11 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Number", "value": 2 }, + "alterationType": { + "metadata": {}, + "type": "Text", + "value": "entityChange" + }, "id": "E", "type": "T" } diff --git a/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_dates_builtins.test b/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_dates_builtins.test new file mode 100644 index 0000000000..bf42e38344 --- /dev/null +++ b/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_dates_builtins.test @@ -0,0 +1,182 @@ +# Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Notify only attributes that change get with dateCreated and dateModified + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + + +--SHELL-- + +# +# 01. Create entity E-A,B,C +# 02. Create a subscription with onlyChangedAttrs: true and dateCreated/dateModified +# 03. Update E-A +# 04. Dump notification get E-A/dateCreated/dateModified +# + + +echo '01. Create entity E-A,B,C' +echo '=========================' +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo '02. Create a subscription with onlyChangedAttrs: true and dateCreated/dateModified' +echo '==================================================================================' +payload='{ + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "A","B","C", "dateModified", "dateCreated" + ], + "onlyChangedAttrs": true + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Update E-A' +echo '==============' +payload='{ + "A": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs -X PATCH --payload "$payload" +echo +echo + + +echo '04. Dump notification get E-A/dateCreated/dateModified' +echo '======================================================' +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create entity E-A,B,C +========================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +02. Create a subscription with onlyChangedAttrs: true and dateCreated/dateModified +================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Update E-A +============== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump notification get E-A/dateCreated/dateModified +====================================================== +POST http://localhost:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 289 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: localhost:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "dateCreated": { + "metadata": {}, + "type": "DateTime", + "value": "REGEX(.*)" + }, + "dateModified": { + "metadata": {}, + "type": "DateTime", + "value": "REGEX(.*)" + }, + "id": "E", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From b8843f3cc21d082fe42dc67461c8928e7e51e729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 16 Jan 2024 11:41:23 +0100 Subject: [PATCH 091/390] FIX CNR --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 56bfe44c65..161d84c644 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -4,7 +4,7 @@ - Add: log deprecation traces for usage of legacyForwarding mode in registrations - Add: log deprecation traces for usage of attrsFormat legacy in subscriptions - Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) -- Fix: builtin attributes alterationType, dateCreated and dateModified were not included in notifications when onlyChangedAttrs was true +- Fix: builtin attributes alterationType, dateCreated and dateModified included in notifications even when onlyChangedAttrs was true - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: update forwarding was not working when entity type is not included in the request (#4460) - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) From 3ac33cb848d29c166cc268268980537a117062b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 16 Jan 2024 11:42:30 +0100 Subject: [PATCH 092/390] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index bcf5dd812f..cfece4418e 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -5,7 +5,7 @@ - Add: log deprecation traces for usage of attrsFormat legacy in subscriptions - Add: deprecatedFeatures counters block in GET /v2/statistics - Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) -- Fix: builtin attributes alterationType, dateCreated and dateModified included in notifications even when onlyChangedAttrs was true +- Fix: builtin attributes alterationType, dateCreated and dateModified included in notifications even when onlyChangedAttrs is true - Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) - Fix: update forwarding was not working when entity type is not included in the request (#4460) - Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) From 82c7e1dca8c7ca444191b5f12183ff5fd817af5a Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 16 Jan 2024 21:48:11 +0900 Subject: [PATCH 093/390] (JP) ADD servicePath note in orion-api.md (#4477) --- doc/manuals.jp/orion-api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index dbd7c90f21..9b930131b9 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -564,6 +564,8 @@ Orion は階層スコープをサポートしているため、エンティテ - `Fiware-ServicePath` ヘッダは、Orion から送信される通知リクエストに含まれています +- [`servicePath` 組み込み属性](#builtin-attributes)を使用してエンティティ・サービス・パスを取得できます + - スコープ・エンティティは、[マルチ・テナンシー機能](#multi-tenancy) と直交的に組み合わせることができます。 その場合、各 `scope tree` (スコープ・ツリー) は異なるサービス/テナントに存在し、完全なデータベース ベースの分離で 同じ名前を使用することもできます。下の図を参照してください。 From ae910fb0c6a147428c29a0031cf85cddb5fd3041 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 16 Jan 2024 21:56:36 +0900 Subject: [PATCH 094/390] (JP) ADD deprecatedFeatures counters (#4476) --- doc/manuals.jp/admin/logs.md | 3 +++ doc/manuals.jp/admin/statistics.md | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/doc/manuals.jp/admin/logs.md b/doc/manuals.jp/admin/logs.md index 9697b65ac5..140e29de3a 100644 --- a/doc/manuals.jp/admin/logs.md +++ b/doc/manuals.jp/admin/logs.md @@ -367,6 +367,9 @@ time=2023-06-08T15:14:20.999Z | lvl=WARN | corr=24fd2acc-060f-11ee-94cc-000c2958 time=2023-06-08T15:14:21.176Z | lvl=WARN | corr=2518249e-060f-11ee-9e76-000c29583ca5 | trans=1686237259-703-00000000004 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=location.cpp[353]:getGeoJson | msg=Deprecated usage of geo:point detected in attribute location at entity update, please use geo:json instead ``` +非推奨機能の使用状況は、`-logDeprecate` CLI (またはログ管理 REST API の同等の `deprecate` パラメータ) が使用されていない場合でも、 +[統計 API の counters ブロック](statistics#counter-block) の `deprecatedFeature` オブジェクトで追跡されます。 + 非推奨の機能とその解決方法の詳細については、[非推奨ドキュメント](../deprecated.md) を参照してください。 [トップ](#top) diff --git a/doc/manuals.jp/admin/statistics.md b/doc/manuals.jp/admin/statistics.md index b20f5234ac..7d5580f380 100644 --- a/doc/manuals.jp/admin/statistics.md +++ b/doc/manuals.jp/admin/statistics.md @@ -42,6 +42,13 @@ Orion Context broker は、`GET /statistics` と `GET /cache/statistics` を介 { ... "counters": { + "deprecatedFeatures": { + "geoFormat": 2, + "metadataLocation": 1, + "ngsiv1Forwarding": 4, + "ngsiv1NotifFormat": 4, + "ngsiv1Requests": 4 + }, "invalidRequests": 2, "jsonRequests": 4, "noPayloadRequests": 250, From 56675e5e8a154cab1ac67ef2b2d5acd40932a140 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 16 Jan 2024 22:11:01 +0900 Subject: [PATCH 095/390] (JP) ADD documentation about builtin attributes included in notifications (#4471) --- doc/manuals.jp/orion-api.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index dbd7c90f21..f81b0547c1 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -626,7 +626,7 @@ Orion は階層スコープをサポートしているため、エンティテ クライアントによって直接変更できないエンティティのプロパティがありますが、追加情報を提供するために Orion によってレンダリングすることができます。表現の観点から見ると、それらは名前、値、型とともに通常の属性と同じです。 -組み込み属性はデフォルトでレンダリングされません。特定の属性をレンダリングするには、URLs (または、POST /v2/op/query +組み込み属性はデフォルトでレンダリングされません。特定の属性をレンダリングするには、URLs (または、`POST /v2/op/query` オペレーションのペイロード・フィールド) または、サブスクリプション (`notification` 内の `attrs` サブフィールド) の `attrs` パラメータにその名前を追加してください。 @@ -652,6 +652,13 @@ Orion は階層スコープをサポートしているため、エンティテ 通常の属性と同様に、`q` フィルタと `orderBy` (`alterationType` を除く) で使用できます。 ただし、リソース URLs では使用できません。 +次の組み込み属性は、`onlyChangedAttrs` が `true` に設定されている場合でも、通知に含まれます (`notification` 内の `attrs` +サブ・フィールドに追加された場合): + +- `alterationType` +- `dateCreated` +- `dateModified` + ## 特殊なメタデータ型 (Special Metadata Types) From 4a6c0a5744d535d947c09a974e87f6dd879ff701 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 16 Jan 2024 22:39:49 +0900 Subject: [PATCH 096/390] (JP) Update links to docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d9de4cda24..70ad7e6957 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ recommended to have a look to the brief - Orion Context Broker [(en)](https://www.slideshare.net/slideshows/orion-context-broker-introduction-20240115/265436732) - [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-20230606-258279602) + [(jp)](https://www.slideshare.net/slideshows/orion-context-broker-20240116/265462443) - NGSIv2 Overview for Developers That Already Know NGSIv1 [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220523) [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220526) From 933614dba859f7449a9bb97639ac64db545160ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 16 Jan 2024 17:01:57 +0100 Subject: [PATCH 097/390] FIX onlyChagnedAttrs true with alterationType for entityDelete case --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 4 +- src/lib/mongoBackend/MongoGlobal.cpp | 20 ++- ...with_alteration_type_on_entity_delete.test | 167 ++++++++++++++++++ 3 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type_on_entity_delete.test diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 2c1ad6c507..eca3ed90b0 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -3533,7 +3533,7 @@ static unsigned int updateEntity attrNames.push_back(eP->attributeVector[ix]->name); } - // Note we cannot usse eP->type for the type, as it may be blank in the request + // Note we cannot use eP->type for the type, as it may be blank in the request // (that would break the cases/1494_subscription_alteration_types/sub_alteration_type_entity_delete2.test case) if (!addTriggeredSubscriptions(notifyCerP->entity.id, notifyCerP->entity.type, @@ -3557,7 +3557,7 @@ static unsigned int updateEntity addBuiltins(notifyCerP, subAltType2string(ngsiv2::SubAltType::EntityDelete)); unsigned int notifSent = processSubscriptions(subsToNotify, notifyCerP, - tenant, + tenant, xauthToken, fiwareCorrelator, notifStartCounter); diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp index b582615eb7..c21c65f6a3 100644 --- a/src/lib/mongoBackend/MongoGlobal.cpp +++ b/src/lib/mongoBackend/MongoGlobal.cpp @@ -2292,14 +2292,22 @@ static void getCommonAttributes std::vector& resultVector ) { - for (unsigned int cavOc = 0; cavOc < fVector.size(); ++cavOc) + for (unsigned int avOc = 0; avOc < sVector.size(); ++avOc) { - for (unsigned int avOc = 0; avOc < sVector.size(); ++avOc) + // some builtin attributes are always include (even when onlyChangedAttrs is true) + if ((sVector[avOc] == ALTERATION_TYPE) || (sVector[avOc] == DATE_CREATED) || (sVector[avOc] == DATE_MODIFIED)) { - // some builtin attributes are always include (even when onlyChangedAttrs is true) - if ((sVector[avOc] == ALTERATION_TYPE) || (sVector[avOc] == DATE_CREATED) || (sVector[avOc] == DATE_MODIFIED) || (fVector[cavOc] == sVector[avOc])) - { - resultVector.push_back(sVector[avOc]); + resultVector.push_back(sVector[avOc]); + } + else + { + for (unsigned int cavOc = 0; cavOc < fVector.size(); ++cavOc) + { + if (fVector[cavOc] == sVector[avOc]) + { + resultVector.push_back(sVector[avOc]); + break; + } } } } diff --git a/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type_on_entity_delete.test b/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type_on_entity_delete.test new file mode 100644 index 0000000000..5617ed8fcf --- /dev/null +++ b/test/functionalTest/cases/3190_notify_only_attributes_that_change/notify_only_attributes_that_change_with_alteration_type_on_entity_delete.test @@ -0,0 +1,167 @@ +# Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Notify only attributes that change get with alterationType on entityDelete + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + + +--SHELL-- + +# +# 01. Create entity E-A,B,C +# 02. Create a subscription with onlyChangedAttrs: true, alterationsTypes: entityDelete and notify alterationType +# 03. Delete E +# 04. Dump notification get E with alterationType entityDelete +# + + +echo '01. Create entity E-A,B,C' +echo '=========================' +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 1, + "type": "Number" + }, + "C": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo '02. Create a subscription with onlyChangedAttrs: true, alterationsTypes: entityDelete and notify alterationType' +echo '===============================================================================================================' +payload='{ + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ], + "condition": { + "alterationTypes": [ "entityDelete" ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ "alterationType" ], + "onlyChangedAttrs": true + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Delete E' +echo '============' +orionCurl --url /v2/entities/E -X DELETE +echo +echo + + +echo '04. Dump notification get E with alterationType entityDelete' +echo '============================================================' +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create entity E-A,B,C +========================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +02. Create a subscription with onlyChangedAttrs: true, alterationsTypes: entityDelete and notify alterationType +=============================================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Delete E +============ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump notification get E with alterationType entityDelete +============================================================ +POST http://localhost:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 146 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: localhost:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "alterationType": { + "metadata": {}, + "type": "Text", + "value": "entityDelete" + }, + "id": "E", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From f4e5eb253d0ab87d55cd0a100660ee6dce3de2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 16 Jan 2024 17:03:12 +0100 Subject: [PATCH 098/390] REMOVE trailing whitespace --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 2 +- src/lib/mongoBackend/MongoGlobal.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index eca3ed90b0..1a3e667084 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -3557,7 +3557,7 @@ static unsigned int updateEntity addBuiltins(notifyCerP, subAltType2string(ngsiv2::SubAltType::EntityDelete)); unsigned int notifSent = processSubscriptions(subsToNotify, notifyCerP, - tenant, + tenant, xauthToken, fiwareCorrelator, notifStartCounter); diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp index c21c65f6a3..9b0cca0054 100644 --- a/src/lib/mongoBackend/MongoGlobal.cpp +++ b/src/lib/mongoBackend/MongoGlobal.cpp @@ -2292,7 +2292,7 @@ static void getCommonAttributes std::vector& resultVector ) { - for (unsigned int avOc = 0; avOc < sVector.size(); ++avOc) + for (unsigned int avOc = 0; avOc < sVector.size(); ++avOc) { // some builtin attributes are always include (even when onlyChangedAttrs is true) if ((sVector[avOc] == ALTERATION_TYPE) || (sVector[avOc] == DATE_CREATED) || (sVector[avOc] == DATE_MODIFIED)) From b49717a9d01b35b33f21e79055c4c51a24f3d423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 22 Jan 2024 18:17:42 +0100 Subject: [PATCH 099/390] ADD first version of the entities_consistency.py script (along with test and documentation) --- scripts/entities_consistency/README.md | 176 + .../entities_consistency.py | 848 ++++ scripts/entities_consistency/requirements.txt | 2 + .../test_entities_consistency.py | 55 + .../entities_consistency/validation_data.js | 3530 +++++++++++++++++ 5 files changed, 4611 insertions(+) create mode 100644 scripts/entities_consistency/README.md create mode 100644 scripts/entities_consistency/entities_consistency.py create mode 100644 scripts/entities_consistency/requirements.txt create mode 100644 scripts/entities_consistency/test_entities_consistency.py create mode 100644 scripts/entities_consistency/validation_data.js diff --git a/scripts/entities_consistency/README.md b/scripts/entities_consistency/README.md new file mode 100644 index 0000000000..34cd10846f --- /dev/null +++ b/scripts/entities_consistency/README.md @@ -0,0 +1,176 @@ +The entities consistency script analyze the contents of the entities database in orion DBs and +check several consistency rules, reporting violations found. + +Ref: [entity document datamodel]([../..//doc/manuals/admin/database_model.md#entities-collection]) + +## Requirements + +This script is designed to work with Python 3. Install the dependencies in the `requirements.txt` file before using it. +Usage of virtual env is recommended. + +## Usage + +Run `entities_consistency.py -h` for arguments details. + +## Rules + +* Rules 1x: DB inconsistencies (severe problems) +* Rules 2x: Syntax restrictions +* Rules 9x: Usage of legacy features + +### Rule 10: `_id` field inconsistency + +Each entity in DB has a `_id` fields with three subfields: + +* `id` +* `type` +* `servicePath` + +## Rule 11: mandatory fields in entity + +The following field are mandatory: + +* `attrNames` +* `creDate` +* `modDate` + +It is not an exhaustive check of every attribute in the datamodel, but some entities created/updated with old Orion version may be missing them. + +## Rule 12: mandatory fields in attribute + +The following subfield are mandatory for every attribute: + +* `mdNames` +* `creDate` +* `modDate` + +It is not an exhaustive check of every attribute in the datamodel, but some entities created/updated with old Orion version may be missing them. + +### Rule 13: `attrNames` field inconsistency + +Each item in `attrNames` array has a corresponding key in `attrs` object and the other way around. + +### Rule 14: `mdNames` field inconsistency + +For every attribute, for each item in `mdNames` array has a corresponding key in `md` object and the other way around. + +### Rule 15: swapped subkeys in `_id` + +In MongoDB JSON objects are stored taking order into account, so DB allows to have a document with +`_id` equal to `{"id": "E", "type": "T", "servicePath": "/"}` and at the same time have another document with `_id` +equal to `{"type": "T", "id": "E", "servicePath": "/"}` without violating `_id` uniqueness constraint. + +This rule checks that this is not happening. + +### Rule 16: `location` field inconsistency + +Check that location in consistent. In particular: + +* As much as one attribute with `geo:point`, `geo:line`, `geo:box`, `geo:polygon` or `geo:json` type +* If an attribute is found with one of above types, check the `location` field is consistent +* If no attribute is found with one of above types, check no `location` field is not found + * If the location of the entity is defined using deprecated `location` metadata it will be detected as this case. Additional rules in the 9x group can help to diagnose this situation. + +This rule is for location inconsistencies. For usage of deprecated geo types, there are additional rules in the 9x group. + +### Rule 17: missing `lastCorrelator` + +Check if `lastCorrelator` is included. + +This field was introduced in Orion 1.8.0 (September 11th, 2017). + +### Rule 20: entity id syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 21: entity type syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 22: entity servicePath syntax + +https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path + +### Rule 23: attribute name syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 24: attribute type syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 25: metadata name syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 26: metadata type syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 90: usage of `geo:x` attribute type where `x` different from `json` + +Check usage of deprecated geo-location types, i.e: + +* `geo:point` +* `geo:line` +* `geo:box` +* `geo:polygon` + +Suggested action is to: + +* Change attribute type to `geo:json` +* Set the attribute value to the same GeoJSON in `location.coords` field + +Note this rule doesn't check location consistency for this case (e.g. more than one geo-location attribute in the same entity). That's done by another rule in the 1x group). + +### Rule 91: usage of more than one legacy `location` metadata + +Check usage of `location` in more than one attribute of the same entity. + +Note this rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +### Rule 92: legacy `location` metadata should be `WGS84` or `WSG84` + +The value of the `location` metadata should be `WGS84` or `WSG84`. + +Additional consideration: + +* Entities attributes may have `location` metadata with values different from `WGS84` or `WSG84` if created using NGSIv2. That would be a false positive in this rule validation +* This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +### Rule 93: usage of redundant legacy `location` + +Checks usage of redundant `location` metadata, i.e. when at the same time a `geo:` type is used in the +same attribute. + +Suggested action is to remove the `location` metadata. + +Additional, considerations: + + * This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. + * This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +### Rule 94: usage of not redundant legacy `location` + +Checks usage of not redundant `location` metadata, i.e. when at the same time the type of the attribute is nog `geo:`. +same attribute. + +Suggested action is to: + +* Change attribute type to `geo:json` +* Set the attribute value to the same GeoJSON in `location.coords` field +* Remove the `location` metadata from the attribute + +Additional, considerations: + +* This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. +* This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +## Testing + +You can test the `entities_consistency.py` script this qy: + +1. Populate `orion-validation` DB with testing document. To do so, copy-paste the content of the `validation_data.js` in `mongosh` +2. Run `test_entities_consistency.py` test and check the log output + +You can also run `test_entities_consistenct.py` under coverage to check every rule is covering all the possible validation cases. diff --git a/scripts/entities_consistency/entities_consistency.py b/scripts/entities_consistency/entities_consistency.py new file mode 100644 index 0000000000..8caeb1f3a0 --- /dev/null +++ b/scripts/entities_consistency/entities_consistency.py @@ -0,0 +1,848 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +__author__ = 'fermin' + +from pymongo import MongoClient +from deepdiff import DeepDiff +import argparse +import logging +import json +import sys +import re + + +# Helper functions + +def is_geo_type(attr_type): + """ + Return True if attr type passed as argument is a geo type + + :param attr_type: the attr type to evaluate + """ + + return attr_type == 'geo:point' or attr_type == 'geo:line' or attr_type == 'geo:box' \ + or attr_type == 'geo:polygon' or attr_type == 'geo:json' + + +def ignore_type(attr): + """ + Return true if attribute has the ignoreType metadata + """ + return 'md' in attr and 'ignoreType' in attr['md'] + + +def to_geo_json(attr): + """ + Return the GeoJSON corresponding to an attribute location, taking into account the type + + Useful ref: https://github.com/telefonicaid/fiware-orion/blob/3.9.0/doc/manuals/orion-api.md + """ + + if attr['type'] == 'geo:point': + # "value": "41.3763726, 2.186447514", + coords = attr['value'].split(",") + return { + 'type': 'Point', + 'coordinates': [float(coords[1]), float(coords[0])] + } + elif attr['type'] == 'geo:line': + # "value": [ + # "40.63913831188419, -8.653321266174316", + # "40.63881265804603, -8.653149604797363" + # ] + coordinates = [] + for item in attr['value']: + coords = item.split(",") + coordinates.append([float(coords[1]), float(coords[0])]) + return { + 'type': 'LineString', + 'coordinates': coordinates + } + elif attr['type'] == 'geo:box': + # "value": [ + # "40.63913831188419, -8.653321266174316", + # "40.63881265804603, -8.653149604797363" + # ] + # The first pair is the lower corner, the second is the upper corner. + lower_corner_0 = attr['value'][0].split(",") + upper_corner_0 = attr['value'][1].split(",") + lower_corner_1 = [upper_corner_0[0], lower_corner_0[1]] + upper_corner_1 = [lower_corner_0[0], upper_corner_0[1]] + return { + 'type': 'Polygon', + 'coordinates': [[ + [float(lower_corner_0[1]), float(lower_corner_0[0])], + [float(lower_corner_1[1]), float(lower_corner_1[0])], + [float(upper_corner_0[1]), float(upper_corner_0[0])], + [float(upper_corner_1[1]), float(upper_corner_1[0])], + [float(lower_corner_0[1]), float(lower_corner_0[0])] + ]] + } + elif attr['type'] == 'geo:polygon': + # "value": [ + # "40.63913831188419, -8.653321266174316", + # "40.63881265804603, -8.653149604797363", + # "40.63913831188419, -8.653321266174316" + # ] + coordinates = [] + for item in attr['value']: + coords = item.split(",") + last = [float(coords[1]), float(coords[0])] + coordinates.append([float(coords[1]), float(coords[0])]) + coords.append(last) # so we have a closed shape + return { + 'type': 'Polygon', + 'coordinates': [coordinates] + } + elif attr['type'] == 'geo:json': + return attr['value'] + else: + logger.error(f"unknown geo location type: {attr['type']}") + return None + + +def convert_strings_to_numbers(data): + """ + Generated by ChatGPT :) + """ + if isinstance(data, dict): + # If it's a dictionary, apply the function recursively to its values + return {key: convert_strings_to_numbers(value) for key, value in data.items()} + elif isinstance(data, list): + # If it's a list, apply the function recursively to its elements + return [convert_strings_to_numbers(item) for item in data] + elif isinstance(data, str): + # If it's a string, try converting it to a number + try: + return int(data) + except ValueError: + try: + return float(data) + except ValueError: + # If it's not a valid number, leave it as it is + return data + else: + # If it's neither a dictionary, list, nor string, leave it as it is + return data + + +def check_id(id): + """ + Common checks for several rules + + Ref: https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#identifiers-syntax-restrictions + """ + # Minimum field length is 1 character + l = len(id) + if l == 0: + return f'length ({l}) shorter than minimum allowed (1)' + # Maximum field length is 256 characters + if l > 256: + return f'length ({l}) greater than maximum allowed (256)' + + # Following chars are not used (general): < > " ' = ; ( ) + if re.search('[<>"\'=;()]', id): + return f'contains forbidden chars (general)' + + # Following chars are not used (identifiers): whitespace, & ? / # + if re.search('[\s&?/#]', id): + return f'contains forbidden chars (identifiers)' + + return None + + +# Rules functions +def rule10(entity): + """ + Rule 10: `_id` field inconsistency + + See README.md for an explanation of the rule + """ + missing_fields = [] + + for field in ['id', 'type', 'servicePath']: + if field not in entity['_id']: + missing_fields.append(field) + + if len(missing_fields) > 0: + return f"missing subfields in _id: {', '.join(missing_fields)}" + else: + return None + + +def rule11(entity): + """ + Rule 11: mandatory fields in entity + + See README.md for an explanation of the rule + """ + missing_fields = [] + + for field in ['attrNames', 'creDate', 'modDate']: + if field not in entity: + missing_fields.append(field) + + if len(missing_fields) > 0: + return f"missing fields: {', '.join(missing_fields)}" + else: + return None + + +def rule12(entity): + """ + Rule 12: mandatory fields in attribute + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs'].keys(): + missing_fields = [] + + for field in ['mdNames', 'creDate', 'modDate']: + if field not in entity['attrs'][attr]: + missing_fields.append(field) + + if len(missing_fields) > 0: + s.append(f"in attribute '{attr}' missing fields: {', '.join(missing_fields)}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule13(entity): + """ + Rule 13: `attrNames` field inconsistency + + See README.md for an explanation of the rule + """ + # note the omission of attrNames is checked by another rule. In this rule we include + # some guards for not breaking in that case + + # attrNames in attrs + attrnames_not_in_attrs = [] + if 'attrNames' in entity: + for attr in entity['attrNames']: + if attr.replace('.', '=') not in entity['attrs']: + attrnames_not_in_attrs.append(attr) + + # attrs in attrNames + attrs_not_in_attrnames = [] + if 'attrs' in entity: + for attr in entity['attrs'].keys(): + if 'attrNames' not in entity or attr.replace('=', '.') not in entity['attrNames']: + attrs_not_in_attrnames.append(attr) + + s = [] + if len(attrnames_not_in_attrs) > 0: + s.append(f"attributes in attrNames not found in attrs object: {','.join(attrnames_not_in_attrs)}") + if len(attrs_not_in_attrnames) > 0: + s.append(f"attributes in attrs object not found in attrNames: {','.join(attrs_not_in_attrnames)}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule14(entity): + """ + Rule 14: `mdNames` field inconsistency + + See README.md for an explanation of the rule + """ + # note the omission of mdNames is checked by another rule. In this rule we include + # some guards for not breaking in that case + + s = [] + for item in entity['attrs'].keys(): + attr = entity['attrs'][item] + # mdNames in md + if 'mdNames' in attr: + mdnames_not_in_md = [] + for md in attr['mdNames']: + if md.replace('.', '=') not in attr['md']: + mdnames_not_in_md.append(md) + + # md in mdNames + md_not_in_mdnames = [] + if 'md' in attr: + for md in attr['md'].keys(): + if 'mdNames' not in attr or md.replace('=', '.') not in attr['mdNames']: + md_not_in_mdnames.append(md) + + if len(mdnames_not_in_md) > 0: + s.append( + f"in attribute '{item}' metadata in mdNames not found in md object: {', '.join(mdnames_not_in_md)}") + if len(md_not_in_mdnames) > 0: + s.append( + f"in attribute '{item}' metadata in md object not found in mdNames: {', '.join(md_not_in_mdnames)}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule15(entities_collection): + """ + Rule 15: swapped subkeys in `_id` + + See README.md for an explanation of the rule + + This is a global rule, so the parameter is not an individual entity but the whole entities collection. + """ + s = [] + for entity in entities_collection.aggregate( + [ + { + '$group': { + '_id': {'id': '$_id.id', 'type': '$_id.type', 'servicePath': '$_id.servicePath'}, + 'count': {'$sum': 1} + } + }, + { + '$match': {'count': {'$gt': 1}} + } + ] + ): + id = entity['_id']['id'] + type = entity['_id']['type'] + service_path = entity['_id']['servicePath'] + count = entity['count'] + s.append( + f"_id uniqueness violation for entity id='{id}' type='{type}' servicePath='{service_path}' found {count} times") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule16(entity): + """ + Rule 16: `location` field inconsistency + + See README.md for an explanation of the rule + """ + # check that as much as one attribute is using geo type + geo_attrs = [] + for attr in entity['attrs']: + if is_geo_type(entity['attrs'][attr]['type']) and not ignore_type(entity['attrs'][attr]): + geo_attrs.append(attr) + + if len(geo_attrs) > 1: + return f"more than one attribute with geo type: {', '.join(geo_attrs)}" + + if len(geo_attrs) == 1: + # If geo attr found, then check that there is consistent location field + geo_attr = geo_attrs[0] + geo_type = entity['attrs'][geo_attr]['type'] + if entity['attrs'][geo_attr]['value'] is None: + # if null value in geo location attribute, then location field must not be present + if 'location' in entity: + return f"geo location '{geo_attr}' ({geo_type}) with null value and location field found" + else: + # not null value in geo location attribute case + if 'location' not in entity: + return f"geo location detected in '{geo_attr}' ({geo_type}) but location field not found in entity" + if entity['location']['attrName'] != geo_attr: + return f"location.attrName ({entity['location']['attrName']}) differs from '{geo_attr}'" + + geo_json = to_geo_json(entity['attrs'][geo_attr]) + + # https://www.testcult.com/deep-comparison-of-json-in-python/ + diff = DeepDiff(geo_json, entity['location']['coords'], ignore_order=True) + if diff: + # A typical difference is that attribute value uses strings and location uses numbers + # (this happens when the location was created/updated using NGSIv1). We try to identify that case + geo_json = convert_strings_to_numbers(geo_json) + if not DeepDiff(geo_json, entity['location']['coords'], ignore_order=True): + return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) is consistent, but value " \ + f"should use numbers for coordinates instead of strings" + else: + # Other causes + return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) value: {diff}" + else: # len(geo_attrs) == 0 + # If no geo attr found, check there isn't a location field + if 'location' in entity: + return f"location field detected but no geo attribute is present (or metadata location is used)" + + +def rule17(entity): + """ + Rule 17: missing `lastCorrelator` + + See README.md for an explanation of the rule + """ + if 'lastCorrelator' not in entity: + return f"missing lastCorrelator" + else: + return None + + +def rule20(entity): + """ + Rule 20: entity id syntax + + See README.md for an explanation of the rule + """ + + # The existence of id in _id is checked by another rule + if 'id' in entity['_id']: + r = check_id(entity['_id']['id']) + if r is not None: + return f"entity id ({entity['_id']['id']}) syntax violation: {r}" + + return None + + +def rule21(entity): + """ + Rule 21: entity type syntax + + See README.md for an explanation of the rule + """ + + # The existence of type in _id is checked by another rule + if 'type' in entity['_id']: + r = check_id(entity['_id']['type']) + if r is not None: + return f"entity type ({entity['_id']['type']}) syntax violation: {r}" + + return None + + +def rule22(entity): + """ + Rule 22: entity servicePath syntax + + See README.md for an explanation of the rule + + Ref: https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path + """ + + # servicePath existence is checked by another rule + if 'servicePath' not in entity['_id']: + return None + + sp = entity['_id']['servicePath'] + # Scope must start with / (only "absolute" scopes are allowed) + if not sp.startswith('/'): + return f"servicePath '{sp}' does not starts with '/'" + + # 10 maximum scope levels in a path + sp_levels = sp.split('/') + if len(sp_levels) > 10: + return f"servicePath has {len(sp_levels)} tokens but the limit is 10" + + # 50 maximum characters in each level (1 char is minimum), only alphanumeric and underscore allowed + for level in sp_levels: + if len(level) == 0: + return f'servicePath length is 0 but minimum is 1' + if len(level) > 50: + return f'servicePath length is {len(level)} but maximum is 50' + if re.search('[^a-zA-Z0-9_]', level): + return f"unallowed characters in '{level}' in servicePath level" + + +def rule23(entity): + """ + Rule 23: attribute name syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + r = check_id(attr) + if r is not None: + s.append(f"attribute name ({attr}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule24(entity): + """ + Rule 24: attribute type syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + if 'type' not in entity['attrs'][attr]: + s.append(f"in attribute '{attr}' type is missing") + else: + type = entity['attrs'][attr]['type'] + r = check_id(type) + if r is not None: + s.append(f"in attribute '{attr}' type ({type}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule25(entity): + """ + Rule 25: metadata name syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + r = check_id(md) + if r is not None: + s.append(f"in attribute '{attr}' metadata name ({md}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule26(entity): + """ + Rule 26: metadata type syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + if 'type' not in entity['attrs'][attr]['md'][md]: + s.append(f"in attribute '{attr}' metadata '{md}' type is missing") + else: + type = entity['attrs'][attr]['md'][md]['type'] + r = check_id(type) + if r is not None: + s.append(f"in attribute '{attr}' metadata '{md}' type ({type}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule90(entity): + """ + Rule 90: usage of `geo:x` attribute type where `x` different from `json` + + See README.md for an explanation of the rule + """ + for attr in entity['attrs'].keys(): + type = entity['attrs'][attr]['type'] + if is_geo_type(type) and type != 'geo:json': + return f"in attribute '{attr}' usage of deprecated {type} found" + + return None + + +def rule91(entity): + """ + Rule 91: usage of more than one legacy `location` metadata + + See README.md for an explanation of the rule + """ + n = 0 + for attr in entity['attrs'].keys(): + if 'md' in entity['attrs'][attr]: + if 'location' in entity['attrs'][attr]['md'].keys(): + n += 1 + + if n > 1: + return f'location metadata found {n} times among entity attributes (maximum should be just 1)' + else: + return None + + +def rule92(entity): + """ + Rule 92: legacy `location` metadata should be `WGS84` or `WSG84` + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs'].keys(): + if 'md' in entity['attrs'][attr]: + if 'location' in entity['attrs'][attr]['md'].keys(): + location_value = entity['attrs'][attr]['md']['location']['value'] + if location_value != 'WGS84' and location_value != 'WSG84': + s.append(f"in attribute '{attr}' location metadata value is {location_value} (should be WGS84 or WSG84)") + + if len(s) > 1: + return ', '.join(s) + else: + return None + + +def rule93(entity): + """ + Rule 93: usage of redundant legacy `location` + + See README.md for an explanation of the rule + """ + for attr in entity['attrs'].keys(): + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + if md == 'location': + if is_geo_type(entity['attrs'][attr]['type']): + return f"in attribute '{attr}' redundant location metadata found (attribute is " \ + f"already using {entity['attrs'][attr]['type']} type)" + + return None + + +def rule94(entity): + """ + Rule 94: usage of not redundant legacy `location` + + See README.md for an explanation of the rule + """ + for attr in entity['attrs'].keys(): + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + if md == 'location': + if not is_geo_type(entity['attrs'][attr]['type']): + return f"in attribute '{attr}' location metadata found (attribute type " \ + f"is {entity['attrs'][attr]['type']})" + + return None + + +rules = [ + # Rules 1x + { + 'label': 'Rule10', + 'global': False, + 'func': rule10 + }, + { + 'label': 'Rule11', + 'global': False, + 'func': rule11 + }, + { + 'label': 'Rule12', + 'global': False, + 'func': rule12 + }, + { + 'label': 'Rule13', + 'global': False, + 'func': rule13 + }, + { + 'label': 'Rule14', + 'global': False, + 'func': rule14 + }, + { + 'label': 'Rule15', + 'global': True, + 'func': rule15 + }, + { + 'label': 'Rule16', + 'global': False, + 'func': rule16 + }, + { + 'label': 'Rule17', + 'global': False, + 'func': rule17 + }, + # Rules 2x + { + 'label': 'Rule20', + 'global': False, + 'func': rule20 + }, + { + 'label': 'Rule21', + 'global': False, + 'func': rule21 + }, + { + 'label': 'Rule22', + 'global': False, + 'func': rule22 + }, + { + 'label': 'Rule23', + 'global': False, + 'func': rule23 + }, + { + 'label': 'Rule24', + 'global': False, + 'func': rule24 + }, + { + 'label': 'Rule25', + 'global': False, + 'func': rule25 + }, + { + 'label': 'Rule26', + 'global': False, + 'func': rule26 + }, + # Rules 9x + { + 'label': 'Rule90', + 'global': False, + 'func': rule90 + }, + { + 'label': 'Rule91', + 'global': False, + 'func': rule91 + }, + { + 'label': 'Rule92', + 'global': False, + 'func': rule92 + }, + { + 'label': 'Rule93', + 'global': False, + 'func': rule93 + }, + { + 'label': 'Rule94', + 'global': False, + 'func': rule94 + } +] + + +def process_db(logger, db_name, db_conn, query, rules_exp): + """ + Process an individual DB + + :param logger: logger object + :param db_name: the name of the DB to process + :param db_conn: connection to MongoDB + :param query: query to filter entities to be processed + :param rules_exp: regular expression to filter rules to apply + :return: fails + """ + + logger.info(f'Processing {db_name}') + n = 0 + fails = 0 + + # check collection existence + if 'entities' not in db_conn[db_name].list_collection_names(): + logger.warning(f'collections entities not found in {db_name} database, nothing to do') + return + + # apply global rules + for rule in rules: + if rules_exp is not None and not re.search(rules_exp, rule['label']): + continue + + if rule['global']: + s = rule['func'](db_conn[db_name]['entities']) + if s is not None: + logger.warning(f'DB {db_name} {rule["label"]} violation in entities collection: {s}') + fails += 1 + + # apply per-entity rules + for entity in db_conn[db_name]['entities'].find(query): + n += 1 + id_string = json.dumps(entity['_id']) + logger.debug(f'* processing entity {id_string}') + for rule in rules: + if rules_exp is not None and not re.search(rules_exp, rule['label']): + continue + + if not rule['global']: + s = rule['func'](entity) + if s is not None: + logger.warning(f'DB {db_name} {rule["label"]} violation for entity {id_string}: {s}') + fails += 1 + + logger.info(f'processed {n} entities ({fails} rule violations)') + + return fails + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog='entities_consistency', + description='Check consistency in Orion entities collection in DB') + + parser.add_argument('--mongoUri', dest='mongo_uri', default='mongodb://localhost:27017', + help='MongoDB URI. Default is mongodb://localhost:27017') + parser.add_argument('--db', dest='db', + help='DB name to check. If omitted all DBs starting with "orion" will be checked.') + parser.add_argument('--query', dest='query', default='{}', + help='query to filter entities to check, in JSON MongoDB query language. By default, ' + 'all entities in the collection will be checked.') + parser.add_argument('--rulesExp', dest='rules_exp', + help='Specifies the rules to apply, as a regular expression. By default all rules are applied.') + parser.add_argument('--logLevel', dest='log_level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'], default='INFO', + help='log level. Default is INFO') + args = parser.parse_args() + + # sets the logging configuration + logging.basicConfig( + level=logging.getLevelName(args.log_level), + format="time=%(asctime)s | lvl=%(levelname)s | msg=%(message)s", + handlers=[ + logging.StreamHandler() + ] + ) + logger = logging.getLogger() + + # connect to MongoDB + mongo_client = MongoClient(args.mongo_uri) + db_names = mongo_client.list_database_names() + + # to remove starting and trailing ' char, in case it is used + query = json.loads(args.query.replace("'", "")) + + fails = 0 + if args.db is not None: + if args.db in db_names: + fails += process_db(logger, args.db, mongo_client[args.db], query, args.rules_exp) + else: + logger.fatal(f'database {args.db} does not exist') + sys.exit(1) + else: + # Process all Orion databases + for db_name in db_names: + if db_name.startswith('orion-'): + fails += process_db(logger, db_name, mongo_client[db_name], query, args.rules_exp) + + logger.info(f'total rule violations: {fails}') diff --git a/scripts/entities_consistency/requirements.txt b/scripts/entities_consistency/requirements.txt new file mode 100644 index 0000000000..0c5b4e9193 --- /dev/null +++ b/scripts/entities_consistency/requirements.txt @@ -0,0 +1,2 @@ +pymongo==4.6.1 +deepdiff==6.7.1 diff --git a/scripts/entities_consistency/test_entities_consistency.py b/scripts/entities_consistency/test_entities_consistency.py new file mode 100644 index 0000000000..87013558b9 --- /dev/null +++ b/scripts/entities_consistency/test_entities_consistency.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +__author__ = 'fermin' + +# This is not the usual test that stimulates function and check assertions in the results :) +# +# Before running this test you have to load the entities testing set (validation_data.js) in the 'orion-validation' +# database in the local MongoDB database (check the MongoURI to match the one in your environment). You will get +# the result as log output. +# +# You can run this test under coverage, so you can check the coverage of the different rules in +# entities_consistency.py + +import unittest +from pymongo import MongoClient +import logging + +from entities_consistency import process_db + +class TestEntitiesConsistency(unittest.TestCase): + def test_process_db(self): + # sets the logging configuration + logging.basicConfig( + level=logging.getLevelName('INFO'), + format="time=%(asctime)s | lvl=%(levelname)s | msg=%(message)s", + handlers=[ + logging.StreamHandler() + ] + ) + logger = logging.getLogger() + + # connect to MongoDB and process validation DB + mongo_client = MongoClient('mongodb://localhost:47017') + process_db(logger, 'orion-validation', mongo_client, {}, None) diff --git a/scripts/entities_consistency/validation_data.js b/scripts/entities_consistency/validation_data.js new file mode 100644 index 0000000000..17761fc027 --- /dev/null +++ b/scripts/entities_consistency/validation_data.js @@ -0,0 +1,3530 @@ +db.getSiblingDB("orion-validation").entities.insertMany([ + { + "_id": { + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule10.1: missing entity id", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule10.2", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule10.2: missing entity type", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule10.3", + "type": "T" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule10.3: missing entity servicePath", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule11.1", + "type": "T", + "servicePath": "/SS" + }, + "attrs": { + "desc": { + "value": "Rule11.1: missing attrNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule11.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule11.2: missing entity creDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule11.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule11.3: missing entity modDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule12.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule12.1: missing attribute mdNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + } + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule12.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule12.2: missing attribute creDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule12.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule12.3: missing attribute modDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule13.1: attribute in attrsName but not in attrs object", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule13.2: attribute in attrs object but not in attrNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule13.3: attribute in attrsName but not in attrs object with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2" + ], + "attrs": { + "desc": { + "value": "Rule13.4: attribute in attrs object but not in attrNames with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.1: md in mdNames but not in md object", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.2: md in md object but not in mdNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.3: md in mdNames but not in md object with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.4: md in md object but not in mdNames with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule15", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule15.1: conflicting entity", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "type": "T", + "id": "Rule15", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule15.2: conflicting entity", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "servicePath": "/SS", + "id": "Rule15", + "type": "T" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule15.3: conflicting entity", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation", + "otherLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.1: more than one geo-attribute", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + }, + "otherLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.2", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.2: no geo-attribute but location field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.3", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.3: geo-attribute with null value and location field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": null, + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.4", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.4: geo-attribute but not location field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.5", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.5: location.attrName inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "ignoredLocation", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.6", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.6: geo:point coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 20, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.7", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.7: geo:line coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:line", + "creDate": 1705933908.9088006, + "modDate": 1705933908.9088006, + "mdNames": [] + } + }, + "creDate": 1705933908.9088006, + "modDate": 1705933908.9088006, + "location": { + "attrName": "location", + "coords": { + "type": "LineString", + "coordinates": [ + [ + 2, + 1 + ], + [ + 40, + 3 + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.8", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.8: geo:box coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:box", + "creDate": 1705933908.9101331, + "modDate": 1705933908.9101331, + "mdNames": [] + } + }, + "creDate": 1705933908.9101331, + "modDate": 1705933908.9101331, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 2, + 3 + ], + [ + 4, + 30 + ], + [ + 4, + 1 + ], + [ + 2, + 1 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.9", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.9: geo:polygon inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "10, 20", + "10, -20", + "1, 2" + ], + "type": "geo:polygon", + "creDate": 1705933908.9129946, + "modDate": 1705933908.9129946, + "mdNames": [] + } + }, + "creDate": 1705933908.9129946, + "modDate": 1705933908.9129946, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 20, + 10 + ], + [ + -20, + 10 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.10", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.10: geo:json coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + }, + "type": "geo:json", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "mdNames": [] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -200 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.11", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.11: geo:json coords strings", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + "1", + "2" + ], + [ + "10", + "20" + ], + [ + "10", + "-20" + ], + [ + "1", + "2" + ] + ] + ] + }, + "type": "geo:json", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "mdNames": [] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule17.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule17.1: missing lastCorrelator field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858 + }, + { + "_id": { + "id": "", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.1: id syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule20.2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.2: id syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule20.3(", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.3: id syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule20.4#", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.4: id syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.1", + "type": "", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.1: type syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.2", + "type": "Txxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.2: type syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.3", + "type": "T(", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.3: type syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.4", + "type": "T#", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.4: type syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.1", + "type": "T", + "servicePath": "SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.1: servicePath does not starts with slash", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.2", + "type": "T", + "servicePath": "/S1/S2/S3/S4/S5/S6/S7/S8/S9/S10/S11" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.2: more than 10 levels in servicePath", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.3", + "type": "T", + "servicePath": "/S1//S3/" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule22.3: servicePath level less than minimum", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.4", + "type": "T", + "servicePath": "/S1/Sxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/S3/" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule22.4: servicePath level greater than maximum", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.5", + "type": "T", + "servicePath": "/S1/S#/S3/" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule22.5: servicePath syntax problem", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.1: attr name syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.2: attr name syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1(", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.3: attr name syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1(": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1#", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.4: attr name syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1#": { + "value": 10, + "type": "Number#", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.1: attr type syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.2: attr type syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Numberxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.3: attr type syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number(", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.4: attr type syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number#", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.1: md name syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.2: md name syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.3: md name syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.4: md name syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1#": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1#", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.1: md type syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.2: md type syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Numberxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.3: md type syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number(", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.4: md type syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number#", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + } +]) \ No newline at end of file From 6c93a5f1cc237643321555b0617a50011505c24b Mon Sep 17 00:00:00 2001 From: mapedraza <40356341+mapedraza@users.noreply.github.com> Date: Tue, 23 Jan 2024 08:51:30 +0100 Subject: [PATCH 100/390] Update roadmap.md --- doc/roadmap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/roadmap.md b/doc/roadmap.md index 96fc7a5121..0926f30fc3 100644 --- a/doc/roadmap.md +++ b/doc/roadmap.md @@ -32,7 +32,6 @@ Disclaimer: The following list of features are planned to be addressed in the short term, and incorporated into the coming release(s) of the product: -- MQTT Retain flag ([#4388](https://github.com/telefonicaid/fiware-orion/issues/4388)) - MQTT notification retrial ([#4439](https://github.com/telefonicaid/fiware-orion/issues/4439)) - Allow multiple types in entity to support UNE 178503 requirements ([#3638](https://github.com/telefonicaid/fiware-orion/issues/3638)) - Pattern/filter batch updates ([#2389](https://github.com/telefonicaid/fiware-orion/issues/2389)) @@ -75,6 +74,7 @@ you wish to get involved in the implementation or influence the roadmap The following list contains all features that were in the roadmap and have already been implemented. +- MQTT Retain flag ([#4388](https://github.com/telefonicaid/fiware-orion/issues/4388)) ([3.11.0](https://github.com/telefonicaid/fiware-orion/releases/tag/3.11.0)) - Custom notifications: simplifying sending JSON requests ([#2560](https://github.com/telefonicaid/fiware-orion/issues/2560)) ([3.8.0](https://github.com/telefonicaid/fiware-orion/releases/tag/3.8.0)) - New subscripition modes (create only, update only, delete only and combinations) ([#1494](https://github.com/telefonicaid/fiware-orion/issues/1494)) ([3.7.0](https://github.com/telefonicaid/fiware-orion/releases/tag/3.7.0)) - Per sub/reg HTTP timeout ([#3842](https://github.com/telefonicaid/fiware-orion/issues/3842)) From 8dd783e23336e354467fce0aebe24ea77d057762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 23 Jan 2024 09:14:16 +0100 Subject: [PATCH 101/390] FIX some typos in orion-api.md --- doc/manuals/orion-api.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index bf07439d41..41d9909b76 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -542,8 +542,8 @@ In order to search for `Tree1` in that scope, the same Fiware-ServicePath will be used. Scopes are hierarchical and hierarchical search can be done. In order to -do that the `\#` special keyword is used. Thus, a query with -pattern entity id `.\*` of type `Tree` in `/Madrid/Gardens/ParqueNorte/#` +do that the `#` special keyword is used. Thus, a query with +pattern entity id `.*` of type `Tree` in `/Madrid/Gardens/ParqueNorte/#` will return all the trees in `ParqueNorte`, `Parterre1` and `Parterre2`. Finally, you can query for disjoint scopes, using a comma-separated list @@ -571,7 +571,7 @@ Some additional remarks: entities created without `Fiware-ServicePath` (or that don't include service path information in the database) belongs to a root scope `/` implicitly. All the queries without using `Fiware-ServicePath` - (including subscriptions) are on `\#` implicitly. This behavior + (including subscriptions) are on `/#` implicitly. This behavior ensures backward compatibility to pre-0.14.0 versions. - It is possible to have an entity with the same ID and type in From 9c98629c7bc3d27b24d3120d051e5780cc1dafdc Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 23 Jan 2024 20:08:07 +0900 Subject: [PATCH 102/390] (JP) FIX some typos in orion-api.md (#4488) --- doc/manuals.jp/orion-api.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index 67aa56efe9..9296c2644c 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -527,8 +527,8 @@ Orion は階層スコープをサポートしているため、エンティテ そのスコープで `Tree1` を検索するために、同じ Fiware-ServicePath が使用されます。 -スコープは階層的で、階層的な検索が可能です。これを行うために、`\#` 特別なキーワードが使用されます。 したがって、 -`/Madrid/Gardens/ParqueNorte/#` の `Tree` 型のエンティティ ID `.\*` のパターンを持つ query は、`ParqueNorte`, +スコープは階層的で、階層的な検索が可能です。これを行うために、`#` 特別なキーワードが使用されます。 したがって、 +`/Madrid/Gardens/ParqueNorte/#` の `Tree` 型のエンティティ ID `.*` のパターンを持つ query は、`ParqueNorte`, `Parterre1`, `Parterre2` のすべてのツリー (trees) を返します。 最後に、`Fiware-ServicePath` ヘッダでカンマ区切りのリストを使用して、ばらばらなスコープをクエリできます。たとえば、 @@ -551,7 +551,7 @@ Orion は階層スコープをサポートしているため、エンティテ - `Fiware-ServicePath` はオプションのヘッダです。`Fiware-ServicePath` なしで作成された (またはデータベースにサービス ・パス情報を含まない) すべてのエンティティは、暗黙的にルート・スコープ `/` に属していると想定されます。 - `Fiware-ServicePath` を使用しないすべてのクエリ (サブスクリプションを含む) は、暗黙的に `\#` にあります。 + `Fiware-ServicePath` を使用しないすべてのクエリ (サブスクリプションを含む) は、暗黙的に `/#` にあります。 この動作により、0.14.0より前のバージョンとの下位互換性が保証されます - 異なるスコープで同じID と型を持つエンティティを持つことが可能です。例えば、`/Madrid/Gardens/ParqueNorte/Parterre1` From 6d01e0615696d1163aecee152f23cf054bc2bcf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 23 Jan 2024 12:53:19 +0100 Subject: [PATCH 103/390] FIX first complete version of script and tests set --- scripts/entities_consistency/README.md | 352 +- .../entities_consistency.py | 1720 ++-- scripts/entities_consistency/requirements.txt | 4 +- .../test_entities_consistency.py | 110 +- .../entities_consistency/validation_data.js | 7714 +++++++++-------- 5 files changed, 5290 insertions(+), 4610 deletions(-) diff --git a/scripts/entities_consistency/README.md b/scripts/entities_consistency/README.md index 34cd10846f..aa3ddb578a 100644 --- a/scripts/entities_consistency/README.md +++ b/scripts/entities_consistency/README.md @@ -1,176 +1,176 @@ -The entities consistency script analyze the contents of the entities database in orion DBs and -check several consistency rules, reporting violations found. - -Ref: [entity document datamodel]([../..//doc/manuals/admin/database_model.md#entities-collection]) - -## Requirements - -This script is designed to work with Python 3. Install the dependencies in the `requirements.txt` file before using it. -Usage of virtual env is recommended. - -## Usage - -Run `entities_consistency.py -h` for arguments details. - -## Rules - -* Rules 1x: DB inconsistencies (severe problems) -* Rules 2x: Syntax restrictions -* Rules 9x: Usage of legacy features - -### Rule 10: `_id` field inconsistency - -Each entity in DB has a `_id` fields with three subfields: - -* `id` -* `type` -* `servicePath` - -## Rule 11: mandatory fields in entity - -The following field are mandatory: - -* `attrNames` -* `creDate` -* `modDate` - -It is not an exhaustive check of every attribute in the datamodel, but some entities created/updated with old Orion version may be missing them. - -## Rule 12: mandatory fields in attribute - -The following subfield are mandatory for every attribute: - -* `mdNames` -* `creDate` -* `modDate` - -It is not an exhaustive check of every attribute in the datamodel, but some entities created/updated with old Orion version may be missing them. - -### Rule 13: `attrNames` field inconsistency - -Each item in `attrNames` array has a corresponding key in `attrs` object and the other way around. - -### Rule 14: `mdNames` field inconsistency - -For every attribute, for each item in `mdNames` array has a corresponding key in `md` object and the other way around. - -### Rule 15: swapped subkeys in `_id` - -In MongoDB JSON objects are stored taking order into account, so DB allows to have a document with -`_id` equal to `{"id": "E", "type": "T", "servicePath": "/"}` and at the same time have another document with `_id` -equal to `{"type": "T", "id": "E", "servicePath": "/"}` without violating `_id` uniqueness constraint. - -This rule checks that this is not happening. - -### Rule 16: `location` field inconsistency - -Check that location in consistent. In particular: - -* As much as one attribute with `geo:point`, `geo:line`, `geo:box`, `geo:polygon` or `geo:json` type -* If an attribute is found with one of above types, check the `location` field is consistent -* If no attribute is found with one of above types, check no `location` field is not found - * If the location of the entity is defined using deprecated `location` metadata it will be detected as this case. Additional rules in the 9x group can help to diagnose this situation. - -This rule is for location inconsistencies. For usage of deprecated geo types, there are additional rules in the 9x group. - -### Rule 17: missing `lastCorrelator` - -Check if `lastCorrelator` is included. - -This field was introduced in Orion 1.8.0 (September 11th, 2017). - -### Rule 20: entity id syntax - -Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. - -### Rule 21: entity type syntax - -Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. - -### Rule 22: entity servicePath syntax - -https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path - -### Rule 23: attribute name syntax - -Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. - -### Rule 24: attribute type syntax - -Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. - -### Rule 25: metadata name syntax - -Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. - -### Rule 26: metadata type syntax - -Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. - -### Rule 90: usage of `geo:x` attribute type where `x` different from `json` - -Check usage of deprecated geo-location types, i.e: - -* `geo:point` -* `geo:line` -* `geo:box` -* `geo:polygon` - -Suggested action is to: - -* Change attribute type to `geo:json` -* Set the attribute value to the same GeoJSON in `location.coords` field - -Note this rule doesn't check location consistency for this case (e.g. more than one geo-location attribute in the same entity). That's done by another rule in the 1x group). - -### Rule 91: usage of more than one legacy `location` metadata - -Check usage of `location` in more than one attribute of the same entity. - -Note this rule doesn't check location consistency for this case (that's done by another rule in the 1x group). - -### Rule 92: legacy `location` metadata should be `WGS84` or `WSG84` - -The value of the `location` metadata should be `WGS84` or `WSG84`. - -Additional consideration: - -* Entities attributes may have `location` metadata with values different from `WGS84` or `WSG84` if created using NGSIv2. That would be a false positive in this rule validation -* This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). - -### Rule 93: usage of redundant legacy `location` - -Checks usage of redundant `location` metadata, i.e. when at the same time a `geo:` type is used in the -same attribute. - -Suggested action is to remove the `location` metadata. - -Additional, considerations: - - * This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. - * This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). - -### Rule 94: usage of not redundant legacy `location` - -Checks usage of not redundant `location` metadata, i.e. when at the same time the type of the attribute is nog `geo:`. -same attribute. - -Suggested action is to: - -* Change attribute type to `geo:json` -* Set the attribute value to the same GeoJSON in `location.coords` field -* Remove the `location` metadata from the attribute - -Additional, considerations: - -* This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. -* This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). - -## Testing - -You can test the `entities_consistency.py` script this qy: - -1. Populate `orion-validation` DB with testing document. To do so, copy-paste the content of the `validation_data.js` in `mongosh` -2. Run `test_entities_consistency.py` test and check the log output - -You can also run `test_entities_consistenct.py` under coverage to check every rule is covering all the possible validation cases. +The entities consistency script analyze the contents of the entities database in orion DBs and +check several consistency rules, reporting violations found. + +Ref: [entity document datamodel]([../..//doc/manuals/admin/database_model.md#entities-collection]) + +## Requirements + +This script is designed to work with Python 3. Install the dependencies in the `requirements.txt` file before using it. +Usage of virtual env is recommended. + +## Usage + +Run `entities_consistency.py -h` for arguments details. + +## Rules + +* Rules 1x: DB inconsistencies (severe problems) +* Rules 2x: Syntax restrictions +* Rules 9x: Usage of legacy features + +### Rule 10: `_id` field inconsistency + +Each entity in DB has a `_id` fields with three subfields: + +* `id` +* `type` +* `servicePath` + +## Rule 11: mandatory fields in entity + +The following field are mandatory: + +* `attrNames` +* `creDate` +* `modDate` + +It is not an exhaustive check of every attribute in the datamodel, but some entities created/updated with old Orion version may be missing them. + +## Rule 12: mandatory fields in attribute + +The following subfield are mandatory for every attribute: + +* `mdNames` +* `creDate` +* `modDate` + +It is not an exhaustive check of every attribute in the datamodel, but some entities created/updated with old Orion version may be missing them. + +### Rule 13: `attrNames` field inconsistency + +Each item in `attrNames` array has a corresponding key in `attrs` object and the other way around. + +### Rule 14: `mdNames` field inconsistency + +For every attribute, for each item in `mdNames` array has a corresponding key in `md` object and the other way around. + +### Rule 15: swapped subkeys in `_id` + +In MongoDB JSON objects are stored taking order into account, so DB allows to have a document with +`_id` equal to `{"id": "E", "type": "T", "servicePath": "/"}` and at the same time have another document with `_id` +equal to `{"type": "T", "id": "E", "servicePath": "/"}` without violating `_id` uniqueness constraint. + +This rule checks that this is not happening. + +### Rule 16: `location` field inconsistency + +Check that location in consistent. In particular: + +* As much as one attribute with `geo:point`, `geo:line`, `geo:box`, `geo:polygon` or `geo:json` type +* If an attribute is found with one of above types, check the `location` field is consistent +* If no attribute is found with one of above types, check no `location` field is not found + * If the location of the entity is defined using deprecated `location` metadata it will be detected as this case. Additional rules in the 9x group can help to diagnose this situation. + +This rule is for location inconsistencies. For usage of deprecated geo types, there are additional rules in the 9x group. + +### Rule 17: missing `lastCorrelator` + +Check if `lastCorrelator` is included. + +This field was introduced in Orion 1.8.0 (September 11th, 2017). + +### Rule 20: entity id syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 21: entity type syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 22: entity servicePath syntax + +https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path + +### Rule 23: attribute name syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 24: attribute type syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 25: metadata name syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 26: metadata type syntax + +Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. + +### Rule 90: usage of `geo:x` attribute type where `x` different from `json` + +Check usage of deprecated geo-location types, i.e: + +* `geo:point` +* `geo:line` +* `geo:box` +* `geo:polygon` + +Suggested action is to: + +* Change attribute type to `geo:json` +* Set the attribute value to the same GeoJSON in `location.coords` field + +Note this rule doesn't check location consistency for this case (e.g. more than one geo-location attribute in the same entity). That's done by another rule in the 1x group). + +### Rule 91: usage of more than one legacy `location` metadata + +Check usage of `location` in more than one attribute of the same entity. + +Note this rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +### Rule 92: legacy `location` metadata should be `WGS84` or `WSG84` + +The value of the `location` metadata should be `WGS84` or `WSG84`. + +Additional consideration: + +* Entities attributes may have `location` metadata with values different from `WGS84` or `WSG84` if created using NGSIv2. That would be a false positive in this rule validation +* This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +### Rule 93: usage of redundant legacy `location` + +Checks usage of redundant `location` metadata, i.e. when at the same time a `geo:` type is used in the +same attribute. + +Suggested action is to remove the `location` metadata. + +Additional, considerations: + + * This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. + * This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +### Rule 94: usage of not redundant legacy `location` + +Checks usage of not redundant `location` metadata, i.e. when at the same time the type of the attribute is nog `geo:`. +same attribute. + +Suggested action is to: + +* Change attribute type to `geo:json` +* Set the attribute value to the same GeoJSON in `location.coords` field +* Remove the `location` metadata from the attribute + +Additional, considerations: + +* This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. +* This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). + +## Testing + +You can test the `entities_consistency.py` script this qy: + +1. Populate `orion-validation` DB with testing document. To do so, copy-paste the content of the `validation_data.js` in `mongosh` +2. Run `test_entities_consistency.py` test and check the log output + +You can also run `test_entities_consistenct.py` under coverage to check every rule is covering all the possible validation cases. diff --git a/scripts/entities_consistency/entities_consistency.py b/scripts/entities_consistency/entities_consistency.py index 8caeb1f3a0..5bf2786dfa 100644 --- a/scripts/entities_consistency/entities_consistency.py +++ b/scripts/entities_consistency/entities_consistency.py @@ -1,848 +1,872 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -__author__ = 'fermin' - -from pymongo import MongoClient -from deepdiff import DeepDiff -import argparse -import logging -import json -import sys -import re - - -# Helper functions - -def is_geo_type(attr_type): - """ - Return True if attr type passed as argument is a geo type - - :param attr_type: the attr type to evaluate - """ - - return attr_type == 'geo:point' or attr_type == 'geo:line' or attr_type == 'geo:box' \ - or attr_type == 'geo:polygon' or attr_type == 'geo:json' - - -def ignore_type(attr): - """ - Return true if attribute has the ignoreType metadata - """ - return 'md' in attr and 'ignoreType' in attr['md'] - - -def to_geo_json(attr): - """ - Return the GeoJSON corresponding to an attribute location, taking into account the type - - Useful ref: https://github.com/telefonicaid/fiware-orion/blob/3.9.0/doc/manuals/orion-api.md - """ - - if attr['type'] == 'geo:point': - # "value": "41.3763726, 2.186447514", - coords = attr['value'].split(",") - return { - 'type': 'Point', - 'coordinates': [float(coords[1]), float(coords[0])] - } - elif attr['type'] == 'geo:line': - # "value": [ - # "40.63913831188419, -8.653321266174316", - # "40.63881265804603, -8.653149604797363" - # ] - coordinates = [] - for item in attr['value']: - coords = item.split(",") - coordinates.append([float(coords[1]), float(coords[0])]) - return { - 'type': 'LineString', - 'coordinates': coordinates - } - elif attr['type'] == 'geo:box': - # "value": [ - # "40.63913831188419, -8.653321266174316", - # "40.63881265804603, -8.653149604797363" - # ] - # The first pair is the lower corner, the second is the upper corner. - lower_corner_0 = attr['value'][0].split(",") - upper_corner_0 = attr['value'][1].split(",") - lower_corner_1 = [upper_corner_0[0], lower_corner_0[1]] - upper_corner_1 = [lower_corner_0[0], upper_corner_0[1]] - return { - 'type': 'Polygon', - 'coordinates': [[ - [float(lower_corner_0[1]), float(lower_corner_0[0])], - [float(lower_corner_1[1]), float(lower_corner_1[0])], - [float(upper_corner_0[1]), float(upper_corner_0[0])], - [float(upper_corner_1[1]), float(upper_corner_1[0])], - [float(lower_corner_0[1]), float(lower_corner_0[0])] - ]] - } - elif attr['type'] == 'geo:polygon': - # "value": [ - # "40.63913831188419, -8.653321266174316", - # "40.63881265804603, -8.653149604797363", - # "40.63913831188419, -8.653321266174316" - # ] - coordinates = [] - for item in attr['value']: - coords = item.split(",") - last = [float(coords[1]), float(coords[0])] - coordinates.append([float(coords[1]), float(coords[0])]) - coords.append(last) # so we have a closed shape - return { - 'type': 'Polygon', - 'coordinates': [coordinates] - } - elif attr['type'] == 'geo:json': - return attr['value'] - else: - logger.error(f"unknown geo location type: {attr['type']}") - return None - - -def convert_strings_to_numbers(data): - """ - Generated by ChatGPT :) - """ - if isinstance(data, dict): - # If it's a dictionary, apply the function recursively to its values - return {key: convert_strings_to_numbers(value) for key, value in data.items()} - elif isinstance(data, list): - # If it's a list, apply the function recursively to its elements - return [convert_strings_to_numbers(item) for item in data] - elif isinstance(data, str): - # If it's a string, try converting it to a number - try: - return int(data) - except ValueError: - try: - return float(data) - except ValueError: - # If it's not a valid number, leave it as it is - return data - else: - # If it's neither a dictionary, list, nor string, leave it as it is - return data - - -def check_id(id): - """ - Common checks for several rules - - Ref: https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#identifiers-syntax-restrictions - """ - # Minimum field length is 1 character - l = len(id) - if l == 0: - return f'length ({l}) shorter than minimum allowed (1)' - # Maximum field length is 256 characters - if l > 256: - return f'length ({l}) greater than maximum allowed (256)' - - # Following chars are not used (general): < > " ' = ; ( ) - if re.search('[<>"\'=;()]', id): - return f'contains forbidden chars (general)' - - # Following chars are not used (identifiers): whitespace, & ? / # - if re.search('[\s&?/#]', id): - return f'contains forbidden chars (identifiers)' - - return None - - -# Rules functions -def rule10(entity): - """ - Rule 10: `_id` field inconsistency - - See README.md for an explanation of the rule - """ - missing_fields = [] - - for field in ['id', 'type', 'servicePath']: - if field not in entity['_id']: - missing_fields.append(field) - - if len(missing_fields) > 0: - return f"missing subfields in _id: {', '.join(missing_fields)}" - else: - return None - - -def rule11(entity): - """ - Rule 11: mandatory fields in entity - - See README.md for an explanation of the rule - """ - missing_fields = [] - - for field in ['attrNames', 'creDate', 'modDate']: - if field not in entity: - missing_fields.append(field) - - if len(missing_fields) > 0: - return f"missing fields: {', '.join(missing_fields)}" - else: - return None - - -def rule12(entity): - """ - Rule 12: mandatory fields in attribute - - See README.md for an explanation of the rule - """ - s = [] - for attr in entity['attrs'].keys(): - missing_fields = [] - - for field in ['mdNames', 'creDate', 'modDate']: - if field not in entity['attrs'][attr]: - missing_fields.append(field) - - if len(missing_fields) > 0: - s.append(f"in attribute '{attr}' missing fields: {', '.join(missing_fields)}") - - if len(s) > 0: - return ', '.join(s) - else: - return None - - -def rule13(entity): - """ - Rule 13: `attrNames` field inconsistency - - See README.md for an explanation of the rule - """ - # note the omission of attrNames is checked by another rule. In this rule we include - # some guards for not breaking in that case - - # attrNames in attrs - attrnames_not_in_attrs = [] - if 'attrNames' in entity: - for attr in entity['attrNames']: - if attr.replace('.', '=') not in entity['attrs']: - attrnames_not_in_attrs.append(attr) - - # attrs in attrNames - attrs_not_in_attrnames = [] - if 'attrs' in entity: - for attr in entity['attrs'].keys(): - if 'attrNames' not in entity or attr.replace('=', '.') not in entity['attrNames']: - attrs_not_in_attrnames.append(attr) - - s = [] - if len(attrnames_not_in_attrs) > 0: - s.append(f"attributes in attrNames not found in attrs object: {','.join(attrnames_not_in_attrs)}") - if len(attrs_not_in_attrnames) > 0: - s.append(f"attributes in attrs object not found in attrNames: {','.join(attrs_not_in_attrnames)}") - - if len(s) > 0: - return ', '.join(s) - else: - return None - - -def rule14(entity): - """ - Rule 14: `mdNames` field inconsistency - - See README.md for an explanation of the rule - """ - # note the omission of mdNames is checked by another rule. In this rule we include - # some guards for not breaking in that case - - s = [] - for item in entity['attrs'].keys(): - attr = entity['attrs'][item] - # mdNames in md - if 'mdNames' in attr: - mdnames_not_in_md = [] - for md in attr['mdNames']: - if md.replace('.', '=') not in attr['md']: - mdnames_not_in_md.append(md) - - # md in mdNames - md_not_in_mdnames = [] - if 'md' in attr: - for md in attr['md'].keys(): - if 'mdNames' not in attr or md.replace('=', '.') not in attr['mdNames']: - md_not_in_mdnames.append(md) - - if len(mdnames_not_in_md) > 0: - s.append( - f"in attribute '{item}' metadata in mdNames not found in md object: {', '.join(mdnames_not_in_md)}") - if len(md_not_in_mdnames) > 0: - s.append( - f"in attribute '{item}' metadata in md object not found in mdNames: {', '.join(md_not_in_mdnames)}") - - if len(s) > 0: - return ', '.join(s) - else: - return None - - -def rule15(entities_collection): - """ - Rule 15: swapped subkeys in `_id` - - See README.md for an explanation of the rule - - This is a global rule, so the parameter is not an individual entity but the whole entities collection. - """ - s = [] - for entity in entities_collection.aggregate( - [ - { - '$group': { - '_id': {'id': '$_id.id', 'type': '$_id.type', 'servicePath': '$_id.servicePath'}, - 'count': {'$sum': 1} - } - }, - { - '$match': {'count': {'$gt': 1}} - } - ] - ): - id = entity['_id']['id'] - type = entity['_id']['type'] - service_path = entity['_id']['servicePath'] - count = entity['count'] - s.append( - f"_id uniqueness violation for entity id='{id}' type='{type}' servicePath='{service_path}' found {count} times") - - if len(s) > 0: - return ', '.join(s) - else: - return None - - -def rule16(entity): - """ - Rule 16: `location` field inconsistency - - See README.md for an explanation of the rule - """ - # check that as much as one attribute is using geo type - geo_attrs = [] - for attr in entity['attrs']: - if is_geo_type(entity['attrs'][attr]['type']) and not ignore_type(entity['attrs'][attr]): - geo_attrs.append(attr) - - if len(geo_attrs) > 1: - return f"more than one attribute with geo type: {', '.join(geo_attrs)}" - - if len(geo_attrs) == 1: - # If geo attr found, then check that there is consistent location field - geo_attr = geo_attrs[0] - geo_type = entity['attrs'][geo_attr]['type'] - if entity['attrs'][geo_attr]['value'] is None: - # if null value in geo location attribute, then location field must not be present - if 'location' in entity: - return f"geo location '{geo_attr}' ({geo_type}) with null value and location field found" - else: - # not null value in geo location attribute case - if 'location' not in entity: - return f"geo location detected in '{geo_attr}' ({geo_type}) but location field not found in entity" - if entity['location']['attrName'] != geo_attr: - return f"location.attrName ({entity['location']['attrName']}) differs from '{geo_attr}'" - - geo_json = to_geo_json(entity['attrs'][geo_attr]) - - # https://www.testcult.com/deep-comparison-of-json-in-python/ - diff = DeepDiff(geo_json, entity['location']['coords'], ignore_order=True) - if diff: - # A typical difference is that attribute value uses strings and location uses numbers - # (this happens when the location was created/updated using NGSIv1). We try to identify that case - geo_json = convert_strings_to_numbers(geo_json) - if not DeepDiff(geo_json, entity['location']['coords'], ignore_order=True): - return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) is consistent, but value " \ - f"should use numbers for coordinates instead of strings" - else: - # Other causes - return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) value: {diff}" - else: # len(geo_attrs) == 0 - # If no geo attr found, check there isn't a location field - if 'location' in entity: - return f"location field detected but no geo attribute is present (or metadata location is used)" - - -def rule17(entity): - """ - Rule 17: missing `lastCorrelator` - - See README.md for an explanation of the rule - """ - if 'lastCorrelator' not in entity: - return f"missing lastCorrelator" - else: - return None - - -def rule20(entity): - """ - Rule 20: entity id syntax - - See README.md for an explanation of the rule - """ - - # The existence of id in _id is checked by another rule - if 'id' in entity['_id']: - r = check_id(entity['_id']['id']) - if r is not None: - return f"entity id ({entity['_id']['id']}) syntax violation: {r}" - - return None - - -def rule21(entity): - """ - Rule 21: entity type syntax - - See README.md for an explanation of the rule - """ - - # The existence of type in _id is checked by another rule - if 'type' in entity['_id']: - r = check_id(entity['_id']['type']) - if r is not None: - return f"entity type ({entity['_id']['type']}) syntax violation: {r}" - - return None - - -def rule22(entity): - """ - Rule 22: entity servicePath syntax - - See README.md for an explanation of the rule - - Ref: https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path - """ - - # servicePath existence is checked by another rule - if 'servicePath' not in entity['_id']: - return None - - sp = entity['_id']['servicePath'] - # Scope must start with / (only "absolute" scopes are allowed) - if not sp.startswith('/'): - return f"servicePath '{sp}' does not starts with '/'" - - # 10 maximum scope levels in a path - sp_levels = sp.split('/') - if len(sp_levels) > 10: - return f"servicePath has {len(sp_levels)} tokens but the limit is 10" - - # 50 maximum characters in each level (1 char is minimum), only alphanumeric and underscore allowed - for level in sp_levels: - if len(level) == 0: - return f'servicePath length is 0 but minimum is 1' - if len(level) > 50: - return f'servicePath length is {len(level)} but maximum is 50' - if re.search('[^a-zA-Z0-9_]', level): - return f"unallowed characters in '{level}' in servicePath level" - - -def rule23(entity): - """ - Rule 23: attribute name syntax - - See README.md for an explanation of the rule - """ - s = [] - for attr in entity['attrs']: - r = check_id(attr) - if r is not None: - s.append(f"attribute name ({attr}) syntax violation: {r}") - - if len(s) > 0: - return ', '.join(s) - else: - return None - - -def rule24(entity): - """ - Rule 24: attribute type syntax - - See README.md for an explanation of the rule - """ - s = [] - for attr in entity['attrs']: - if 'type' not in entity['attrs'][attr]: - s.append(f"in attribute '{attr}' type is missing") - else: - type = entity['attrs'][attr]['type'] - r = check_id(type) - if r is not None: - s.append(f"in attribute '{attr}' type ({type}) syntax violation: {r}") - - if len(s) > 0: - return ', '.join(s) - else: - return None - - -def rule25(entity): - """ - Rule 25: metadata name syntax - - See README.md for an explanation of the rule - """ - s = [] - for attr in entity['attrs']: - if 'md' in entity['attrs'][attr]: - for md in entity['attrs'][attr]['md']: - r = check_id(md) - if r is not None: - s.append(f"in attribute '{attr}' metadata name ({md}) syntax violation: {r}") - - if len(s) > 0: - return ', '.join(s) - else: - return None - - -def rule26(entity): - """ - Rule 26: metadata type syntax - - See README.md for an explanation of the rule - """ - s = [] - for attr in entity['attrs']: - if 'md' in entity['attrs'][attr]: - for md in entity['attrs'][attr]['md']: - if 'type' not in entity['attrs'][attr]['md'][md]: - s.append(f"in attribute '{attr}' metadata '{md}' type is missing") - else: - type = entity['attrs'][attr]['md'][md]['type'] - r = check_id(type) - if r is not None: - s.append(f"in attribute '{attr}' metadata '{md}' type ({type}) syntax violation: {r}") - - if len(s) > 0: - return ', '.join(s) - else: - return None - - -def rule90(entity): - """ - Rule 90: usage of `geo:x` attribute type where `x` different from `json` - - See README.md for an explanation of the rule - """ - for attr in entity['attrs'].keys(): - type = entity['attrs'][attr]['type'] - if is_geo_type(type) and type != 'geo:json': - return f"in attribute '{attr}' usage of deprecated {type} found" - - return None - - -def rule91(entity): - """ - Rule 91: usage of more than one legacy `location` metadata - - See README.md for an explanation of the rule - """ - n = 0 - for attr in entity['attrs'].keys(): - if 'md' in entity['attrs'][attr]: - if 'location' in entity['attrs'][attr]['md'].keys(): - n += 1 - - if n > 1: - return f'location metadata found {n} times among entity attributes (maximum should be just 1)' - else: - return None - - -def rule92(entity): - """ - Rule 92: legacy `location` metadata should be `WGS84` or `WSG84` - - See README.md for an explanation of the rule - """ - s = [] - for attr in entity['attrs'].keys(): - if 'md' in entity['attrs'][attr]: - if 'location' in entity['attrs'][attr]['md'].keys(): - location_value = entity['attrs'][attr]['md']['location']['value'] - if location_value != 'WGS84' and location_value != 'WSG84': - s.append(f"in attribute '{attr}' location metadata value is {location_value} (should be WGS84 or WSG84)") - - if len(s) > 1: - return ', '.join(s) - else: - return None - - -def rule93(entity): - """ - Rule 93: usage of redundant legacy `location` - - See README.md for an explanation of the rule - """ - for attr in entity['attrs'].keys(): - if 'md' in entity['attrs'][attr]: - for md in entity['attrs'][attr]['md']: - if md == 'location': - if is_geo_type(entity['attrs'][attr]['type']): - return f"in attribute '{attr}' redundant location metadata found (attribute is " \ - f"already using {entity['attrs'][attr]['type']} type)" - - return None - - -def rule94(entity): - """ - Rule 94: usage of not redundant legacy `location` - - See README.md for an explanation of the rule - """ - for attr in entity['attrs'].keys(): - if 'md' in entity['attrs'][attr]: - for md in entity['attrs'][attr]['md']: - if md == 'location': - if not is_geo_type(entity['attrs'][attr]['type']): - return f"in attribute '{attr}' location metadata found (attribute type " \ - f"is {entity['attrs'][attr]['type']})" - - return None - - -rules = [ - # Rules 1x - { - 'label': 'Rule10', - 'global': False, - 'func': rule10 - }, - { - 'label': 'Rule11', - 'global': False, - 'func': rule11 - }, - { - 'label': 'Rule12', - 'global': False, - 'func': rule12 - }, - { - 'label': 'Rule13', - 'global': False, - 'func': rule13 - }, - { - 'label': 'Rule14', - 'global': False, - 'func': rule14 - }, - { - 'label': 'Rule15', - 'global': True, - 'func': rule15 - }, - { - 'label': 'Rule16', - 'global': False, - 'func': rule16 - }, - { - 'label': 'Rule17', - 'global': False, - 'func': rule17 - }, - # Rules 2x - { - 'label': 'Rule20', - 'global': False, - 'func': rule20 - }, - { - 'label': 'Rule21', - 'global': False, - 'func': rule21 - }, - { - 'label': 'Rule22', - 'global': False, - 'func': rule22 - }, - { - 'label': 'Rule23', - 'global': False, - 'func': rule23 - }, - { - 'label': 'Rule24', - 'global': False, - 'func': rule24 - }, - { - 'label': 'Rule25', - 'global': False, - 'func': rule25 - }, - { - 'label': 'Rule26', - 'global': False, - 'func': rule26 - }, - # Rules 9x - { - 'label': 'Rule90', - 'global': False, - 'func': rule90 - }, - { - 'label': 'Rule91', - 'global': False, - 'func': rule91 - }, - { - 'label': 'Rule92', - 'global': False, - 'func': rule92 - }, - { - 'label': 'Rule93', - 'global': False, - 'func': rule93 - }, - { - 'label': 'Rule94', - 'global': False, - 'func': rule94 - } -] - - -def process_db(logger, db_name, db_conn, query, rules_exp): - """ - Process an individual DB - - :param logger: logger object - :param db_name: the name of the DB to process - :param db_conn: connection to MongoDB - :param query: query to filter entities to be processed - :param rules_exp: regular expression to filter rules to apply - :return: fails - """ - - logger.info(f'Processing {db_name}') - n = 0 - fails = 0 - - # check collection existence - if 'entities' not in db_conn[db_name].list_collection_names(): - logger.warning(f'collections entities not found in {db_name} database, nothing to do') - return - - # apply global rules - for rule in rules: - if rules_exp is not None and not re.search(rules_exp, rule['label']): - continue - - if rule['global']: - s = rule['func'](db_conn[db_name]['entities']) - if s is not None: - logger.warning(f'DB {db_name} {rule["label"]} violation in entities collection: {s}') - fails += 1 - - # apply per-entity rules - for entity in db_conn[db_name]['entities'].find(query): - n += 1 - id_string = json.dumps(entity['_id']) - logger.debug(f'* processing entity {id_string}') - for rule in rules: - if rules_exp is not None and not re.search(rules_exp, rule['label']): - continue - - if not rule['global']: - s = rule['func'](entity) - if s is not None: - logger.warning(f'DB {db_name} {rule["label"]} violation for entity {id_string}: {s}') - fails += 1 - - logger.info(f'processed {n} entities ({fails} rule violations)') - - return fails - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - prog='entities_consistency', - description='Check consistency in Orion entities collection in DB') - - parser.add_argument('--mongoUri', dest='mongo_uri', default='mongodb://localhost:27017', - help='MongoDB URI. Default is mongodb://localhost:27017') - parser.add_argument('--db', dest='db', - help='DB name to check. If omitted all DBs starting with "orion" will be checked.') - parser.add_argument('--query', dest='query', default='{}', - help='query to filter entities to check, in JSON MongoDB query language. By default, ' - 'all entities in the collection will be checked.') - parser.add_argument('--rulesExp', dest='rules_exp', - help='Specifies the rules to apply, as a regular expression. By default all rules are applied.') - parser.add_argument('--logLevel', dest='log_level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'], default='INFO', - help='log level. Default is INFO') - args = parser.parse_args() - - # sets the logging configuration - logging.basicConfig( - level=logging.getLevelName(args.log_level), - format="time=%(asctime)s | lvl=%(levelname)s | msg=%(message)s", - handlers=[ - logging.StreamHandler() - ] - ) - logger = logging.getLogger() - - # connect to MongoDB - mongo_client = MongoClient(args.mongo_uri) - db_names = mongo_client.list_database_names() - - # to remove starting and trailing ' char, in case it is used - query = json.loads(args.query.replace("'", "")) - - fails = 0 - if args.db is not None: - if args.db in db_names: - fails += process_db(logger, args.db, mongo_client[args.db], query, args.rules_exp) - else: - logger.fatal(f'database {args.db} does not exist') - sys.exit(1) - else: - # Process all Orion databases - for db_name in db_names: - if db_name.startswith('orion-'): - fails += process_db(logger, db_name, mongo_client[db_name], query, args.rules_exp) - - logger.info(f'total rule violations: {fails}') +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +__author__ = 'fermin' + +from pymongo import MongoClient +from deepdiff import DeepDiff +import argparse +import logging +import json +import sys +import re + + +# Helper functions + +def is_geo_type(attr_type): + """ + Return True if attr type passed as argument is a geo type + + :param attr_type: the attr type to evaluate + """ + + return attr_type == 'geo:point' or attr_type == 'geo:line' or attr_type == 'geo:box' \ + or attr_type == 'geo:polygon' or attr_type == 'geo:json' + + +def ignore_type(attr): + """ + Return true if attribute has the ignoreType metadata + """ + return 'md' in attr and 'ignoreType' in attr['md'] + + +def to_geo_json(attr): + """ + Return the GeoJSON corresponding to an attribute location, taking into account the type + + Useful ref: https://github.com/telefonicaid/fiware-orion/blob/3.9.0/doc/manuals/orion-api.md + """ + + if attr['type'] == 'geo:point': + # "value": "41.3763726, 2.186447514", + coords = attr['value'].split(",") + return { + 'type': 'Point', + 'coordinates': [float(coords[1]), float(coords[0])] + } + elif attr['type'] == 'geo:line': + # "value": [ + # "40.63913831188419, -8.653321266174316", + # "40.63881265804603, -8.653149604797363" + # ] + coordinates = [] + for item in attr['value']: + coords = item.split(",") + coordinates.append([float(coords[1]), float(coords[0])]) + return { + 'type': 'LineString', + 'coordinates': coordinates + } + elif attr['type'] == 'geo:box': + # "value": [ + # "40.63913831188419, -8.653321266174316", + # "40.63881265804603, -8.653149604797363" + # ] + # The first pair is the lower corner, the second is the upper corner. + lower_corner_0 = attr['value'][0].split(",") + upper_corner_0 = attr['value'][1].split(",") + lower_corner_1 = [upper_corner_0[0], lower_corner_0[1]] + upper_corner_1 = [lower_corner_0[0], upper_corner_0[1]] + return { + 'type': 'Polygon', + 'coordinates': [[ + [float(lower_corner_0[1]), float(lower_corner_0[0])], + [float(lower_corner_1[1]), float(lower_corner_1[0])], + [float(upper_corner_0[1]), float(upper_corner_0[0])], + [float(upper_corner_1[1]), float(upper_corner_1[0])], + [float(lower_corner_0[1]), float(lower_corner_0[0])] + ]] + } + elif attr['type'] == 'geo:polygon': + # "value": [ + # "40.63913831188419, -8.653321266174316", + # "40.63881265804603, -8.653149604797363", + # "40.63913831188419, -8.653321266174316" + # ] + coordinates = [] + for item in attr['value']: + coords = item.split(",") + last = [float(coords[1]), float(coords[0])] + coordinates.append([float(coords[1]), float(coords[0])]) + coords.append(last) # so we have a closed shape + return { + 'type': 'Polygon', + 'coordinates': [coordinates] + } + elif attr['type'] == 'geo:json': + return attr['value'] + else: + logger.error(f"unknown geo location type: {attr['type']}") + return None + + +def convert_strings_to_numbers(data): + """ + Generated by ChatGPT :) + """ + if isinstance(data, dict): + # If it's a dictionary, apply the function recursively to its values + return {key: convert_strings_to_numbers(value) for key, value in data.items()} + elif isinstance(data, list): + # If it's a list, apply the function recursively to its elements + return [convert_strings_to_numbers(item) for item in data] + elif isinstance(data, str): + # If it's a string, try converting it to a number + try: + return int(data) + except ValueError: + try: + return float(data) + except ValueError: + # If it's not a valid number, leave it as it is + return data + else: + # If it's neither a dictionary, list, nor string, leave it as it is + return data + + +def check_id(id): + """ + Common checks for several rules + + Ref: https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#identifiers-syntax-restrictions + """ + # Minimum field length is 1 character + l = len(id) + if l == 0: + return f'length ({l}) shorter than minimum allowed (1)' + # Maximum field length is 256 characters + if l > 256: + return f'length ({l}) greater than maximum allowed (256)' + + # Following chars are not used (general): < > " ' = ; ( ) + if re.search('[<>"\'=;()]', id): + return f'contains forbidden chars (general)' + + # Following chars are not used (identifiers): whitespace, & ? / # + if re.search('[\s&?/#]', id): + return f'contains forbidden chars (identifiers)' + + return None + + +# Rules functions +def rule10(entity): + """ + Rule 10: `_id` field inconsistency + + See README.md for an explanation of the rule + """ + missing_fields = [] + + for field in ['id', 'type', 'servicePath']: + if field not in entity['_id']: + missing_fields.append(field) + + if len(missing_fields) > 0: + return f"missing subfields in _id: {', '.join(missing_fields)}" + else: + return None + + +def rule11(entity): + """ + Rule 11: mandatory fields in entity + + See README.md for an explanation of the rule + """ + missing_fields = [] + + for field in ['attrNames', 'creDate', 'modDate']: + if field not in entity: + missing_fields.append(field) + + if len(missing_fields) > 0: + return f"missing fields: {', '.join(missing_fields)}" + else: + return None + + +def rule12(entity): + """ + Rule 12: mandatory fields in attribute + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + missing_fields = [] + + for field in ['mdNames', 'creDate', 'modDate']: + if field not in entity['attrs'][attr]: + missing_fields.append(field) + + if len(missing_fields) > 0: + s.append(f"in attribute '{attr}' missing fields: {', '.join(missing_fields)}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule13(entity): + """ + Rule 13: `attrNames` field inconsistency + + See README.md for an explanation of the rule + """ + # note the omission of attrNames is checked by another rule. In this rule we include + # some guards for not breaking in that case + + # attrNames in attrs + attrnames_not_in_attrs = [] + if 'attrNames' in entity: + for attr in entity['attrNames']: + if attr.replace('.', '=') not in entity['attrs']: + attrnames_not_in_attrs.append(attr) + + # attrs in attrNames + attrs_not_in_attrnames = [] + if 'attrs' in entity: + for attr in entity['attrs']: + if 'attrNames' not in entity or attr.replace('=', '.') not in entity['attrNames']: + attrs_not_in_attrnames.append(attr) + + s = [] + if len(attrnames_not_in_attrs) > 0: + s.append(f"attributes in attrNames not found in attrs object: {','.join(attrnames_not_in_attrs)}") + if len(attrs_not_in_attrnames) > 0: + s.append(f"attributes in attrs object not found in attrNames: {','.join(attrs_not_in_attrnames)}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule14(entity): + """ + Rule 14: `mdNames` field inconsistency + + See README.md for an explanation of the rule + """ + # note the omission of mdNames is checked by another rule. In this rule we include + # some guards for not breaking in that case + + s = [] + for item in entity['attrs']: + attr = entity['attrs'][item] + # mdNames in md + if 'mdNames' in attr: + mdnames_not_in_md = [] + for md in attr['mdNames']: + if md.replace('.', '=') not in attr['md']: + mdnames_not_in_md.append(md) + + # md in mdNames + md_not_in_mdnames = [] + if 'md' in attr: + for md in attr['md']: + if 'mdNames' not in attr or md.replace('=', '.') not in attr['mdNames']: + md_not_in_mdnames.append(md) + + if len(mdnames_not_in_md) > 0: + s.append( + f"in attribute '{item}' metadata in mdNames not found in md object: {', '.join(mdnames_not_in_md)}") + if len(md_not_in_mdnames) > 0: + s.append( + f"in attribute '{item}' metadata in md object not found in mdNames: {', '.join(md_not_in_mdnames)}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule15(entities_collection): + """ + Rule 15: swapped subkeys in `_id` + + See README.md for an explanation of the rule + + This is a global rule, so the parameter is not an individual entity but the whole entities collection. + """ + s = [] + for entity in entities_collection.aggregate( + [ + { + '$group': { + '_id': {'id': '$_id.id', 'type': '$_id.type', 'servicePath': '$_id.servicePath'}, + 'count': {'$sum': 1} + } + }, + { + '$match': {'count': {'$gt': 1}} + } + ] + ): + id = entity['_id']['id'] + type = entity['_id']['type'] + service_path = entity['_id']['servicePath'] + count = entity['count'] + s.append( + f"_id uniqueness violation for entity id='{id}' type='{type}' servicePath='{service_path}' found {count} times") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule16(entity): + """ + Rule 16: `location` field inconsistency + + See README.md for an explanation of the rule + """ + # check that as much as one attribute is using geo type + geo_attrs = [] + for attr in entity['attrs']: + # type existence in attribute is checked by another rule + if 'type' in entity['attrs'][attr] and is_geo_type(entity['attrs'][attr]['type']) and not ignore_type(entity['attrs'][attr]): + geo_attrs.append(attr) + + if len(geo_attrs) > 1: + return f"more than one attribute with geo type: {', '.join(geo_attrs)}" + + if len(geo_attrs) == 1: + # If geo attr found, then check that there is consistent location field + geo_attr = geo_attrs[0] + geo_type = entity['attrs'][geo_attr]['type'] + if entity['attrs'][geo_attr]['value'] is None: + # if null value in geolocation attribute, then location field must not be present + if 'location' in entity: + return f"geo location '{geo_attr}' ({geo_type}) with null value and location field found" + else: + # not null value in geo location attribute case + if 'location' not in entity: + return f"geo location detected in '{geo_attr}' ({geo_type}) but location field not found in entity" + if entity['location']['attrName'] != geo_attr: + return f"location.attrName ({entity['location']['attrName']}) differs from '{geo_attr}'" + + geo_json = to_geo_json(entity['attrs'][geo_attr]) + + # https://www.testcult.com/deep-comparison-of-json-in-python/ + diff = DeepDiff(geo_json, entity['location']['coords'], ignore_order=True) + if diff: + # A typical difference is that attribute value uses strings and location uses numbers + # (this happens when the location was created/updated using NGSIv1). We try to identify that case + geo_json = convert_strings_to_numbers(geo_json) + if not DeepDiff(geo_json, entity['location']['coords'], ignore_order=True): + return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) is consistent, but value " \ + f"should use numbers for coordinates instead of strings" + else: + # Other causes + return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) value: {diff}" + else: # len(geo_attrs) == 0 + # If no geo attr found, check there isn't a location field + if 'location' in entity: + return f"location field detected but no geo attribute is present (maybe metadata location is used?)" + + +def rule17(entity): + """ + Rule 17: missing `lastCorrelator` + + See README.md for an explanation of the rule + """ + if 'lastCorrelator' not in entity: + return f"missing lastCorrelator" + else: + return None + + +def rule20(entity): + """ + Rule 20: entity id syntax + + See README.md for an explanation of the rule + """ + + # The existence of id in _id is checked by another rule + if 'id' in entity['_id']: + r = check_id(entity['_id']['id']) + if r is not None: + return f"entity id ({entity['_id']['id']}) syntax violation: {r}" + + return None + + +def rule21(entity): + """ + Rule 21: entity type syntax + + See README.md for an explanation of the rule + """ + + # The existence of type in _id is checked by another rule + if 'type' in entity['_id']: + r = check_id(entity['_id']['type']) + if r is not None: + return f"entity type ({entity['_id']['type']}) syntax violation: {r}" + + return None + + +def rule22(entity): + """ + Rule 22: entity servicePath syntax + + See README.md for an explanation of the rule + + Ref: https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path + """ + + # servicePath existence is checked by another rule + if 'servicePath' not in entity['_id']: + return None + + sp = entity['_id']['servicePath'] + # Scope must start with / (only "absolute" scopes are allowed) + if not sp.startswith('/'): + return f"servicePath '{sp}' does not starts with '/'" + + # This special case can be problematic (as split will detect always a level and the minimum length rule + # will break. So early return + if sp == '/': + return None + + # 10 maximum scope levels in a path + sp_levels = sp[1:].split('/') + if len(sp_levels) > 10: + return f"servicePath has {len(sp_levels)} tokens but the limit is 10" + + # 50 maximum characters in each level (1 char is minimum), only alphanumeric and underscore allowed + for i in range(len(sp_levels)): + if len(sp_levels[i]) == 0: + return f'servicePath level #{i} length is 0 but minimum is 1' + if len(sp_levels[i]) > 50: + return f'servicePath level #{i} length is {len(sp_levels[i])} but maximum is 50' + if re.search('[^a-zA-Z0-9_]', sp_levels[i]): + return f"unallowed characters in '{sp_levels[i]}' in servicePath level #{i}" + + +def rule23(entity): + """ + Rule 23: attribute name syntax + + See README.md for an explanation of the rule + """ + s = [] + + # attrNames-attrs consistency is checked by another rule. In the present rule we cannot assume + # they are consistent, so we do a union of both sets + attrs_to_check = set() + if 'attrNames' in entity: + attrs_to_check.update(entity['attrNames']) + for attr in entity['attrs']: + attrs_to_check.add(attr.replace('=', '.')) + + # check in the resulting union set + for attr in attrs_to_check: + r = check_id(attr) + if r is not None: + s.append(f"attribute name ({attr}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule24(entity): + """ + Rule 24: attribute type syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + if 'type' not in entity['attrs'][attr]: + s.append(f"in attribute '{attr}' type is missing") + else: + type = entity['attrs'][attr]['type'] + r = check_id(type) + if r is not None: + s.append(f"in attribute '{attr}' type ({type}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule25(entity): + """ + Rule 25: metadata name syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + # mdNames-md consistency is checked by another rule. In the present rule we cannot assume + # they are consistent, so we do a union of both sets + md_to_check = set() + if 'mdNames' in entity['attrs'][attr]: + md_to_check.update(entity['attrs'][attr]['mdNames']) + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + md_to_check.add(md.replace('=', '.')) + + # check in the resulting union set + for md in md_to_check: + r = check_id(md) + if r is not None: + s.append(f"in attribute '{attr}' metadata name ({md}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule26(entity): + """ + Rule 26: metadata type syntax + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + if 'type' not in entity['attrs'][attr]['md'][md]: + s.append(f"in attribute '{attr}' metadata '{md}' type is missing") + else: + type = entity['attrs'][attr]['md'][md]['type'] + r = check_id(type) + if r is not None: + s.append(f"in attribute '{attr}' metadata '{md}' type ({type}) syntax violation: {r}") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule90(entity): + """ + Rule 90: usage of `geo:x` attribute type where `x` different from `json` + + See README.md for an explanation of the rule + """ + for attr in entity['attrs']: + # type existence in attribute is checked by another rule + if 'type' in entity['attrs'][attr]: + type = entity['attrs'][attr]['type'] + if is_geo_type(type) and type != 'geo:json': + return f"in attribute '{attr}' usage of deprecated {type} found" + + return None + + +def rule91(entity): + """ + Rule 91: usage of more than one legacy `location` metadata + + See README.md for an explanation of the rule + """ + n = 0 + attrs = [] + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + if 'location' in entity['attrs'][attr]['md']: + attrs.append(attr) + + if len(attrs) > 1: + return f"location metadata found {n} times in attributes: {', '.join(attrs)} (maximum should be just 1)" + else: + return None + + +def rule92(entity): + """ + Rule 92: legacy `location` metadata should be `WGS84` or `WSG84` + + See README.md for an explanation of the rule + """ + s = [] + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + if 'location' in entity['attrs'][attr]['md']: + location_value = entity['attrs'][attr]['md']['location']['value'] + if location_value != 'WGS84' and location_value != 'WSG84': + s.append(f"in attribute '{attr}' location metadata value is {location_value} (should be WGS84 or WSG84)") + + if len(s) > 0: + return ', '.join(s) + else: + return None + + +def rule93(entity): + """ + Rule 93: usage of redundant legacy `location` + + See README.md for an explanation of the rule + """ + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + if md == 'location' and is_geo_type(entity['attrs'][attr]['type']): + return f"in attribute '{attr}' redundant location metadata found (attribute is already using {entity['attrs'][attr]['type']} type)" + + return None + + +def rule94(entity): + """ + Rule 94: usage of not redundant legacy `location` + + See README.md for an explanation of the rule + """ + for attr in entity['attrs']: + if 'md' in entity['attrs'][attr]: + for md in entity['attrs'][attr]['md']: + if md == 'location' and not is_geo_type(entity['attrs'][attr]['type']): + return f"in attribute '{attr}' location metadata found (attribute type is {entity['attrs'][attr]['type']})" + + return None + + +rules = [ + # Rules 1x + { + 'label': 'Rule10', + 'global': False, + 'func': rule10 + }, + { + 'label': 'Rule11', + 'global': False, + 'func': rule11 + }, + { + 'label': 'Rule12', + 'global': False, + 'func': rule12 + }, + { + 'label': 'Rule13', + 'global': False, + 'func': rule13 + }, + { + 'label': 'Rule14', + 'global': False, + 'func': rule14 + }, + { + 'label': 'Rule15', + 'global': True, + 'func': rule15 + }, + { + 'label': 'Rule16', + 'global': False, + 'func': rule16 + }, + { + 'label': 'Rule17', + 'global': False, + 'func': rule17 + }, + # Rules 2x + { + 'label': 'Rule20', + 'global': False, + 'func': rule20 + }, + { + 'label': 'Rule21', + 'global': False, + 'func': rule21 + }, + { + 'label': 'Rule22', + 'global': False, + 'func': rule22 + }, + { + 'label': 'Rule23', + 'global': False, + 'func': rule23 + }, + { + 'label': 'Rule24', + 'global': False, + 'func': rule24 + }, + { + 'label': 'Rule25', + 'global': False, + 'func': rule25 + }, + { + 'label': 'Rule26', + 'global': False, + 'func': rule26 + }, + # Rules 9x + { + 'label': 'Rule90', + 'global': False, + 'func': rule90 + }, + { + 'label': 'Rule91', + 'global': False, + 'func': rule91 + }, + { + 'label': 'Rule92', + 'global': False, + 'func': rule92 + }, + { + 'label': 'Rule93', + 'global': False, + 'func': rule93 + }, + { + 'label': 'Rule94', + 'global': False, + 'func': rule94 + } +] + + +def process_db(logger, db_name, db_conn, query, rules_exp): + """ + Process an individual DB + + :param logger: logger object + :param db_name: the name of the DB to process + :param db_conn: connection to MongoDB + :param query: query to filter entities to be processed + :param rules_exp: regular expression to filter rules to apply + :return: fails + """ + + logger.info(f'Processing {db_name}') + n = 0 + fails = 0 + + # check collection existence + if 'entities' not in db_conn[db_name].list_collection_names(): + logger.warning(f'collections entities not found in {db_name} database, nothing to do') + return + + # apply global rules + for rule in rules: + if rules_exp is not None and not re.search(rules_exp, rule['label']): + continue + + if rule['global']: + s = rule['func'](db_conn[db_name]['entities']) + if s is not None: + logger.warning(f'DB {db_name} {rule["label"]} violation in entities collection: {s}') + fails += 1 + + # apply per-entity rules + for entity in db_conn[db_name]['entities'].find(query): + n += 1 + id_string = json.dumps(entity['_id']) + logger.debug(f'* processing entity {id_string}') + for rule in rules: + if rules_exp is not None and not re.search(rules_exp, rule['label']): + continue + + if not rule['global']: + s = rule['func'](entity) + if s is not None: + logger.warning(f'DB {db_name} {rule["label"]} violation for entity {id_string}: {s}') + fails += 1 + + logger.info(f'processed {n} entities ({fails} rule violations)') + + return fails + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog='entities_consistency', + description='Check consistency in Orion entities collection in DB') + + parser.add_argument('--mongoUri', dest='mongo_uri', default='mongodb://localhost:27017', + help='MongoDB URI. Default is mongodb://localhost:27017') + parser.add_argument('--db', dest='db', + help='DB name to check. If omitted all DBs starting with "orion" will be checked.') + parser.add_argument('--query', dest='query', default='{}', + help='query to filter entities to check, in JSON MongoDB query language. By default, ' + 'all entities in the collection will be checked.') + parser.add_argument('--rulesExp', dest='rules_exp', + help='Specifies the rules to apply, as a regular expression. By default all rules are applied.') + parser.add_argument('--logLevel', dest='log_level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'], default='INFO', + help='log level. Default is INFO') + args = parser.parse_args() + + # sets the logging configuration + logging.basicConfig( + level=logging.getLevelName(args.log_level), + format="time=%(asctime)s | lvl=%(levelname)s | msg=%(message)s", + handlers=[ + logging.StreamHandler() + ] + ) + logger = logging.getLogger() + + # connect to MongoDB + mongo_client = MongoClient(args.mongo_uri) + db_names = mongo_client.list_database_names() + + # to remove starting and trailing ' char, in case it is used + query = json.loads(args.query.replace("'", "")) + + fails = 0 + if args.db is not None: + if args.db in db_names: + fails += process_db(logger, args.db, mongo_client, query, args.rules_exp) + else: + logger.fatal(f'database {args.db} does not exist') + sys.exit(1) + else: + # Process all Orion databases + for db_name in db_names: + if db_name.startswith('orion-'): + fails += process_db(logger, db_name, mongo_client, query, args.rules_exp) + + logger.info(f'total rule violations: {fails}') diff --git a/scripts/entities_consistency/requirements.txt b/scripts/entities_consistency/requirements.txt index 0c5b4e9193..99c8a008ba 100644 --- a/scripts/entities_consistency/requirements.txt +++ b/scripts/entities_consistency/requirements.txt @@ -1,2 +1,2 @@ -pymongo==4.6.1 -deepdiff==6.7.1 +pymongo==4.6.1 +deepdiff==6.7.1 diff --git a/scripts/entities_consistency/test_entities_consistency.py b/scripts/entities_consistency/test_entities_consistency.py index 87013558b9..bf2a76ffd4 100644 --- a/scripts/entities_consistency/test_entities_consistency.py +++ b/scripts/entities_consistency/test_entities_consistency.py @@ -1,55 +1,55 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -__author__ = 'fermin' - -# This is not the usual test that stimulates function and check assertions in the results :) -# -# Before running this test you have to load the entities testing set (validation_data.js) in the 'orion-validation' -# database in the local MongoDB database (check the MongoURI to match the one in your environment). You will get -# the result as log output. -# -# You can run this test under coverage, so you can check the coverage of the different rules in -# entities_consistency.py - -import unittest -from pymongo import MongoClient -import logging - -from entities_consistency import process_db - -class TestEntitiesConsistency(unittest.TestCase): - def test_process_db(self): - # sets the logging configuration - logging.basicConfig( - level=logging.getLevelName('INFO'), - format="time=%(asctime)s | lvl=%(levelname)s | msg=%(message)s", - handlers=[ - logging.StreamHandler() - ] - ) - logger = logging.getLogger() - - # connect to MongoDB and process validation DB - mongo_client = MongoClient('mongodb://localhost:47017') - process_db(logger, 'orion-validation', mongo_client, {}, None) +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +__author__ = 'fermin' + +# This is not the usual test that stimulates function and check assertions in the results :) +# +# Before running this test you have to load the entities testing set (validation_data.js) in the 'orion-validation' +# database in the local MongoDB database (check the MongoURI to match the one in your environment). You will get +# the result as log output. +# +# You can run this test under coverage, so you can check the coverage of the different rules in +# entities_consistency.py + +import unittest +from pymongo import MongoClient +import logging + +from entities_consistency import process_db + +class TestEntitiesConsistency(unittest.TestCase): + def test_process_db(self): + # sets the logging configuration + logging.basicConfig( + level=logging.getLevelName('INFO'), + format="time=%(asctime)s | lvl=%(levelname)s | msg=%(message)s", + handlers=[ + logging.StreamHandler() + ] + ) + logger = logging.getLogger() + + # connect to MongoDB and process validation DB + mongo_client = MongoClient('mongodb://localhost:47017') + process_db(logger, 'orion-validation', mongo_client, {}, None) diff --git a/scripts/entities_consistency/validation_data.js b/scripts/entities_consistency/validation_data.js index 17761fc027..94a41054ee 100644 --- a/scripts/entities_consistency/validation_data.js +++ b/scripts/entities_consistency/validation_data.js @@ -1,3530 +1,4186 @@ -db.getSiblingDB("orion-validation").entities.insertMany([ - { - "_id": { - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule10.1: missing entity id", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule10.2", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule10.2: missing entity type", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule10.3", - "type": "T" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule10.3: missing entity servicePath", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule11.1", - "type": "T", - "servicePath": "/SS" - }, - "attrs": { - "desc": { - "value": "Rule11.1: missing attrNames", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule11.2", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule11.2: missing entity creDate", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule11.3", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule11.3: missing entity modDate", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule12.1", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule12.1: missing attribute mdNames", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - } - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule12.2", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule12.2: missing attribute creDate", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule12.3", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule12.3: missing attribute modDate", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule13.1", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule13.1: attribute in attrsName but not in attrs object", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule13.2", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule13.2: attribute in attrs object but not in attrNames", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule13.3", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule13.3: attribute in attrsName but not in attrs object with dot in the name", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule13.4", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2" - ], - "attrs": { - "desc": { - "value": "Rule13.4: attribute in attrs object but not in attrNames with dot in the name", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule14.1", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule14.1: md in mdNames but not in md object", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule14.2", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule14.2: md in md object but not in mdNames", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule14.3", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule14.3: md in mdNames but not in md object with dot in the name", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule14.4", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule14.4: md in md object but not in mdNames with dot in the name", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule15", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule15.1: conflicting entity", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "type": "T", - "id": "Rule15", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule15.2: conflicting entity", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "servicePath": "/SS", - "id": "Rule15", - "type": "T" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule15.3: conflicting entity", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.1", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location", - "ignoredLocation", - "otherLocation" - ], - "attrs": { - "desc": { - "value": "Rule16.1: more than one geo-attribute", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "mdNames": [] - }, - "ignoredLocation": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "md": { - "ignoreType": { - "type": "Bool", - "value": true - } - }, - "mdNames": [ - "ignoreType" - ] - }, - "otherLocation": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "mdNames": [] - } - }, - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "location": { - "attrName": "location", - "coords": { - "type": "Point", - "coordinates": [ - 2, - 1 - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.2", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "ignoredLocation" - ], - "attrs": { - "desc": { - "value": "Rule16.2: no geo-attribute but location field", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "ignoredLocation": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "md": { - "ignoreType": { - "type": "Bool", - "value": true - } - }, - "mdNames": [ - "ignoreType" - ] - } - }, - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "location": { - "attrName": "location", - "coords": { - "type": "Point", - "coordinates": [ - 2, - 1 - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.3", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location", - "ignoredLocation" - ], - "attrs": { - "desc": { - "value": "Rule16.3: geo-attribute with null value and location field", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": null, - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "mdNames": [] - }, - "ignoredLocation": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "md": { - "ignoreType": { - "type": "Bool", - "value": true - } - }, - "mdNames": [ - "ignoreType" - ] - } - }, - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "location": { - "attrName": "location", - "coords": { - "type": "Point", - "coordinates": [ - 2, - 1 - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.4", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location", - "ignoredLocation" - ], - "attrs": { - "desc": { - "value": "Rule16.4: geo-attribute but not location field", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "mdNames": [] - }, - "ignoredLocation": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "md": { - "ignoreType": { - "type": "Bool", - "value": true - } - }, - "mdNames": [ - "ignoreType" - ] - } - }, - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.5", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location", - "ignoredLocation" - ], - "attrs": { - "desc": { - "value": "Rule16.5: location.attrName inconsistency", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "mdNames": [] - }, - "ignoredLocation": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "md": { - "ignoreType": { - "type": "Bool", - "value": true - } - }, - "mdNames": [ - "ignoreType" - ] - } - }, - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "location": { - "attrName": "ignoredLocation", - "coords": { - "type": "Point", - "coordinates": [ - 2, - 1 - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.6", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location" - ], - "attrs": { - "desc": { - "value": "Rule16.6: geo:point coords inconsistency", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": "1, 2", - "type": "geo:point", - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "mdNames": [] - } - }, - "creDate": 1705933908.9073899, - "modDate": 1705933908.9073899, - "location": { - "attrName": "location", - "coords": { - "type": "Point", - "coordinates": [ - 20, - 1 - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.7", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location" - ], - "attrs": { - "desc": { - "value": "Rule16.7: geo:line coords inconsistency", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": [ - "1, 2", - "3, 4" - ], - "type": "geo:line", - "creDate": 1705933908.9088006, - "modDate": 1705933908.9088006, - "mdNames": [] - } - }, - "creDate": 1705933908.9088006, - "modDate": 1705933908.9088006, - "location": { - "attrName": "location", - "coords": { - "type": "LineString", - "coordinates": [ - [ - 2, - 1 - ], - [ - 40, - 3 - ] - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.8", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location" - ], - "attrs": { - "desc": { - "value": "Rule16.8: geo:box coords inconsistency", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": [ - "1, 2", - "3, 4" - ], - "type": "geo:box", - "creDate": 1705933908.9101331, - "modDate": 1705933908.9101331, - "mdNames": [] - } - }, - "creDate": 1705933908.9101331, - "modDate": 1705933908.9101331, - "location": { - "attrName": "location", - "coords": { - "type": "Polygon", - "coordinates": [ - [ - [ - 2, - 1 - ], - [ - 2, - 3 - ], - [ - 4, - 30 - ], - [ - 4, - 1 - ], - [ - 2, - 1 - ] - ] - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.9", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location" - ], - "attrs": { - "desc": { - "value": "Rule16.9: geo:polygon inconsistency", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": [ - "1, 2", - "10, 20", - "10, -20", - "1, 2" - ], - "type": "geo:polygon", - "creDate": 1705933908.9129946, - "modDate": 1705933908.9129946, - "mdNames": [] - } - }, - "creDate": 1705933908.9129946, - "modDate": 1705933908.9129946, - "location": { - "attrName": "location", - "coords": { - "type": "Polygon", - "coordinates": [ - [ - [ - 2, - 1 - ], - [ - 20, - 10 - ], - [ - -20, - 10 - ] - ] - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.10", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location" - ], - "attrs": { - "desc": { - "value": "Rule16.10: geo:json coords inconsistency", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": { - "type": "Polygon", - "coordinates": [ - [ - [ - 1, - 2 - ], - [ - 10, - 20 - ], - [ - 10, - -20 - ], - [ - 1, - 2 - ] - ] - ] - }, - "type": "geo:json", - "creDate": 1705933908.9142942, - "modDate": 1705933908.9142942, - "mdNames": [] - } - }, - "creDate": 1705933908.9142942, - "modDate": 1705933908.9142942, - "location": { - "attrName": "location", - "coords": { - "type": "Polygon", - "coordinates": [ - [ - [ - 1, - 2 - ], - [ - 10, - 20 - ], - [ - 10, - -200 - ], - [ - 1, - 2 - ] - ] - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule16.11", - "type": "T", - "servicePath": "/" - }, - "attrNames": [ - "desc", - "location" - ], - "attrs": { - "desc": { - "value": "Rule16.11: geo:json coords strings", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "location": { - "value": { - "type": "Polygon", - "coordinates": [ - [ - [ - "1", - "2" - ], - [ - "10", - "20" - ], - [ - "10", - "-20" - ], - [ - "1", - "2" - ] - ] - ] - }, - "type": "geo:json", - "creDate": 1705933908.9142942, - "modDate": 1705933908.9142942, - "mdNames": [] - } - }, - "creDate": 1705933908.9142942, - "modDate": 1705933908.9142942, - "location": { - "attrName": "location", - "coords": { - "type": "Polygon", - "coordinates": [ - [ - [ - 1, - 2 - ], - [ - 10, - 20 - ], - [ - 10, - -20 - ], - [ - 1, - 2 - ] - ] - ] - } - }, - "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" - }, - { - "_id": { - "id": "Rule17.1", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule17.1: missing lastCorrelator field", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858 - }, - { - "_id": { - "id": "", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule20.1: id syntax minimum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule20.2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule20.2: id syntax maximum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule20.3(", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule20.3: id syntax general forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule20.4#", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule20.4: id syntax identifier forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule21.1", - "type": "", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule21.1: type syntax minimum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule21.2", - "type": "Txxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule21.2: type syntax maximum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule21.3", - "type": "T(", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule21.3: type syntax general forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule21.4", - "type": "T#", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule21.4: type syntax identifier forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule22.1", - "type": "T", - "servicePath": "SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule21.1: servicePath does not starts with slash", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule22.2", - "type": "T", - "servicePath": "/S1/S2/S3/S4/S5/S6/S7/S8/S9/S10/S11" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule21.2: more than 10 levels in servicePath", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule22.3", - "type": "T", - "servicePath": "/S1//S3/" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule22.3: servicePath level less than minimum", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule22.4", - "type": "T", - "servicePath": "/S1/Sxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/S3/" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule22.4: servicePath level greater than maximum", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule22.5", - "type": "T", - "servicePath": "/S1/S#/S3/" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule22.5: servicePath syntax problem", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule23.1", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule23.1: attr name syntax minimum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule23.2", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule23.2: attr name syntax maximum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule23.3", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1(", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule23.3: attr name syntax general forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1(": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule23.4", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1#", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule23.4: attr name syntax identifier forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1#": { - "value": 10, - "type": "Number#", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule24.1", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule24.1: attr type syntax minimum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule24.2", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule24.2: attr type syntax maximum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Numberxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule24.3", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule24.3: attr type syntax general forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number(", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule24.4", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule24.4: attr type syntax identifier forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number#", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule25.1", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule25.1: md name syntax minimum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule25.2", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule25.2: md name syntax maximum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule25.3", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule25.3: md name syntax general forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule25.4", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule25.4: md name syntax identifier forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1#": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1#", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule26.1", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule26.1: md type syntax minimum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule26.2", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule26.2: md type syntax maximum length", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Numberxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule26.3", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule26.3: md type syntax general forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number(", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - }, - { - "_id": { - "id": "Rule26.4", - "type": "T", - "servicePath": "/SS" - }, - "attrNames": [ - "desc", - "A1", - "A2", - "A.3" - ], - "attrs": { - "desc": { - "value": "Rule26.4: md type syntax identifier forbidden", - "type": "Text", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A1": { - "value": 10, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "md": { - "MD1": { - "type": "Number#", - "value": 100 - }, - "MD=2": { - "type": "Number", - "value": 200 - } - }, - "mdNames": [ - "MD1", - "MD.2" - ] - }, - "A2": { - "value": 20, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - }, - "A=3": { - "value": 30, - "type": "Number", - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "mdNames": [] - } - }, - "creDate": 1705931202.187858, - "modDate": 1705931202.187858, - "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" - } +db.getSiblingDB("orion-validation").entities.insertMany([ + { + "_id": { + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule10.1: missing entity id", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule10.2", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule10.2: missing entity type", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule10.3", + "type": "T" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule10.3: missing entity servicePath", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule11.1", + "type": "T", + "servicePath": "/SS" + }, + "attrs": { + "desc": { + "value": "Rule11.1: missing attrNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule11.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule11.2: missing entity creDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule11.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule11.3: missing entity modDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule12.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule12.1: missing attribute mdNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + } + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule12.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule12.2: missing attribute creDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule12.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule12.3: missing attribute modDate", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule13.1: attribute in attrsName but not in attrs object", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule13.2: attribute in attrs object but not in attrNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule13.3: attribute in attrsName but not in attrs object with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule13.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2" + ], + "attrs": { + "desc": { + "value": "Rule13.4: attribute in attrs object but not in attrNames with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.1: md in mdNames but not in md object", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.2: md in md object but not in mdNames", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.3: md in mdNames but not in md object with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule14.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule14.4: md in md object but not in mdNames with dot in the name", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule15", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule15.1: conflicting entity", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "type": "T", + "id": "Rule15", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule15.2: conflicting entity", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "servicePath": "/SS", + "id": "Rule15", + "type": "T" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule15.3: conflicting entity", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation", + "otherLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.1: more than one geo-attribute", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + }, + "otherLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.2", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.2: no geo-attribute but location field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.3", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.3: geo-attribute with null value and location field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": null, + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.4", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.4: geo-attribute but not location field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.5", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule16.5: location.attrName inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + }, + "ignoredLocation": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + } + }, + "mdNames": [ + "ignoreType" + ] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "ignoredLocation", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.6", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.6: geo:point coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "mdNames": [] + } + }, + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 20, + 1 + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.7", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.7: geo:line coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:line", + "creDate": 1705933908.9088006, + "modDate": 1705933908.9088006, + "mdNames": [] + } + }, + "creDate": 1705933908.9088006, + "modDate": 1705933908.9088006, + "location": { + "attrName": "location", + "coords": { + "type": "LineString", + "coordinates": [ + [ + 2, + 1 + ], + [ + 40, + 3 + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.8", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.8: geo:box coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:box", + "creDate": 1705933908.9101331, + "modDate": 1705933908.9101331, + "mdNames": [] + } + }, + "creDate": 1705933908.9101331, + "modDate": 1705933908.9101331, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 2, + 3 + ], + [ + 4, + 30 + ], + [ + 4, + 1 + ], + [ + 2, + 1 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.9", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.9: geo:polygon inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "10, 20", + "10, -20", + "1, 2" + ], + "type": "geo:polygon", + "creDate": 1705933908.9129946, + "modDate": 1705933908.9129946, + "mdNames": [] + } + }, + "creDate": 1705933908.9129946, + "modDate": 1705933908.9129946, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 20, + 10 + ], + [ + -20, + 10 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.10", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.10: geo:json coords inconsistency", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + }, + "type": "geo:json", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "mdNames": [] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -200 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.11", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.11: geo:json coords strings", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + "1", + "2" + ], + [ + "10", + "20" + ], + [ + "10", + "-20" + ], + [ + "1", + "2" + ] + ] + ] + }, + "type": "geo:json", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "mdNames": [] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule17.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule17.1: missing lastCorrelator field", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858 + }, + { + "_id": { + "id": "", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.1: id syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule20.2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.2: id syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule20.3(", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.3: id syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule20.4#", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule20.4: id syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.1", + "type": "", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.1: type syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.2", + "type": "Txxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.2: type syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.3", + "type": "T(", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.3: type syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule21.4", + "type": "T#", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.4: type syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.1", + "type": "T", + "servicePath": "SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.1: servicePath does not starts with slash", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.2", + "type": "T", + "servicePath": "/S1/S2/S3/S4/S5/S6/S7/S8/S9/S10/S11" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule21.2: more than 10 levels in servicePath", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.3", + "type": "T", + "servicePath": "/S1//S3/" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule22.3: servicePath level less than minimum", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.4", + "type": "T", + "servicePath": "/S1/Sxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/S3/" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule22.4: servicePath level greater than maximum", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule22.5", + "type": "T", + "servicePath": "/S1/S#/S3/" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule22.5: servicePath syntax problem", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.1: attr name syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.2: attr name syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1(", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.3: attr name syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1(": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule23.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1#", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule23.4: attr name syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1#": { + "value": 10, + "type": "Number#", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.1: attr type syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.2: attr type syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Numberxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.3: attr type syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number(", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.4: attr type syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number#", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule24.5", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule24.5: attr type is missing", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.1: md name syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.2: md name syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.3: md name syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule25.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule25.4: md name syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1#": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1#", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.1", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.1: md type syntax minimum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.2", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.2: md type syntax maximum length", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Numberxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.3", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.3: md type syntax general forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number(", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.4", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.4: md type syntax identifier forbidden", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "type": "Number#", + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule26.5", + "type": "T", + "servicePath": "/SS" + }, + "attrNames": [ + "desc", + "A1", + "A2", + "A.3" + ], + "attrs": { + "desc": { + "value": "Rule26.5: md type is missing", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A1": { + "value": 10, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "md": { + "MD1": { + "value": 100 + }, + "MD=2": { + "type": "Number", + "value": 200 + } + }, + "mdNames": [ + "MD1", + "MD.2" + ] + }, + "A2": { + "value": 20, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "A=3": { + "value": 30, + "type": "Number", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + } + }, + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "lastCorrelator": "acae5f4c-b92c-11ee-8f0c-080027cd35f1" + }, + { + "_id": { + "id": "Rule90.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule90.1: usage of legacy geo:point", + "type": "Text", + "creDate": 1706006146.1601377, + "modDate": 1706006146.1601377, + "mdNames": [] + }, + "location": { + "value": "1, 2", + "type": "geo:point", + "creDate": 1706006146.1601377, + "modDate": 1706006146.1601377, + "mdNames": [] + } + }, + "creDate": 1706006146.1601377, + "modDate": 1706006146.1601377, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + 2, + 1 + ] + } + }, + "lastCorrelator": "2ac4f6a8-b9db-11ee-8c49-080027cd35f1" + }, + { + "_id": { + "id": "Rule90.2", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule90.2: usage of legacy geo:line", + "type": "Text", + "creDate": 1706006146.17774, + "modDate": 1706006146.17774, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:line", + "creDate": 1706006146.17774, + "modDate": 1706006146.17774, + "mdNames": [] + } + }, + "creDate": 1706006146.17774, + "modDate": 1706006146.17774, + "location": { + "attrName": "location", + "coords": { + "type": "LineString", + "coordinates": [ + [ + 2, + 1 + ], + [ + 4, + 3 + ] + ] + } + }, + "lastCorrelator": "2ac4f6a8-b9db-11ee-8c49-080027cd35f1" + }, + { + "_id": { + "id": "Rule90.3", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule90.3: usage of legacy geo:box", + "type": "Text", + "creDate": 1706006146.179189, + "modDate": 1706006146.179189, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "3, 4" + ], + "type": "geo:box", + "creDate": 1706006146.179189, + "modDate": 1706006146.179189, + "mdNames": [] + } + }, + "creDate": 1706006146.179189, + "modDate": 1706006146.179189, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 2, + 3 + ], + [ + 4, + 3 + ], + [ + 4, + 1 + ], + [ + 2, + 1 + ] + ] + ] + } + }, + "lastCorrelator": "2ac4f6a8-b9db-11ee-8c49-080027cd35f1" + }, + { + "_id": { + "id": "Rule90.4", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule90.4: usage of legacy geo:polygon", + "type": "Text", + "creDate": 1706006146.1812057, + "modDate": 1706006146.1812057, + "mdNames": [] + }, + "location": { + "value": [ + "1, 2", + "10, 20", + "10, -20", + "1, 2" + ], + "type": "geo:polygon", + "creDate": 1706006146.1812057, + "modDate": 1706006146.1812057, + "mdNames": [] + } + }, + "creDate": 1706006146.1812057, + "modDate": 1706006146.1812057, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 2, + 1 + ], + [ + 20, + 10 + ], + [ + -20, + 10 + ], + [ + 2, + 1 + ] + ] + ] + } + }, + "lastCorrelator": "2ac4f6a8-b9db-11ee-8c49-080027cd35f1" + }, + { + "_id": { + "id": "Rule91.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location", + "ignoredLocation" + ], + "attrs": { + "desc": { + "value": "Rule91.1: no more than one location metadata", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + }, + "type": "polygon", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "md": { + "location": { + "type": "string", + "value": "WSG84" + } + }, + "mdNames": [ + "location" + ] + }, + "ignoredLocation": { + "value": null, + "type": "point", + "creDate": 1705933908.9073899, + "modDate": 1705933908.9073899, + "md": { + "ignoreType": { + "type": "Bool", + "value": true + }, + "location": { + "type": "string", + "value": "WSG84" + } + }, + "mdNames": [ + "ignoreType", + "location" + ] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule92.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule92.1: location must be WSG84 or WGS84", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + }, + "type": "polygon", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "md": { + "location": { + "type": "string", + "value": "WSG99" + } + }, + "mdNames": [ + "location" + ] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule93.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule93.1: redundant metadata location", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + }, + "type": "geo:json", + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "md": { + "location": { + "type": "string", + "value": "WSG84" + } + }, + "mdNames": [ + "location" + ] + } + }, + "creDate": 1705933908.9142942, + "modDate": 1705933908.9142942, + "location": { + "attrName": "location", + "coords": { + "type": "Polygon", + "coordinates": [ + [ + [ + 1, + 2 + ], + [ + 10, + 20 + ], + [ + 10, + -20 + ], + [ + 1, + 2 + ] + ] + ] + } + }, + "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" + }, + { + "_id": { + "id": "Rule94.1", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule94.1: non redundant location metadata", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": "40.418889, -3.691944", + "type": "coords", + "creDate": 1706007163.0475833, + "modDate": 1706007163.0475833, + "md": { + "location": { + "type": "string", + "value": "WSG84" + } + }, + "mdNames": [ + "location" + ] + } + }, + "creDate": 1706007163.0475833, + "modDate": 1706007163.0475833, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + -3.691944, + 40.418889 + ] + } + }, + "lastCorrelator": "88e1bc10-b9dd-11ee-8755-080027cd35f1" + } ]) \ No newline at end of file From 2ae8bc8b17a0c05cdbdf36caf5bd56703176a57c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 23 Jan 2024 15:10:23 +0100 Subject: [PATCH 104/390] FIX improvements --- scripts/entities_consistency/README.md | 52 +- .../entities_consistency.py | 36 +- .../entities_consistency/validation_data.js | 12 +- .../managedb/check_entities_consistency.py | 189 ----- scripts/managedb/check_location_coherence.py | 718 ------------------ 5 files changed, 52 insertions(+), 955 deletions(-) delete mode 100755 scripts/managedb/check_entities_consistency.py delete mode 100755 scripts/managedb/check_location_coherence.py diff --git a/scripts/entities_consistency/README.md b/scripts/entities_consistency/README.md index aa3ddb578a..0fbbc8a542 100644 --- a/scripts/entities_consistency/README.md +++ b/scripts/entities_consistency/README.md @@ -1,7 +1,7 @@ The entities consistency script analyze the contents of the entities database in orion DBs and check several consistency rules, reporting violations found. -Ref: [entity document datamodel]([../..//doc/manuals/admin/database_model.md#entities-collection]) +Ref: [entity document database model]([../../doc/manuals/admin/database_model.md#entities-collection]) ## Requirements @@ -14,13 +14,13 @@ Run `entities_consistency.py -h` for arguments details. ## Rules -* Rules 1x: DB inconsistencies (severe problems) +* Rules 1x: DB inconsistencies (use to be severe problems) * Rules 2x: Syntax restrictions * Rules 9x: Usage of legacy features -### Rule 10: `_id` field inconsistency +### Rule 10: `_id` field consistency -Each entity in DB has a `_id` fields with three subfields: +Each entity in DB has an `_id` field with three subfields: * `id` * `type` @@ -28,56 +28,56 @@ Each entity in DB has a `_id` fields with three subfields: ## Rule 11: mandatory fields in entity -The following field are mandatory: +The following fields are mandatory: * `attrNames` * `creDate` * `modDate` -It is not an exhaustive check of every attribute in the datamodel, but some entities created/updated with old Orion version may be missing them. +It is not an exhaustive check of every field in the database model, but some entities created/updated with old Orion versions may be missing them. ## Rule 12: mandatory fields in attribute -The following subfield are mandatory for every attribute: +The following subfields are mandatory for every attribute: * `mdNames` * `creDate` * `modDate` -It is not an exhaustive check of every attribute in the datamodel, but some entities created/updated with old Orion version may be missing them. +It is not an exhaustive check of every field in the database model, but some entities created/updated with old Orion versions may be missing them. -### Rule 13: `attrNames` field inconsistency +### Rule 13: `attrNames` field consistency -Each item in `attrNames` array has a corresponding key in `attrs` object and the other way around. +For each item in `attrNames` array there is a corresponding key in `attrs` object and the other way around. -### Rule 14: `mdNames` field inconsistency +### Rule 14: `mdNames` field consistency -For every attribute, for each item in `mdNames` array has a corresponding key in `md` object and the other way around. +For every attribute, for each item in `mdNames` array there is a corresponding key in `md` object and the other way around. -### Rule 15: swapped subkeys in `_id` +### Rule 15: not swapped subkeys in `_id` In MongoDB JSON objects are stored taking order into account, so DB allows to have a document with `_id` equal to `{"id": "E", "type": "T", "servicePath": "/"}` and at the same time have another document with `_id` equal to `{"type": "T", "id": "E", "servicePath": "/"}` without violating `_id` uniqueness constraint. -This rule checks that this is not happening. +This rule checks that this is not happening in the entities collection. -### Rule 16: `location` field inconsistency +### Rule 16: `location` field consistency Check that location in consistent. In particular: -* As much as one attribute with `geo:point`, `geo:line`, `geo:box`, `geo:polygon` or `geo:json` type -* If an attribute is found with one of above types, check the `location` field is consistent -* If no attribute is found with one of above types, check no `location` field is not found +* As much as one attribute with `geo:point`, `geo:line`, `geo:box`, `geo:polygon` or `geo:json` type (without `ignoreTypes` and without `null` value). +* If one of such attribute is found, check the `location` field is found and its content is consistent +* If none of such attribute is found, check no `location` field is not found * If the location of the entity is defined using deprecated `location` metadata it will be detected as this case. Additional rules in the 9x group can help to diagnose this situation. This rule is for location inconsistencies. For usage of deprecated geo types, there are additional rules in the 9x group. -### Rule 17: missing `lastCorrelator` +### Rule 17: `lastCorrelator` existence Check if `lastCorrelator` is included. -This field was introduced in Orion 1.8.0 (September 11th, 2017). +This field was introduced in [Orion 1.8.0](https://github.com/telefonicaid/fiware-orion/releases/tag/1.8.0) (released in September 11th, 2017). ### Rule 20: entity id syntax @@ -107,7 +107,7 @@ Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifie Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. -### Rule 90: usage of `geo:x` attribute type where `x` different from `json` +### Rule 90: detect usage of `geo:x` attribute type where `x` different from `json` Check usage of deprecated geo-location types, i.e: @@ -121,15 +121,15 @@ Suggested action is to: * Change attribute type to `geo:json` * Set the attribute value to the same GeoJSON in `location.coords` field -Note this rule doesn't check location consistency for this case (e.g. more than one geo-location attribute in the same entity). That's done by another rule in the 1x group). +Note this rule doesn't check location consistency for this case (e.g. more than one geo-location attribute in the same entity). That's done by another rule in the 1x group. -### Rule 91: usage of more than one legacy `location` metadata +### Rule 91: detect usage of more than one legacy `location` metadata Check usage of `location` in more than one attribute of the same entity. Note this rule doesn't check location consistency for this case (that's done by another rule in the 1x group). -### Rule 92: legacy `location` metadata should be `WGS84` or `WSG84` +### Rule 92: detect legacy `location` metadata should be `WGS84` or `WSG84` The value of the `location` metadata should be `WGS84` or `WSG84`. @@ -138,7 +138,7 @@ Additional consideration: * Entities attributes may have `location` metadata with values different from `WGS84` or `WSG84` if created using NGSIv2. That would be a false positive in this rule validation * This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). -### Rule 93: usage of redundant legacy `location` +### Rule 93: detect usage of redundant legacy `location` Checks usage of redundant `location` metadata, i.e. when at the same time a `geo:` type is used in the same attribute. @@ -150,7 +150,7 @@ Additional, considerations: * This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. * This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). -### Rule 94: usage of not redundant legacy `location` +### Rule 94: detect usage of not redundant legacy `location` Checks usage of not redundant `location` metadata, i.e. when at the same time the type of the attribute is nog `geo:`. same attribute. diff --git a/scripts/entities_consistency/entities_consistency.py b/scripts/entities_consistency/entities_consistency.py index 5bf2786dfa..fd375cbccc 100644 --- a/scripts/entities_consistency/entities_consistency.py +++ b/scripts/entities_consistency/entities_consistency.py @@ -175,7 +175,7 @@ def check_id(id): # Rules functions def rule10(entity): """ - Rule 10: `_id` field inconsistency + Rule 10: `_id` field consistency See README.md for an explanation of the rule """ @@ -234,7 +234,7 @@ def rule12(entity): def rule13(entity): """ - Rule 13: `attrNames` field inconsistency + Rule 13: `attrNames` field consistency See README.md for an explanation of the rule """ @@ -269,7 +269,7 @@ def rule13(entity): def rule14(entity): """ - Rule 14: `mdNames` field inconsistency + Rule 14: `mdNames` field consistency See README.md for an explanation of the rule """ @@ -308,7 +308,7 @@ def rule14(entity): def rule15(entities_collection): """ - Rule 15: swapped subkeys in `_id` + Rule 15: not swapped subkeys in `_id` See README.md for an explanation of the rule @@ -343,7 +343,7 @@ def rule15(entities_collection): def rule16(entity): """ - Rule 16: `location` field inconsistency + Rule 16: `location` field consistency See README.md for an explanation of the rule """ @@ -368,7 +368,7 @@ def rule16(entity): else: # not null value in geo location attribute case if 'location' not in entity: - return f"geo location detected in '{geo_attr}' ({geo_type}) but location field not found in entity" + return f"geo location '{geo_attr}' ({geo_type}) not null but location field not found in entity" if entity['location']['attrName'] != geo_attr: return f"location.attrName ({entity['location']['attrName']}) differs from '{geo_attr}'" @@ -394,7 +394,7 @@ def rule16(entity): def rule17(entity): """ - Rule 17: missing `lastCorrelator` + Rule 17: `lastCorrelator` existence See README.md for an explanation of the rule """ @@ -579,27 +579,31 @@ def rule26(entity): def rule90(entity): """ - Rule 90: usage of `geo:x` attribute type where `x` different from `json` + Rule 90: detect usage of `geo:x` attribute type where `x` different from `json` See README.md for an explanation of the rule """ + + # note we could have more than one case, as ignoreType can be in use + s = [] for attr in entity['attrs']: # type existence in attribute is checked by another rule if 'type' in entity['attrs'][attr]: type = entity['attrs'][attr]['type'] if is_geo_type(type) and type != 'geo:json': - return f"in attribute '{attr}' usage of deprecated {type} found" + s.append(f"{attr} ({type})") - return None + # + if len(s) > 0: + return f"usage of deprecated geo type in attributes: {', '.join(s)}" def rule91(entity): """ - Rule 91: usage of more than one legacy `location` metadata + Rule 91: detect usage of more than one legacy `location` metadata See README.md for an explanation of the rule """ - n = 0 attrs = [] for attr in entity['attrs']: if 'md' in entity['attrs'][attr]: @@ -607,14 +611,14 @@ def rule91(entity): attrs.append(attr) if len(attrs) > 1: - return f"location metadata found {n} times in attributes: {', '.join(attrs)} (maximum should be just 1)" + return f"location metadata found {len(attrs)} times in attributes: {', '.join(attrs)} (maximum should be just 1)" else: return None def rule92(entity): """ - Rule 92: legacy `location` metadata should be `WGS84` or `WSG84` + Rule 92: detect legacy `location` metadata should be `WGS84` or `WSG84` See README.md for an explanation of the rule """ @@ -634,7 +638,7 @@ def rule92(entity): def rule93(entity): """ - Rule 93: usage of redundant legacy `location` + Rule 93: detect usage of redundant legacy `location` See README.md for an explanation of the rule """ @@ -649,7 +653,7 @@ def rule93(entity): def rule94(entity): """ - Rule 94: usage of not redundant legacy `location` + Rule 94: detect usage of not redundant legacy `location` See README.md for an explanation of the rule """ diff --git a/scripts/entities_consistency/validation_data.js b/scripts/entities_consistency/validation_data.js index 94a41054ee..6564325294 100644 --- a/scripts/entities_consistency/validation_data.js +++ b/scripts/entities_consistency/validation_data.js @@ -1589,7 +1589,7 @@ db.getSiblingDB("orion-validation").entities.insertMany([ 1 ], [ - 20, + 200, 10 ], [ @@ -2782,7 +2782,7 @@ db.getSiblingDB("orion-validation").entities.insertMany([ }, "A1#": { "value": 10, - "type": "Number#", + "type": "Number", "creDate": 1705931202.187858, "modDate": 1705931202.187858, "md": { @@ -2841,7 +2841,7 @@ db.getSiblingDB("orion-validation").entities.insertMany([ }, "A1": { "value": 10, - "type": "Number", + "type": "", "creDate": 1705931202.187858, "modDate": 1705931202.187858, "md": { @@ -3257,7 +3257,7 @@ db.getSiblingDB("orion-validation").entities.insertMany([ "creDate": 1705931202.187858, "modDate": 1705931202.187858, "md": { - "MD1": { + "MD1(": { "type": "Number", "value": 100 }, @@ -3267,7 +3267,7 @@ db.getSiblingDB("orion-validation").entities.insertMany([ } }, "mdNames": [ - "MD1", + "MD1(", "MD.2" ] }, @@ -3376,7 +3376,7 @@ db.getSiblingDB("orion-validation").entities.insertMany([ "modDate": 1705931202.187858, "md": { "MD1": { - "type": "Number", + "type": "", "value": 100 }, "MD=2": { diff --git a/scripts/managedb/check_entities_consistency.py b/scripts/managedb/check_entities_consistency.py deleted file mode 100755 index 192d90803f..0000000000 --- a/scripts/managedb/check_entities_consistency.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# Hint: use 'PYTHONIOENCODING=utf8 python check_metadata_id.py' if you are going to redirect the output of this -# script to a file - -# Checks done: -# -# 1. heck that there is no duplication in _id fields (for mongo {_id: {x: 1, y: 2}} and {_id: {y:2, x:1}} are different documents) -# 2. All attribute in 'attrs' are in 'attrNames' and the other way around -# -# -# FIXME: integrate the checkings in check_location_coherence.py into this script (maybe using some kind of -# modular approach?) - - -__author__ = 'fermin' - -from pymongo import MongoClient -import json -import sys -from datetime import datetime -from time import sleep - -ATTRS = 'attrs' -ATTRNAMES = 'attrNames' - -def date2string(time): - """ - Convert date to string - - :param time: date time (as timestamp) - :return: string representing the date time - """ - - return datetime.fromtimestamp(time).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def base_name(attr): - """ - Returns base name for attr, without take into account id (if any) - :param attr: - :return: - """ - return attr.split('()')[0] - -def check_entity_dup(entity_doc): - """ - Check for entity duplicates in DB - :param entity_doc: entity document - :return: [], a list of errors otherwise - """ - - query = {'_id.id': entity_doc['_id']['id']} - - if 'type' in entity_doc['_id']: - query['_id.type'] = entity_doc['_id']['type'] - else: - query['_id.type'] = {'$exists': False} - - if 'servicePath' in entity_doc['_id']: - query['_id.servicePath'] = entity_doc['_id']['servicePath'] - else: - query['_id.servicePath'] = {'$exists': False} - - c = db[COL].find(query).count() - if c > 1: - return [' * ERROR: duplicated entities number with same _id subfields: {0}'.format(c)] - else: - return [] - - -def check_attrs(attrs, attr_names): - """ - Check attrs - - :param attrs: key-value attrs - :param attr_names: list with attr_names - :return: [], a list of errors otherwise - """ - - r = [] - - # We have found that some times attrs containts non-ASCII characters and printing can be problematic if - # we don't use .encode('utf-8') - - # All in attrs is in attr_names - for attr in attrs.keys(): - if base_name(attr).replace('=','.') not in attr_names: - r.append(' * ERROR: base name of attr <{0}> in attrs keymap cannot be found in attrNames list'.format(attr.encode('utf-8'))) - - # All in attr_names is in attrs - for attr in attr_names: - # map() used to get only the first token in attrs keys - if attr not in map(lambda x: x.split('()')[0].replace('=','.'), attrs.keys()): - r.append(' * ERROR: attr <{0}> in attrName list cannot be found in attrs keymap'.format(attr.encode('utf-8'))) - - return r - - -########################## -# Main program starts here - -if len(sys.argv) != 2: - print "invalid number of arguments, please specifi db name (e.g. 'orion')" - sys.exit() - -DB = sys.argv[1] -COL = 'entities' - -uri = 'mongodb://localhost:27017' -client = MongoClient(uri) -db = client[DB] - -# At the end, n_processed = no_ok + n_not_ok -n_processed = 0 -n_ok = 0 -n_not_ok = 0 - -total = db[COL].count() - -print "- checking entities collection (%d entities), this may take a while... " % total - -# The sort() is a way of ensuring that a modified document doesn't enters again at the end of the cursor (we have -# observed that this may happen with large collections, e.g ~50,000 entities). In addition, we have to use -# batch_size so the cursor doesn't expires at server (see http://stackoverflow.com/questions/10298354/mongodb-cursor-id-not-valid-error). -# The used batch_size value is an heuristic -for doc in db[COL].find().sort([('_id.id', 1), ('_id.type', -1), ('_id.servicePath', 1)]).batch_size(100): - - n_processed += 1 - - sys.stdout.write('- processing entity: %d/%d \r' % (n_processed, total) ) - sys.stdout.flush() - - - errors = check_entity_dup(doc) + check_attrs(doc[ATTRS], doc[ATTRNAMES]) - - if len(errors) == 0: - n_ok += 1 - else: - print '- {0}: entity {1} ({2}): metadata ID detected'.format(n_processed, json.dumps(doc['_id']), date2string(doc['modDate'])) - print '\n'.join(errors) - n_not_ok += 1 - -print '- processing entity: %d/%d' % (n_processed, total) -print '- entities analyzed: %d' % n_processed -print ' * entities OK: %d' % n_ok -print ' * entities with consistency problems: %d' % n_not_ok - -""" -def check_dup_id_subfields(): - - for doc in db[COL].find(): - query = {'_id.id': doc['_id']['id']} - - if 'type' in doc['_id']: - query['_id.type'] = doc['_id']['type'] - else: - query['_id.type'] = {'$exists': False} - - if 'servicePath' in doc['_id']: - query['_id.servicePath'] = doc['_id']['servicePath'] - else: - query['_id.servicePath'] = {'$exists': False} - - c = db[COL].find(query).count() - if c > 1: - warn('<%s> has duplicated entities with same _id subfields: %d' % (str(doc['_id']), c)) - -""" diff --git a/scripts/managedb/check_location_coherence.py b/scripts/managedb/check_location_coherence.py deleted file mode 100755 index 41133bfb33..0000000000 --- a/scripts/managedb/check_location_coherence.py +++ /dev/null @@ -1,718 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# Hint: use 'PYTHONIOENCODING=utf8 python check_metadata_id.py' if you are going to redirect the output of this -# script to a file - -# This script requires: -# -# pymongo==3.0.3 -# geojson==2.4.1 - -__author__ = 'fermin' - -import sys -import json -import pymongo -import geojson -from datetime import datetime - - -############################################################## -# BEGIN of the configuration part (don't touch above this line ;) - -uri = 'mongodb://localhost:27017' - -autofix = False - -verbose = False - -# END of the configuration part (don't touch below this line ;) -############################################################## - - -ATTRS = 'attrs' -GEO_TYPES = [ 'geo:point', 'geo:line', 'geo:box', 'geo:polygon', 'geo:json'] - -def flatten(_id): - """ - The way in which Python manage dictionaries doesn't make easy to be sure - of field ordering, which is important for MongoDB in the case of using an - embedded document for _id. This function helps. - - :param _id: JSON document containing id, type and servicePath - :return: a "flatten" version of the _id - """ - - r = {'_id.id': _id['id']} - - if 'type' in _id: - r['_id.type'] = _id['type'] - else: - r['_id.type'] = {'$exists': False} - - if 'servicePath' in _id: - r['_id.servicePath'] = _id['servicePath'] - else: - r['_id.servicePath'] = {'$exists': False} - - return r - -def update_ok(doc, check_attrs): - """ - Check that entity document was updated correctly at DB. - - :param doc: the doc to check - :param check_attrs: list of attributes which existente is checked - """ - - if not ATTRS in doc: - #print "debug1: no attrs" - return False - - for attr in check_attrs: - - if attr not in doc[ATTRS]: - #print "debug2: %s" % attr - return False - - return True - - -def fix_location_geopoint(entity, geo_attr, coords): - """ - Fix location entity (geo:point), adding the missing field. - - :param entity: entity to fix - :param geo_attr: name of the geo:point attribute - :param coords: coordinates of the geo:point attribute - :return: "OK" if update when ok, "FAIL xxxx" otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - # Get coordinates and revert it (GeoJSON uses that) - try: - # Sanity check - if len(coords.split(',')) != 2: - raise ValueError - - coordinates = [float(coords.split(',')[1]), float(coords.split(',')[0])] - except ValueError: - return 'FAIL coordinates parsing error <%s>' % coords - - location = { - 'attrName': geo_attr, - 'coords': { - 'type': 'Point', - 'coordinates': coordinates - } - } - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {'location': location}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - - -def fix_empty_geopoint(entity, geo_attr): - """ - Fix location with empty geo:point, seting it to 0,0 - - :param entity: entity to fix - :param geo_attr: name of the geo:point attribute - :return: "OK" if update when ok, "FAIL xxxx" otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - entity[ATTRS][geo_attr]['value'] = '0,0' - - location = { - 'attrName': geo_attr, - 'coords': { - 'type': 'Point', - 'coordinates': [0, 0] - } - } - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {ATTRS: entity[ATTRS], 'location': location}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - - -def str2number(d): - """ - Returns a vector of parsed numbers (other elements are not touched) - - :param d: the vector to parse - :return: the parsed vector - """ - - if type(d) == list: - return map(str2number, d) - else: - try: - return float(d) - except: # Not a float, leave as it is - return d - - -def fix_location_geojson(entity, geo_attr, geo_json): - """ - Fix location entity (geo:json), adding the missing field. - - :param entity: entity to fix - :param geo_attr: name of the geo:json attribute - :param geo_json: GeoJSON object to use for location - :return: "OK" if update when ok, "FAIL xxxx" otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - if type(geo_json) != dict or not 'type' in geo_json.keys() or not 'coordinates' in geo_json.keys(): - return 'FAIL not a valid GeoJSON: %s' % json.dumps(geo_json) - - # If the geo:json was updated using NGSIv1, numbers would be string and the GeoJSON will not be valid. Thus, - # we first parse strings to number - geo_json_parsed = { - 'type': geo_json['type'], - 'coordinates': str2number(geo_json['coordinates']) - } - - # This is weird... :) - # - # As far as I have checked and at least for this version of geojson module (2.4.1), the geojson.loads(string - # function returns either a geojson object (if string corresponds to a valid GeoJSON) or the string - # passed as argument. In the first case, is_valid is a property of the object to check for True. In the - # second case a AttributeError: 'dict' object has no attribute 'is_valid' error is raised - - try: - if not geojson.loads(json.dumps(geo_json_parsed)).is_valid: - return 'FAIL not a valid GeoJSON: %s' % json.dumps(geo_json_parsed) - except: - return 'FAIL not a valid GeoJSON: %s' % json.dumps(geo_json_parsed) - - location = { - 'attrName': geo_attr, - 'coords': geo_json_parsed - } - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {'location': location}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - -def fix_empty_geojson(entity, geo_attr): - """ - Fix location with empty geo:json, seting it to 0,0 - - :param entity: entity to fix - :param geo_attr: name of the geo:json attribute - :return: "OK" if update when ok, "FAIL xxxx" otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - entity[ATTRS][geo_attr]['value'] = { - 'type': 'Point', - 'coordinates': [0, 0] - } - - location = { - 'attrName': geo_attr, - 'coords': { - 'type': 'Point', - 'coordinates': [0, 0] - } - } - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {ATTRS: entity[ATTRS], 'location': location}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - - - -def add_loc_point_attr(entity, location): - """ - Fix entity adding an attribute that match the location (of type Point) - - :param entity: the entity to fix - :param location: the location field - :return: - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - attr_name = location['attrName'] - - # Get coordinates and revert it (GeoJSON uses that) - coords0 = location['coords']['coordinates'][0] - coords1 = location['coords']['coordinates'][1] - - # Note coordinates are in opposite orden in attribute that in location - attr = { - 'type': 'geo:point', - 'value': "%d, %d" % (coords1, coords0) - } - - entity[ATTRS][attr_name] = attr - - try: - # Update document with the new attribute fields - db[COL].update(flatten(entity['_id']), {'$set': {ATTRS: entity[ATTRS]}}) - - # Check update was ok (this is not an exhaustive checking that is better than nothing :) - check_doc = db[COL].find_one(flatten(entity['_id'])) - - if update_ok(check_doc, entity[ATTRS]): - return 'OK' - else: - return 'FAIL attrs check after update in DB' - - except pymongo.errors.WriteError as e: - return 'FAIL mongo error %d - location was %s' % (e.code, json.dumps(location)) - - - -def extract_geo_attr(attrs): - """ - Given a key-value of attributes, returns the attribute with geo: type or None - if no geo: attribute is found. - - :param attrs: key-value of attribute to process - :return: an attribute name or None - """ - - for attr in attrs.keys(): - if attrs[attr]['type'] in GEO_TYPES: - return attr - - return None - - -def check_ngsiv1_location(attrs, location): - """ - Check if there is an attribute in the entity which match location field in the "NGSIv1 location way", i.e. - attribute type doesn't means anything but the value coordinates format matching the location - - :param attrs: entity attributes object to look - :param location: location field - :return: TRue if check is ok, False otherwise - """ - - # Location example for reference: - # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } } - - # Only Point is allowed in NGSIv1 - if location['coords']['type'] != 'Point': - return False - - loc_attr = None - for attr in attrs.keys(): - if attr == location['attrName']: - loc_attr = attrs[attr] - break - - if loc_attr is None: - return False - - value = loc_attr['value'] - - # Coordinates format check - if len(value.split(',')) != 2: - return False - - # Check coordinates are equal - try: - if float(value.split(',')[0]) != location['coords']['coordinates'][1]: - return False - if float(value.split(',')[1]) != location['coords']['coordinates'][0]: - return False - - except ValueError: - # Format error: some of the token is not a valid float - return False - - return True - - -def date2string(time): - """ - Convert date to string - - :param time: date time (as timestamp) - :return: string representing the date time - """ - - return datetime.fromtimestamp(time).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def entity_dates(entity): - """ - Return a string describing entity creation and modification dates - :param entity: - :return: - """ - cre = '...' - if 'creDate' in entity: - cre = date2string(doc['creDate']) - - mod = '...' - if 'modDate' in entity: - mod = date2string(doc['modDate']) - - return '%s -> %s' % (cre, mod) - - -def safe_add(d, k): - """ - Add or create a key to dictionary if it doesn't already exists. - - :param d: - :param k: - :return: - """ - - if not k in d.keys(): - d[k] = True - - -def msg(m): - """ - Print a message if verbose is enabled - :param m: message to print - :return: - """ - - if verbose: - print m - - -########################## -# Main program starts here - -if len(sys.argv) != 2: - print "invalid number of arguments, use: ./check_location_coherence.py " - sys.exit() - -DB = sys.argv[1] -COL = 'entities' - -# Warn user -if autofix: - print "WARNING!!!! This script modifies your '%s' database. It is STRONGLY RECOMMENDED that you" % DB - print "do a backup of your database before using it as described in https://fiware-orion.readthedocs.io/en/master/admin/database_admin/index.html#backup. Use this script at your own risk." - print "If you are sure you want to continue type 'yes' and press Enter" - - confirm = raw_input() - - if (confirm != 'yes'): - sys.exit() - -client = pymongo.MongoClient(uri) -db = client[DB] - -need_help = False -processed = 0 - -# At the end, processed = no id + id duplicate attrs + id single attrs -counter_analysis = { - 'ngeo-nloc': 0, - 'geo-loc': 0, - 'geopoint-nloc': 0, - 'geojson-nloc': 0, - 'ngeo-loc': 0, - 'legacy': 0, - 'ngeo-locpoint': 0, - 'unknown-geo': 0, - 'corrupted-location': 0, - 'emptygeopoint': 0, - 'emptygeojson': 0, -} -# At the end, processed = untouched + changed + error -counter_update = { - 'untouched': 0, - 'changed': 0, - 'error': 0, -} - -not_fixable_types_found = {} -location_types_found = {} - -total = db[COL].count() - -print "- processing entities collection (%d entities) looking for location field coherence, this may take a while... " % total - -# The sort() is a way of ensuring that a modified document doesn't enters again at the end of the cursor (we have -# observed that this may happen with large collections, e.g ~50,000 entities). In addition, we have to use -# batch_size so the cursor doesn't expires at server (see http://stackoverflow.com/questions/10298354/mongodb-cursor-id-not-valid-error). -# The used batch_size value is an heuristic -for doc in db[COL].find().sort([('_id.id', 1), ('_id.type', -1), ('_id.servicePath', 1)]).batch_size(100): - - processed += 1 - - # Progress meter - #sys.stdout.write('- processing entity: %d/%d \r' % (processed, total) ) - #sys.stdout.flush() - - # FIXME: code should be refactored in per-case function blocks. Currently it is an awful piece of - # nested if-then code... but ok, this is just a quick and dirty script :) - - location = None - geo_attr = extract_geo_attr(doc[ATTRS]) - if 'location' in doc: - location = doc['location'] - - if location is None: - if geo_attr is None: - # Entity without geo: attribute and without location field. It's ok. - counter_analysis['ngeo-nloc'] += 1 - counter_update['untouched'] += 1 - else: # geo_attr is not None - geo_type = doc[ATTRS][geo_attr]['type'] - geo_value = doc[ATTRS][geo_attr]['value'] - - if geo_type == 'geo:point': - - if geo_value == '': - # Entity with empty geo:point is a degenerated case. Fixable, setting 0,0 as coordinates. - counter_analysis['emptygeopoint'] += 1 - if autofix: - result = fix_empty_geopoint(doc, geo_attr) - msg(' - {0}: fixing empty geo:point {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: empty geo:point {1} ({2})'.format(processed, json.dumps(doc['_id']), - entity_dates(doc))) - - counter_update['untouched'] += 1 - need_help = True - else: - # Entity with geo:point attribute but without location field. Fixable - counter_analysis['geopoint-nloc'] += 1 - if autofix: - result = fix_location_geopoint(doc, geo_attr, geo_value) - msg(' - {0}: fixing loc (geo:point) {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: entity w/ geo:point but wo/ location {1} ({2})'.format(processed, json.dumps(doc['_id']), - entity_dates(doc))) - counter_update['untouched'] += 1 - need_help = True - - elif geo_type == 'geo:json': - - if geo_value == '': - # Entity with empty geo:json is a degenerated case. Fixable, setting 0,0 as coordinates. - counter_analysis['emptygeojson'] += 1 - if autofix: - result = fix_empty_geojson(doc, geo_attr) - msg(' - {0}: fixing empty geo:json {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: empty geo:json {1} ({2})'.format(processed, json.dumps(doc['_id']), - entity_dates(doc))) - - counter_update['untouched'] += 1 - need_help = True - else: - # Entity with geo:point attribute but without location field. Fixable - counter_analysis['geojson-nloc'] += 1 - if autofix: - result = fix_location_geojson(doc, geo_attr, geo_value) - msg(' - {0}: fixing loc (geo:json) {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: entity w/ geo:json but wo/ location {1} ({2})'.format(processed, - json.dumps(doc['_id']), - entity_dates(doc))) - counter_update['untouched'] += 1 - need_help = True - - else: - # Entity with geo: attribute different than geo:point or geo:json but without location field. Not fixable - msg(' - {0}: unfixable {1} {2} ({3})'.format(processed, geo_type, json.dumps(doc['_id']), - entity_dates(doc))) - - - safe_add(not_fixable_types_found, geo_type) - - counter_analysis['unknown-geo'] += 1 - counter_update['untouched'] += 1 - need_help = True - - else: # location is not None - try: - loc_type = location['coords']['type'] - loc_attr = location['attrName'] - - if geo_attr is None: - # Entity without geo: attribute location but with location field. They may come from NGSIv1 - # or be a real problem. - if loc_type == 'Point' and check_ngsiv1_location(doc[ATTRS], location): - counter_analysis['legacy'] += 1 - counter_update['untouched'] += 1 - else: - if loc_type == 'Point' and not loc_attr in doc[ATTRS]: - # Location type is Point and attribute doesn't not exist. We can fix adding the corresponding - # attribute - counter_analysis['ngeo-locpoint'] += 1 - if autofix: - result = add_loc_point_attr(doc, location) - msg(' - {0}: fixing loc (Point) {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), result)) - if result == 'OK': - counter_update['changed'] += 1 - else: - counter_update['error'] += 1 - need_help = True - else: - msg(' - {0}: entity w/ location Point but wo/ geo:point or NGSIv1 point {1} ({2})'.format(processed, - json.dumps(doc['_id']), entity_dates(doc))) - counter_update['untouched'] += 1 - need_help = True - else: - # Location type is not point or attribute exits. Not fixable - msg(' - {0}: entity w/ location but not Point or attribute exists {1} ({2}) - location is {3}'.format(processed, - json.dumps(doc['_id']), - entity_dates(doc), - json.dumps(location))) - counter_analysis['ngeo-loc'] += 1 - counter_update['untouched'] += 1 - safe_add(location_types_found, loc_type) - need_help = True - - else: # geo_attr is not None - # Entity with geo: attribute and with location field. It's ok - # FIXME: it may happen that the GeoJSON at location field doesn't correspond with the one calculated - # from geo: attribute. However, we consider that possibility very rare so we are not checking it - counter_analysis['geo-loc'] += 1 - counter_update['untouched'] += 1 - - except KeyError: - # Location format is wrong - counter_analysis['corrupted-location'] += 1 - counter_update['untouched'] += 1 - - msg(' - {0}: entity w/ corrupted location {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']), - entity_dates(doc), - json.dumps(location))) - - need_help = True - -print '- processing entity: %d/%d' % (processed, total) -print '- documents analyzed: %d' % processed -print ' * entities wo/ geo: attr & wo/ loc field (ok) %d' % counter_analysis['ngeo-nloc'] -print ' * entities w/ geo: attr & w/ loc field (ok) %d' % counter_analysis['geo-loc'] -print ' * entities wo/ geo: attr & w/ loc field - coherent (leg NGSIv1) (ok) %d' % counter_analysis['legacy'] -print ' ! entities wo/ geo: attr & w/ loc field - not coherent (fixable) %d' % counter_analysis['ngeo-locpoint'] -print ' ! entities wo/ geo: attr & w/ loc field - not coherent (unfixable) %d' % counter_analysis['ngeo-loc'] -print ' ! entities w/ geo:point attr & wo/ loc field (fixable) %d' % counter_analysis['geopoint-nloc'] -print ' ! entities w/ empty string in geo:point attr (fixable) %d' % counter_analysis['emptygeopoint'] -print ' ! entities w/ empty string in geo:json attr (fixable) %d' % counter_analysis['emptygeojson'] -print ' ! entities w/ geo:json attr and wo/ loc field (fixable) %d' % counter_analysis['geojson-nloc'] -print ' ! entities w/ other geo: attr and wo/ loc field (unfixable) %d' % counter_analysis['unknown-geo'] -print ' ! entities w/ detected corrupted location (unfixable) %d' % counter_analysis['corrupted-location'] - -if len(not_fixable_types_found.keys()) > 0: - print '* geo: types found without associated location field (except geo:point): %s' % ','.join(not_fixable_types_found.keys()) - -if len(location_types_found.keys()) > 0: - print '* loc types found without associated geo: attr not fixable: %s' % ','.join(location_types_found.keys()) - - -print '- documents processed: %d' % processed -print ' * untouched: %d' % counter_update['untouched'] -print ' * changed: %d' % counter_update['changed'] -print ' * attempt to change but error: %d' % counter_update['error'] - -if need_help: - print "------------------------------------------------------" - print "WARNING: some problem was found during the process. Ask for help!" From 784571125c47d59e2683cedbed07d25cb817aa12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 23 Jan 2024 15:17:12 +0100 Subject: [PATCH 105/390] Update scripts/entities_consistency/test_entities_consistency.py --- scripts/entities_consistency/test_entities_consistency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/entities_consistency/test_entities_consistency.py b/scripts/entities_consistency/test_entities_consistency.py index bf2a76ffd4..eb0319c901 100644 --- a/scripts/entities_consistency/test_entities_consistency.py +++ b/scripts/entities_consistency/test_entities_consistency.py @@ -51,5 +51,5 @@ def test_process_db(self): logger = logging.getLogger() # connect to MongoDB and process validation DB - mongo_client = MongoClient('mongodb://localhost:47017') + mongo_client = MongoClient('mongodb://localhost:27017') process_db(logger, 'orion-validation', mongo_client, {}, None) From fb513ec3ef60b7eaac725e7dfb1423a25b5c99a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 23 Jan 2024 15:19:59 +0100 Subject: [PATCH 106/390] FIX fix file compliance --- scripts/check_files_compliance.py | 2 +- scripts/entities_consistency/README.md | 4 ++-- .../entities_consistency/validation_data.js | 24 ++++++++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/scripts/check_files_compliance.py b/scripts/check_files_compliance.py index 0555aee3c1..9ae990c9c8 100755 --- a/scripts/check_files_compliance.py +++ b/scripts/check_files_compliance.py @@ -145,7 +145,7 @@ def ignore(root, file): return True # For several requirements.txt files we have in this repo - if ('acceptance' in root or 'doc' in root) and (file.endswith('.txt') or file.endswith('.json')): + if ('acceptance' in root or 'doc' in root or 'scripts' in 'root') and (file.endswith('.txt') or file.endswith('.json')): return True return False diff --git a/scripts/entities_consistency/README.md b/scripts/entities_consistency/README.md index 0fbbc8a542..e503b9dba5 100644 --- a/scripts/entities_consistency/README.md +++ b/scripts/entities_consistency/README.md @@ -26,7 +26,7 @@ Each entity in DB has an `_id` field with three subfields: * `type` * `servicePath` -## Rule 11: mandatory fields in entity +### Rule 11: mandatory fields in entity The following fields are mandatory: @@ -36,7 +36,7 @@ The following fields are mandatory: It is not an exhaustive check of every field in the database model, but some entities created/updated with old Orion versions may be missing them. -## Rule 12: mandatory fields in attribute +### Rule 12: mandatory fields in attribute The following subfields are mandatory for every attribute: diff --git a/scripts/entities_consistency/validation_data.js b/scripts/entities_consistency/validation_data.js index 6564325294..dde81aa544 100644 --- a/scripts/entities_consistency/validation_data.js +++ b/scripts/entities_consistency/validation_data.js @@ -1,3 +1,25 @@ +/* + # Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U + # + # This file is part of Orion Context Broker. + # + # Orion Context Broker is free software: you can redistribute it and/or + # modify it under the terms of the GNU Affero General Public License as + # published by the Free Software Foundation, either version 3 of the + # License, or (at your option) any later version. + # + # Orion Context Broker is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero + # General Public License for more details. + # + # You should have received a copy of the GNU Affero General Public License + # along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. + # + # For those usages not covered by this license please contact with + # iot_support at tid dot es + */ + db.getSiblingDB("orion-validation").entities.insertMany([ { "_id": { @@ -4183,4 +4205,4 @@ db.getSiblingDB("orion-validation").entities.insertMany([ }, "lastCorrelator": "88e1bc10-b9dd-11ee-8755-080027cd35f1" } -]) \ No newline at end of file +]) From 27e70e847de9a9e1c646d3b9d9d9ccd6766ced22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 23 Jan 2024 15:22:26 +0100 Subject: [PATCH 107/390] FIX check_files_compliance.py --- scripts/check_files_compliance.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/check_files_compliance.py b/scripts/check_files_compliance.py index 9ae990c9c8..d14574cb39 100755 --- a/scripts/check_files_compliance.py +++ b/scripts/check_files_compliance.py @@ -137,7 +137,8 @@ def ignore(root, file): files_names = ['.gitignore', '.valgrindrc', '.valgrindSuppressions', 'LICENSE', 'ContributionPolicy.txt', 'CHANGES_NEXT_RELEASE', 'Changelog', 'compileInfo.h', 'unittests_that_fail_sporadically.txt', 'Vagrantfile', 'contextBroker.ubuntu', - 'mkdocs.yml', 'fiware-ngsiv2-reference.errata', 'ServiceRoutines.txt', '.readthedocs.yml', 'uncrustify.cfg' ] + 'mkdocs.yml', 'fiware-ngsiv2-reference.errata', 'ServiceRoutines.txt', '.readthedocs.yml', 'uncrustify.cfg', + 'requirements.txt'] if file in files_names: return True if 'scripts' in root and (file == 'cpplint.py' or file == 'pdi-pep8.py' or file == 'uncrustify.cfg' \ @@ -145,7 +146,7 @@ def ignore(root, file): return True # For several requirements.txt files we have in this repo - if ('acceptance' in root or 'doc' in root or 'scripts' in 'root') and (file.endswith('.txt') or file.endswith('.json')): + if ('acceptance' in root or 'doc' in root) and (file.endswith('.txt') or file.endswith('.json')): return True return False From 24bb3e6b8f7bcf4d8b5b53ccb0cf5a73ec546095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Jan 2024 09:27:02 +0100 Subject: [PATCH 108/390] Update scripts/entities_consistency/README.md Co-authored-by: mapedraza <40356341+mapedraza@users.noreply.github.com> --- scripts/entities_consistency/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/entities_consistency/README.md b/scripts/entities_consistency/README.md index e503b9dba5..30474120a6 100644 --- a/scripts/entities_consistency/README.md +++ b/scripts/entities_consistency/README.md @@ -89,7 +89,7 @@ Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifie ### Rule 22: entity servicePath syntax -https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path +Check [servicePath documentation](https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path) ### Rule 23: attribute name syntax From 338c13e0a42e523b6ba3ccf355ce0a5fe2747996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Jan 2024 13:13:53 +0100 Subject: [PATCH 109/390] FIX remove old check_legac_location_metadata.py --- .../check_legacy_location_metadata.py | 219 ------------------ 1 file changed, 219 deletions(-) delete mode 100644 scripts/managedb/upgrade-3.5.0/check_legacy_location_metadata.py diff --git a/scripts/managedb/upgrade-3.5.0/check_legacy_location_metadata.py b/scripts/managedb/upgrade-3.5.0/check_legacy_location_metadata.py deleted file mode 100644 index 4c66409db3..0000000000 --- a/scripts/managedb/upgrade-3.5.0/check_legacy_location_metadata.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright 2022 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# Hint: use 'PYTHONIOENCODING=utf8 python check_legacy_location_metadata.py' -# if you are going to redirect the output of this script to a file - - -__author__ = 'fermin' - -from pymongo import MongoClient -import sys - - -def flatten(_id): - """ - The way in which Python manage dictionaries doesn't make easy to be sure - of field ordering, which is important for MongoDB in the case of using an - embedded document for _id. This function helps. - - :param _id: JSON document containing id, type and servicePath - :return: a "flatten" version of the _id - """ - - r = {'_id.id': _id['id']} - - if 'type' in _id: - r['_id.type'] = _id['type'] - else: - r['_id.type'] = {'$exists': False} - - if 'servicePath' in _id: - r['_id.servicePath'] = _id['servicePath'] - else: - r['_id.servicePath'] = {'$exists': False} - - return r - - -def entity_id(doc): - """ - Extracts entity identification keys and returns an object with them - - :param doc: entity document, as taken from DB - :return: {id, type, servicePath} object - """ - id = doc['_id']['id'] - type = doc['_id']['type'] - sp = doc['_id']['servicePath'] - - return {'id': id, 'type': type, 'servicePath': sp} - - -########################## -# Main program starts here - -autofix = False - -if len(sys.argv) != 2: - print "invalid number of arguments, please check https://fiware-orion.readthedocs.io/en/master/admin/upgrading_crossing_3-5-0/index.html" - sys.exit() - -DB = sys.argv[1] - -# Warn user -if autofix: - print "WARNING!!!! This script modifies your '%s' database. It is STRONGLY RECOMMENDED that you" % DB - print "do a backup of your database before using it as described in https://fiware-orion.readthedocs.io/en/master/admin/database_admin/index.html#backup. Use this script at your own risk." - print "If you are sure you want to continue type 'yes' and press Enter" - - confirm = raw_input() - - if confirm != 'yes': - sys.exit() - -uri = 'mongodb://localhost:27017' -client = MongoClient(uri) -db = client[DB] - -need_fix = False -corrupted = False -verbose = True - -# Counters -processed = 0 -counter_location_ngsiv2 = 0 -counter_location_md_wgs84 = 0 -counter_location_md_not_wgs84 = 0 -counter_location_more_than_one = 0 -counter_untouched = 0 -counter_changed = 0 - -corrupted_entities = [] -affected_entities = [] - -total = db['entities'].count() - -print "- processing entities collection (%d entities) looking for attributes with ID metadata, this may take a while... " % total - -# The sort() is a way of ensuring that a modified document doesn't enter again at the end of the cursor (we have -# observed that this may happen with large collections, e.g ~50,000 entities). In addition, we have to use -# batch_size so the cursor doesn't expire at server (see http://stackoverflow.com/questions/10298354/mongodb-cursor-id-not-valid-error). -# The used batch_size value is a heuristic -for entity in db['entities'].find().sort([('_id.id', 1), ('_id.type', -1), ('_id.servicePath', 1)]).batch_size(100): - - processed += 1 - - sys.stdout.write('- processing entity: %d/%d \r' % (processed, total)) - sys.stdout.flush() - - # It may happen that entity doesn't have any attribute. We early detect that situation and skip in that case - if len(entity['attrs'].keys()) == 0: - # print '- %d: entity without attributes %s. Skipping' % (processed, json.dumps(entity['_id'])) - continue # entities loop - - wgs84_attr = None - for attr in entity['attrs']: - if 'md' in entity['attrs'][attr] and 'location' in entity['attrs'][attr]['md']: - if entity['attrs'][attr]['md']['location']['value'] == 'WGS84' \ - or entity['attrs'][attr]['md']['location']['value'] == 'WSG84': - if wgs84_attr is not None: - # more than one location metadata is not allowed due to the checks done by CB - # in processLocationAtEntityCreation() function. However, we check in - # any case as CB could be buggy. Note in this case we don't append to - # affected_entities as that was done first time the location metadata was detected - counter_location_more_than_one += 1 - corrupted = True - corrupted_entities.append(entity_id(entity)) - continue # entities loop - else: - # Note that location metadata doesn't have any semantic in NGSIv2, so we can - # have an entity created with NGSIv2 with location metadata but not location geo-index. - # We need to detect that situation - if "location" in entity: - counter_location_md_wgs84 += 1 - affected_entities.append(entity_id(entity)) - wgs84_attr = attr - else: - counter_location_ngsiv2 += 1 - else: - # location metadata with a value different to WGS84 is not possible taking into - # account the checks done by CB in processLocationAtEntityCreation() function - # However, we check in any case as CB could be buggy - counter_location_md_not_wgs84 += 1 - corrupted = True - corrupted_entities.append(entity_id(entity)) - continue # entities loop - - if wgs84_attr is not None: - if autofix: - # Autofix consist on: - # 1) Remove location metadata (key in 'md' and item in 'mdNames') - # 2) Change attribute type by "geo:point" - - attr = entity['attrs'][wgs84_attr] - attr['mdNames'] = list(filter(lambda x: x != 'location', attr['mdNames'])) - - attr['md'].pop('location', None) - if len(attr['md'].keys()) == 0: - attr.pop('md', None) - - attr['type'] = 'geo:point' - - # it would be easier db['entities'].save(entity) but it seems it has problems when - # _id value is an object and may lead to duplicating entities instead of updating - # note we removed _id from update doc, to avoid possible problems reported by - # pymongo such as "the (immutable) field '_id' was found to have been altered" - query = flatten(entity['_id']) - entity.pop('_id', None) - db['entities'].update(query, entity) - counter_changed += 1 - else: - # Fix should be done by the user - counter_untouched += 1 - need_fix = True - else: - counter_untouched += 1 - -print '- processing entity: %d/%d' % (processed, total) -print '- documents analyzed: %d' % processed -print ' * entities w/ location md w/ WGS84/WSG84 value: %d' % counter_location_md_wgs84 -print ' * entities w/ location md w/o WGS84/WSG84 value (DB corruption!): %d' % counter_location_md_not_wgs84 -print ' * entities w/ more than one location md (DB corruption!): %d' % counter_location_more_than_one -print ' * entities w/ meaningless location md (created by NGSIv2) %d' % counter_location_ngsiv2 -print '- documents processed: %d' % processed -print ' * untouched: %d' % counter_untouched -print ' * changed: %d' % counter_changed - -if verbose: - if len(affected_entities) > 0: - print '- Affected entities:' - for entity in affected_entities: - print ' * ' + str(entity) - if len(corrupted_entities) > 0: - print '- Corrupted entities:' - for entity in corrupted_entities: - print ' * ' + str(entity) - -if need_fix or corrupted: - print "------------------------------------------------------" - print "WARNING: some problem was found during the process. Please check the documentation at https://fiware-orion.readthedocs.io/en/master/admin/upgrading_crossing_3-5-0/index.html" From db965d6773033763b9f2c5c189c1cf1aa8ea74d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Jan 2024 13:17:28 +0100 Subject: [PATCH 110/390] FIX ftest --- .../statistics_with_full_counters.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test b/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test index ae883641d2..403721e557 100644 --- a/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test +++ b/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test @@ -46,7 +46,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1645 +Content-Length: 1624 { "counters": { From b279fcbb72e9001b6efcca3eb2b22696e3492d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Jan 2024 13:18:05 +0100 Subject: [PATCH 111/390] FIX docu --- doc/manuals.jp/admin/statistics.md | 1 - doc/manuals/admin/logs.md | 6 ------ doc/manuals/deprecated.md | 2 +- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/doc/manuals.jp/admin/statistics.md b/doc/manuals.jp/admin/statistics.md index 7d5580f380..2f25673ebb 100644 --- a/doc/manuals.jp/admin/statistics.md +++ b/doc/manuals.jp/admin/statistics.md @@ -44,7 +44,6 @@ Orion Context broker は、`GET /statistics` と `GET /cache/statistics` を介 "counters": { "deprecatedFeatures": { "geoFormat": 2, - "metadataLocation": 1, "ngsiv1Forwarding": 4, "ngsiv1NotifFormat": 4, "ngsiv1Requests": 4 diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index dd24846b57..f88223e73b 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -431,12 +431,6 @@ time=2024-01-11T16:23:24.701Z | lvl=WARN | corr=be709034-b09d-11ee-b5d1-080027cd time=2024-01-11T16:23:24.716Z | lvl=WARN | corr=be7ae5ac-b09d-11ee-98c8-080027cd35f1 | trans=1704990203-652-00000000015 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=Notifier.cpp[680]:buildSenderParams | msg=Deprecated usage of notification legacy format in notification (subId: 65a015fcda947708d30425eb) ``` -* Usages NGSIv1 usages of location metadata. Example: - -``` -time=2023-06-08T15:14:20.999Z | lvl=WARN | corr=24fd2acc-060f-11ee-94cc-000c29583ca5 | trans=1686237259-703-00000000003 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=location.cpp[329]:getGeoJson | msg=Deprecated usage of metadata location coords detected in attribute location at entity update, please use geo:json instead -``` - * Usages of `geo:point`, `geo:line`, `geo:box` or `geo:line`. ``` diff --git a/doc/manuals/deprecated.md b/doc/manuals/deprecated.md index a786106392..d2d5d65f74 100644 --- a/doc/manuals/deprecated.md +++ b/doc/manuals/deprecated.md @@ -110,9 +110,9 @@ The following table provides information about the last Orion version supporting | `attributes` field in `POST /v2/entities` operation | Not yet defined | Not yet defined | | `APPEND`, `UPDATE`, etc. action types in `POST /v2/op/update` | Not yet defined | Not yet defined | | `dateCreated` and `dateModified` in `options` URI parameter | Not yet defined | Not yet defined | -| `location` metadata to specify entity location | Not yet defined | Not yet defined | | `GET /v2` operation | Not yet defined | Not yet defined | | `geo:point`, `geo:line`, `geo:box` and `geo:polygon` attribute types | Not yet defined | Not yet defined | +| `location` metadata to specify entity location | 3.10.1 | June 12th, 2023 | | NGSIv1 API (along with CLI: `-strictNgsiv1Ids` and `-ngsiv1Autocast`) | 3.9.0 (*) | June 2nd, 2023 | | `/ngsi10` and `/ngsi9` URL prefixes | 3.7.0 (*) | May 26th, 2022 | | Initial notification upon subscription creation or update | 3.1.0 | June 9th, 2021 | From 0a593fbb45ad6913a35c5fbf8f71eeb9e709de67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Jan 2024 13:37:08 +0100 Subject: [PATCH 112/390] FIX code comment --- src/lib/mongoBackend/location.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/mongoBackend/location.cpp b/src/lib/mongoBackend/location.cpp index 939f4bbdac..66498b19a3 100644 --- a/src/lib/mongoBackend/location.cpp +++ b/src/lib/mongoBackend/location.cpp @@ -304,7 +304,7 @@ static bool isSpecialGeoJsonType(const ContextAttribute* caP, orion::BSONObjBuil * * It returns true, except in the case of error (in which in addition errDetail gets filled) * -* FIXME PR: try to avoid apiVersion +* FIXME P6: try to avoid apiVersion * * FIXME P6: review the cases in which this function returns false. Maybe many cases (or all them) * can be moved to checkGeoJson() in the parsing layer, as preconditions. @@ -314,7 +314,7 @@ static bool getGeoJson const ContextAttribute* caP, orion::BSONObjBuilder* geoJson, std::string* errDetail, - ApiVersion apiVersion + //ApiVersion apiVersion ) { std::vector coordLat; From 948f22d922cec9f893231c73f5ddc8e03dfe0ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Jan 2024 13:45:57 +0100 Subject: [PATCH 113/390] FIX broken build --- src/lib/mongoBackend/location.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mongoBackend/location.cpp b/src/lib/mongoBackend/location.cpp index 66498b19a3..d6de878cba 100644 --- a/src/lib/mongoBackend/location.cpp +++ b/src/lib/mongoBackend/location.cpp @@ -314,7 +314,7 @@ static bool getGeoJson const ContextAttribute* caP, orion::BSONObjBuilder* geoJson, std::string* errDetail, - //ApiVersion apiVersion + ApiVersion apiVersion ) { std::vector coordLat; From dac655a6b50b4f28bdc9e8f4752464641b3d082d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Jan 2024 15:25:52 +0100 Subject: [PATCH 114/390] FIX deprecated.md --- doc/manuals/deprecated.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/deprecated.md b/doc/manuals/deprecated.md index d2d5d65f74..141ab0d6d5 100644 --- a/doc/manuals/deprecated.md +++ b/doc/manuals/deprecated.md @@ -58,7 +58,7 @@ A list of deprecated features and the version in which they were deprecated foll * `/ngsi10` and `/ngsi9` as URL path prefixes are deprecated in Orion 1.2.0. Please, use `/v1` and `/v1/registry` instead. * `/ngsi9` URL paths removed in Orion 3.8.0 -* `location` metadata to specify entity location is deprecated in Orion 1.1.0. The new way +* `location` metadata to specify entity location is deprecated in Orion 1.1.0 (removed in Orion 3.11.0). The new way of specifying entity location is to use `geo:json` type for the attribute (see details in [the corresponding section of the Orion API specification](orion-api.md#geospatial-properties-of-entities). * Deprecated command line argument in Orion 0.26.1 (removed in Orion 1.0.0). From 70c52b1e282e5e8d7fea86579f0809a1c5c2ad2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Jan 2024 08:35:51 +0100 Subject: [PATCH 115/390] FIX refactor to simplific logic --- src/lib/ngsi/ContextAttributeVector.cpp | 6 ++---- src/lib/ngsi/ContextAttributeVector.h | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/lib/ngsi/ContextAttributeVector.cpp b/src/lib/ngsi/ContextAttributeVector.cpp index 81608abe78..7c24d6064f 100644 --- a/src/lib/ngsi/ContextAttributeVector.cpp +++ b/src/lib/ngsi/ContextAttributeVector.cpp @@ -309,9 +309,7 @@ void ContextAttributeVector::fill ( const orion::BSONObj& attrs, const StringList& attrL, - bool includeEmpty, - const std::string& locAttr, - ApiVersion apiVersion + bool includeEmpty ) { std::set attrNames; @@ -440,7 +438,7 @@ void ContextAttributeVector::fill void ContextAttributeVector::fill(const orion::BSONObj& attrs) { StringList emptyList; - return fill(attrs, emptyList, true, "", V2); + return fill(attrs, emptyList, true); } diff --git a/src/lib/ngsi/ContextAttributeVector.h b/src/lib/ngsi/ContextAttributeVector.h index 05cc96c260..559db1ab3b 100644 --- a/src/lib/ngsi/ContextAttributeVector.h +++ b/src/lib/ngsi/ContextAttributeVector.h @@ -52,7 +52,7 @@ typedef struct ContextAttributeVector unsigned int size(void) const; void release(void); void fill(const ContextAttributeVector& caV, bool useDefaultType = false, bool cloneCompounds = false); - void fill(const orion::BSONObj& attrs, const StringList& attrL, bool includeEmpty, const std::string& locAttr, ApiVersion apiVersion); + void fill(const orion::BSONObj& attrs, const StringList& attrL, bool includeEmpty); void fill(const orion::BSONObj& attrs); int get(const std::string& attributeName) const; From f3fc65146563d7e332d90b1aa27ee469fee8228a Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Thu, 25 Jan 2024 16:56:20 +0900 Subject: [PATCH 116/390] (JP) Remove documentation about location medadata in NGSIv1 (#4048) --- doc/manuals.jp/admin/logs.md | 6 ------ doc/manuals.jp/deprecated.md | 3 ++- doc/manuals.jp/orion-api.md | 1 - 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/doc/manuals.jp/admin/logs.md b/doc/manuals.jp/admin/logs.md index 140e29de3a..2023f5adbc 100644 --- a/doc/manuals.jp/admin/logs.md +++ b/doc/manuals.jp/admin/logs.md @@ -355,12 +355,6 @@ time=2023-05-25T14:27:45.958Z | lvl=WARN | corr=513bd10e-fb08-11ed-8ad7-000c2958 time=2023-05-25T14:27:46.041Z | lvl=WARN | corr=51490536-fb08-11ed-9782-000c29583ca5 | trans=1685024865-125-00000000002 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[114]:logInfoRequestWithoutPayload | msg=Deprecated NGSIv1 request received: GET /v1/contextEntities/E, response code: 200 ``` -* 位置メタデータの NGSIv1 での使用。例： - -``` -time=2023-06-08T15:14:20.999Z | lvl=WARN | corr=24fd2acc-060f-11ee-94cc-000c29583ca5 | trans=1686237259-703-00000000003 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=location.cpp[329]:getGeoJson | msg=Deprecated usage of metadata location coords detected in attribute location at entity update, please use geo:json instead -``` - * `geo:point`, `geo:line`, `geo:box` また `geo:line` の使用 ``` diff --git a/doc/manuals.jp/deprecated.md b/doc/manuals.jp/deprecated.md index aa0004c89b..3b81eacd34 100644 --- a/doc/manuals.jp/deprecated.md +++ b/doc/manuals.jp/deprecated.md @@ -40,7 +40,7 @@ * Orion 1.5.0 では、NGSIv2 で `dateCreated` および/または `dateModified` 属性を含めるために `optionsURL` パラメータ使用することは推奨されていません。代わりに `attrs`URI パラメータを使用してください * パス・プレフィックスとして /ngsi10 そして /ngsi9URL は、orion 1.2.0 で廃止されました。代わりに `/v1` と `/v1/registry` を使用してください * `/ngsi9` URL パスは Orion 3.8.0 で削除されました -* エンティティの場所を指定する `location` メタデータは、Orion 1.1.0 では非推奨です。エンティティの場所を指定する新しい方法は、属性の `geo:json` 型を使用することです。[Orion API の対応するセクション](orion-api.md#geospatial-properties-of-entities)を参照してください) +* エンティティの場所を指定する `location` メタデータは、Orion 1.1.0 では非推奨です (Orion 3.11.0 で削除されました)。エンティティの場所を指定する新しい方法は、属性の `geo:json` 型を使用することです。[Orion API の対応するセクション](orion-api.md#geospatial-properties-of-entities)を参照してください) * Orion 0.26.1 のコマンドライン引数は廃止されました。Orion 1.0.0 で削除されました * **--silent** : エラー以外のすべてのログ出力を抑止します。代わりに `-logLevel ERROR` を使用してください * ONTIMEINTERVAL サブスクリプションは Orion 0.26.0 以降で廃止されました。Orion 1.0.0 では削除されました。ONTIMEINTERVAL サブスクリプションにはいくつかの問題があります。CB に状態 (state) を導入するため、水平スケーリングの設定をより困難にし、ページネーション/フィルタリングの導入を困難にします。実際には、ONTIMEINTERVAL 通知に基づくユース・ケースは、レセプタが queryContext を同じ頻度で実行する等価なユース・ケースに変換できるため、実際には必要ありません。ページ区切りやフィルタリングなどの queryContext の機能を利用してください @@ -83,6 +83,7 @@ | エンティティのロケーションを指定する `location` メタデータ | まだ定義されていません | まだ定義されていません | | `GET /v2` 操作 | まだ定義されていません | まだ定義されていません | | `geo:point`, `geo:line`, `geo:box` および `geo:polygon` 属性タイプ | まだ定義されていません | まだ定義されていません | +| エンティティの場所を指定するための `location` メタデータ | 3.10.1 | 2023年6月12日 | | NGSIv1 (関連する CLI パラメータ : `-strictNgsiv1Ids`, `-ngsiv1Autocast`) | 3.9.0 (*) | 2023年6月2日 | | `/ngsi10` および `/ngsi9` URL プレフィックス | 3.7.0 (*) | 2022年5月26日 | |サブスクリプションの作成または更新時の初期通知 | 3.1.0 | 2021年6月9日 | diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index 9296c2644c..c7684fec8c 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -703,7 +703,6 @@ Orion は階層スコープをサポートしているため、エンティテ - `actionType` (型: `Text`): 通知のみ。添付されている属性が、通知をトリガーしたリクエストに含まれていた場合に 含まれます。その値は、リクエスト・オペレーションのタイプによって異なります。更新の場合は `update`、作成の場合は `append`、削除の場合は `delete` です。その型は常に `Text` です -- 現在、[非推奨](#deprecated.md) ですが、まだサポートされています 通常のメタデータと同様、`mq` フィルタでも使用できます。ただし、リソース URLs では使用できません。 From c67d7ad72482cd66036debddb7b60d194ad16546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Jan 2024 09:56:09 +0100 Subject: [PATCH 117/390] FIX simplification --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 4 ++-- src/lib/mongoBackend/MongoGlobal.cpp | 4 ++-- src/lib/mongoBackend/MongoGlobal.h | 2 +- src/lib/mongoBackend/mongoQueryContext.cpp | 2 +- src/lib/ngsi/ContextAttributeVector.cpp | 8 ++++---- src/lib/ngsi/ContextAttributeVector.h | 2 +- src/lib/ngsi/ContextElementResponse.cpp | 20 ++++++++++---------- src/lib/ngsi/ContextElementResponse.h | 5 ++--- 8 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 8593d6e084..bdaa38140c 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -3504,7 +3504,7 @@ static unsigned int updateEntity /* Build CER used for notifying (if needed) */ StringList emptyAttrL; - ContextElementResponse* notifyCerP = new ContextElementResponse(r, emptyAttrL, true, apiVersion); + ContextElementResponse* notifyCerP = new ContextElementResponse(r, emptyAttrL/*, true*/); // The hasField() check is needed as the entity could have been created with very old Orion version not // supporting modification/creation dates @@ -3921,7 +3921,7 @@ static unsigned int updateEntity notifyCerP->release(); delete notifyCerP; - notifyCerP = new ContextElementResponse(getObjectFieldF(reply, "value"), emptyAttrL, true, apiVersion); + notifyCerP = new ContextElementResponse(getObjectFieldF(reply, "value"), emptyAttrL/*, true*/); } } else diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp index 9b0cca0054..01d3fcabb6 100644 --- a/src/lib/mongoBackend/MongoGlobal.cpp +++ b/src/lib/mongoBackend/MongoGlobal.cpp @@ -1398,7 +1398,7 @@ bool entitiesQuery const Restriction& res, ContextElementResponseVector* cerV, std::string* err, - bool includeEmpty, + //bool includeEmpty, const std::string& tenant, const std::vector& servicePath, int offset, @@ -1636,7 +1636,7 @@ bool entitiesQuery // Build CER from BSON retrieved from DB docs++; LM_T(LmtMongo, ("retrieved document [%d]: '%s'", docs, r.toString().c_str())); - ContextElementResponse* cer = new ContextElementResponse(r, attrL, includeEmpty, apiVersion); + ContextElementResponse* cer = new ContextElementResponse(r, attrL/*, includeEmpty*/); // Add builtin attributes and metadata (only in NGSIv2) if (apiVersion == V2) diff --git a/src/lib/mongoBackend/MongoGlobal.h b/src/lib/mongoBackend/MongoGlobal.h index 9b750e01b5..921d86aa44 100644 --- a/src/lib/mongoBackend/MongoGlobal.h +++ b/src/lib/mongoBackend/MongoGlobal.h @@ -230,7 +230,7 @@ extern bool entitiesQuery const Restriction& res, ContextElementResponseVector* cerV, std::string* err, - bool includeEmpty, + //bool includeEmpty, const std::string& tenant, const std::vector& servicePath, int offset = DEFAULT_PAGINATION_OFFSET_INT, diff --git a/src/lib/mongoBackend/mongoQueryContext.cpp b/src/lib/mongoBackend/mongoQueryContext.cpp index fd8f1799b5..4cfb8c1331 100644 --- a/src/lib/mongoBackend/mongoQueryContext.cpp +++ b/src/lib/mongoBackend/mongoQueryContext.cpp @@ -362,7 +362,7 @@ HttpStatusCode mongoQueryContext requestP->restriction, &rawCerV, &err, - true, + //true, tenant, servicePathV, offset, diff --git a/src/lib/ngsi/ContextAttributeVector.cpp b/src/lib/ngsi/ContextAttributeVector.cpp index 7c24d6064f..e47db81393 100644 --- a/src/lib/ngsi/ContextAttributeVector.cpp +++ b/src/lib/ngsi/ContextAttributeVector.cpp @@ -308,8 +308,8 @@ void ContextAttributeVector::fill(const ContextAttributeVector& caV, bool useDef void ContextAttributeVector::fill ( const orion::BSONObj& attrs, - const StringList& attrL, - bool includeEmpty + const StringList& attrL/*, + bool includeEmpty*/ ) { std::set attrNames; @@ -344,7 +344,7 @@ void ContextAttributeVector::fill { case orion::String: ca.stringValue = getStringFieldF(attr, ENT_ATTRS_VALUE); - if (!includeEmpty && ca.stringValue.empty()) + if (!true && ca.stringValue.empty()) { continue; } @@ -438,7 +438,7 @@ void ContextAttributeVector::fill void ContextAttributeVector::fill(const orion::BSONObj& attrs) { StringList emptyList; - return fill(attrs, emptyList, true); + return fill(attrs, emptyList/*, true*/); } diff --git a/src/lib/ngsi/ContextAttributeVector.h b/src/lib/ngsi/ContextAttributeVector.h index 559db1ab3b..338d7be427 100644 --- a/src/lib/ngsi/ContextAttributeVector.h +++ b/src/lib/ngsi/ContextAttributeVector.h @@ -52,7 +52,7 @@ typedef struct ContextAttributeVector unsigned int size(void) const; void release(void); void fill(const ContextAttributeVector& caV, bool useDefaultType = false, bool cloneCompounds = false); - void fill(const orion::BSONObj& attrs, const StringList& attrL, bool includeEmpty); + void fill(const orion::BSONObj& attrs, const StringList& attrL/*, bool includeEmpty*/); void fill(const orion::BSONObj& attrs); int get(const std::string& attributeName) const; diff --git a/src/lib/ngsi/ContextElementResponse.cpp b/src/lib/ngsi/ContextElementResponse.cpp index 9d57aa0867..ecffd2b76c 100644 --- a/src/lib/ngsi/ContextElementResponse.cpp +++ b/src/lib/ngsi/ContextElementResponse.cpp @@ -98,9 +98,8 @@ ContextElementResponse::ContextElementResponse(ContextElementResponse* cerP, boo ContextElementResponse::ContextElementResponse ( const orion::BSONObj& entityDoc, - const StringList& attrL, - bool includeEmpty, - ApiVersion apiVersion + const StringList& attrL/*, + bool includeEmpty*/ ) { prune = false; @@ -125,7 +124,8 @@ ContextElementResponse::ContextElementResponse // // Attribute vector // - orion::BSONObj attrs = getObjectFieldF(entityDoc, ENT_ATTRS); + entity.attributeVector.fill(getObjectFieldF(entityDoc, ENT_ATTRS), attrL/*, includeEmpty*/); + /*orion::BSONObj attrs = getObjectFieldF(entityDoc, ENT_ATTRS); std::set attrNames; attrs.getFieldNames(&attrNames); @@ -146,8 +146,8 @@ ContextElementResponse::ContextElementResponse continue; } - /* It could happen (although very rarely) that the value field is missing in the - * DB for the attribute. The following is a safety check measure to protect against that */ + // It could happen (although very rarely) that the value field is missing in the + // DB for the attribute. The following is a safety check measure to protect against that if (!attr.hasField(ENT_ATTRS_VALUE)) { caP = new ContextAttribute(ca.name, ca.type, ""); @@ -204,13 +204,13 @@ ContextElementResponse::ContextElementResponse } } - /* dateExpires is managed like a regular attribute in DB, but it is a builtin and it is shadowed */ + // dateExpires is managed like a regular attribute in DB, but it is a builtin and it is shadowed if (caP->name == DATE_EXPIRES) { caP->shadowed = true; } - /* Setting custom metadata (if any) */ + // Setting custom metadata (if any) if (attr.hasField(ENT_ATTRS_MD)) { orion::BSONObj mds = getObjectFieldF(attr, ENT_ATTRS_MD); @@ -225,7 +225,7 @@ ContextElementResponse::ContextElementResponse } } - /* Set creDate and modDate at attribute level */ + // Set creDate and modDate at attribute level if (attr.hasField(ENT_ATTRS_CREATION_DATE)) { caP->creDate = getNumberFieldF(attr, ENT_ATTRS_CREATION_DATE); @@ -237,7 +237,7 @@ ContextElementResponse::ContextElementResponse } entity.attributeVector.push_back(caP); - } + }*/ /* Set creDate and modDate at entity level */ if (entityDoc.hasField(ENT_CREATION_DATE)) diff --git a/src/lib/ngsi/ContextElementResponse.h b/src/lib/ngsi/ContextElementResponse.h index 999dd50a48..b86fe53e92 100644 --- a/src/lib/ngsi/ContextElementResponse.h +++ b/src/lib/ngsi/ContextElementResponse.h @@ -61,9 +61,8 @@ typedef struct ContextElementResponse ContextElementResponse(EntityId* eP, ContextAttribute* aP); ContextElementResponse(ContextElementResponse* cerP, bool cloneCompound = false); ContextElementResponse(const orion::BSONObj& entityDoc, - const StringList& attrL, - bool includeEmpty = true, - ApiVersion apiVersion = V1); + const StringList& attrL/*, + bool includeEmpty = true*/); ContextElementResponse(Entity* eP, bool useDefaultType = false); std::string toJsonV1(bool asJsonObject, From 670495a1eeac27044c98ad74af43fc7b57738896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Jan 2024 10:26:25 +0100 Subject: [PATCH 118/390] FIX cleanup --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 4 ++-- src/lib/mongoBackend/MongoGlobal.cpp | 8 +------- src/lib/mongoBackend/MongoGlobal.h | 1 - src/lib/ngsi/ContextAttributeVector.cpp | 7 +------ src/lib/ngsi/ContextAttributeVector.h | 2 +- src/lib/ngsi/ContextElementResponse.cpp | 5 ++--- src/lib/ngsi/ContextElementResponse.h | 3 +-- 7 files changed, 8 insertions(+), 22 deletions(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index bdaa38140c..03617d9d50 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -3504,7 +3504,7 @@ static unsigned int updateEntity /* Build CER used for notifying (if needed) */ StringList emptyAttrL; - ContextElementResponse* notifyCerP = new ContextElementResponse(r, emptyAttrL/*, true*/); + ContextElementResponse* notifyCerP = new ContextElementResponse(r, emptyAttrL); // The hasField() check is needed as the entity could have been created with very old Orion version not // supporting modification/creation dates @@ -3921,7 +3921,7 @@ static unsigned int updateEntity notifyCerP->release(); delete notifyCerP; - notifyCerP = new ContextElementResponse(getObjectFieldF(reply, "value"), emptyAttrL/*, true*/); + notifyCerP = new ContextElementResponse(getObjectFieldF(reply, "value"), emptyAttrL); } } else diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp index 01d3fcabb6..85308f228b 100644 --- a/src/lib/mongoBackend/MongoGlobal.cpp +++ b/src/lib/mongoBackend/MongoGlobal.cpp @@ -1385,11 +1385,6 @@ static bool isCustomAttr(std::string attrName) * This method is used by queryContext. It takes a vector with entities and a vector * with attributes as input and returns the corresponding ContextElementResponseVector or error. * -* Note the includeEmpty argument. This is used if we don't want the result to include empty -* attributes, i.e. the ones that cause ''. This is aimed at -* subscribeContext case, as empty values can cause problems in the case of federating Context -* Brokers (the notifyContext is processed as an updateContext and in the latter case, an -* empty value causes an error) */ bool entitiesQuery ( @@ -1398,7 +1393,6 @@ bool entitiesQuery const Restriction& res, ContextElementResponseVector* cerV, std::string* err, - //bool includeEmpty, const std::string& tenant, const std::vector& servicePath, int offset, @@ -1636,7 +1630,7 @@ bool entitiesQuery // Build CER from BSON retrieved from DB docs++; LM_T(LmtMongo, ("retrieved document [%d]: '%s'", docs, r.toString().c_str())); - ContextElementResponse* cer = new ContextElementResponse(r, attrL/*, includeEmpty*/); + ContextElementResponse* cer = new ContextElementResponse(r, attrL); // Add builtin attributes and metadata (only in NGSIv2) if (apiVersion == V2) diff --git a/src/lib/mongoBackend/MongoGlobal.h b/src/lib/mongoBackend/MongoGlobal.h index 921d86aa44..caa6058e8f 100644 --- a/src/lib/mongoBackend/MongoGlobal.h +++ b/src/lib/mongoBackend/MongoGlobal.h @@ -230,7 +230,6 @@ extern bool entitiesQuery const Restriction& res, ContextElementResponseVector* cerV, std::string* err, - //bool includeEmpty, const std::string& tenant, const std::vector& servicePath, int offset = DEFAULT_PAGINATION_OFFSET_INT, diff --git a/src/lib/ngsi/ContextAttributeVector.cpp b/src/lib/ngsi/ContextAttributeVector.cpp index e47db81393..e1214bb8a1 100644 --- a/src/lib/ngsi/ContextAttributeVector.cpp +++ b/src/lib/ngsi/ContextAttributeVector.cpp @@ -308,8 +308,7 @@ void ContextAttributeVector::fill(const ContextAttributeVector& caV, bool useDef void ContextAttributeVector::fill ( const orion::BSONObj& attrs, - const StringList& attrL/*, - bool includeEmpty*/ + const StringList& attrL ) { std::set attrNames; @@ -344,10 +343,6 @@ void ContextAttributeVector::fill { case orion::String: ca.stringValue = getStringFieldF(attr, ENT_ATTRS_VALUE); - if (!true && ca.stringValue.empty()) - { - continue; - } caP = new ContextAttribute(ca.name, ca.type, ca.stringValue); break; diff --git a/src/lib/ngsi/ContextAttributeVector.h b/src/lib/ngsi/ContextAttributeVector.h index 338d7be427..6c32b17a4f 100644 --- a/src/lib/ngsi/ContextAttributeVector.h +++ b/src/lib/ngsi/ContextAttributeVector.h @@ -52,7 +52,7 @@ typedef struct ContextAttributeVector unsigned int size(void) const; void release(void); void fill(const ContextAttributeVector& caV, bool useDefaultType = false, bool cloneCompounds = false); - void fill(const orion::BSONObj& attrs, const StringList& attrL/*, bool includeEmpty*/); + void fill(const orion::BSONObj& attrs, const StringList& attrL); void fill(const orion::BSONObj& attrs); int get(const std::string& attributeName) const; diff --git a/src/lib/ngsi/ContextElementResponse.cpp b/src/lib/ngsi/ContextElementResponse.cpp index ecffd2b76c..8da3c10aa0 100644 --- a/src/lib/ngsi/ContextElementResponse.cpp +++ b/src/lib/ngsi/ContextElementResponse.cpp @@ -98,8 +98,7 @@ ContextElementResponse::ContextElementResponse(ContextElementResponse* cerP, boo ContextElementResponse::ContextElementResponse ( const orion::BSONObj& entityDoc, - const StringList& attrL/*, - bool includeEmpty*/ + const StringList& attrL ) { prune = false; @@ -124,7 +123,7 @@ ContextElementResponse::ContextElementResponse // // Attribute vector // - entity.attributeVector.fill(getObjectFieldF(entityDoc, ENT_ATTRS), attrL/*, includeEmpty*/); + entity.attributeVector.fill(getObjectFieldF(entityDoc, ENT_ATTRS), attrL); /*orion::BSONObj attrs = getObjectFieldF(entityDoc, ENT_ATTRS); std::set attrNames; diff --git a/src/lib/ngsi/ContextElementResponse.h b/src/lib/ngsi/ContextElementResponse.h index b86fe53e92..87b3bc17b1 100644 --- a/src/lib/ngsi/ContextElementResponse.h +++ b/src/lib/ngsi/ContextElementResponse.h @@ -61,8 +61,7 @@ typedef struct ContextElementResponse ContextElementResponse(EntityId* eP, ContextAttribute* aP); ContextElementResponse(ContextElementResponse* cerP, bool cloneCompound = false); ContextElementResponse(const orion::BSONObj& entityDoc, - const StringList& attrL/*, - bool includeEmpty = true*/); + const StringList& attrL); ContextElementResponse(Entity* eP, bool useDefaultType = false); std::string toJsonV1(bool asJsonObject, From e64bcdbe53e8e2e8d380183951c9084bb0ce0018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Jan 2024 11:10:16 +0100 Subject: [PATCH 119/390] FIX simplification and cleanup --- src/lib/mongoBackend/mongoQueryContext.cpp | 1 - src/lib/ngsi/ContextAttributeVector.cpp | 2 +- src/lib/ngsi/ContextElementResponse.cpp | 113 --------------------- 3 files changed, 1 insertion(+), 115 deletions(-) diff --git a/src/lib/mongoBackend/mongoQueryContext.cpp b/src/lib/mongoBackend/mongoQueryContext.cpp index 4cfb8c1331..32b51170f2 100644 --- a/src/lib/mongoBackend/mongoQueryContext.cpp +++ b/src/lib/mongoBackend/mongoQueryContext.cpp @@ -362,7 +362,6 @@ HttpStatusCode mongoQueryContext requestP->restriction, &rawCerV, &err, - //true, tenant, servicePathV, offset, diff --git a/src/lib/ngsi/ContextAttributeVector.cpp b/src/lib/ngsi/ContextAttributeVector.cpp index e1214bb8a1..5a11b4d89e 100644 --- a/src/lib/ngsi/ContextAttributeVector.cpp +++ b/src/lib/ngsi/ContextAttributeVector.cpp @@ -433,7 +433,7 @@ void ContextAttributeVector::fill void ContextAttributeVector::fill(const orion::BSONObj& attrs) { StringList emptyList; - return fill(attrs, emptyList/*, true*/); + return fill(attrs, emptyList); } diff --git a/src/lib/ngsi/ContextElementResponse.cpp b/src/lib/ngsi/ContextElementResponse.cpp index 8da3c10aa0..ba9d3ae0b2 100644 --- a/src/lib/ngsi/ContextElementResponse.cpp +++ b/src/lib/ngsi/ContextElementResponse.cpp @@ -124,119 +124,6 @@ ContextElementResponse::ContextElementResponse // Attribute vector // entity.attributeVector.fill(getObjectFieldF(entityDoc, ENT_ATTRS), attrL); - /*orion::BSONObj attrs = getObjectFieldF(entityDoc, ENT_ATTRS); - std::set attrNames; - - attrs.getFieldNames(&attrNames); - for (std::set::iterator i = attrNames.begin(); i != attrNames.end(); ++i) - { - std::string attrName = *i; - orion::BSONObj attr = getObjectFieldF(attrs, attrName); - ContextAttribute* caP = NULL; - ContextAttribute ca; - - // Name and type - ca.name = dbDecode(attrName); - ca.type = getStringFieldF(attr, ENT_ATTRS_TYPE); - - // Skip attribute if the attribute is in the list (or attrL is empty or includes "*") - if (!includedAttribute(ca.name, attrL)) - { - continue; - } - - // It could happen (although very rarely) that the value field is missing in the - // DB for the attribute. The following is a safety check measure to protect against that - if (!attr.hasField(ENT_ATTRS_VALUE)) - { - caP = new ContextAttribute(ca.name, ca.type, ""); - } - else - { - switch(getFieldF(attr, ENT_ATTRS_VALUE).type()) - { - case orion::String: - ca.stringValue = getStringFieldF(attr, ENT_ATTRS_VALUE); - if (!includeEmpty && ca.stringValue.empty()) - { - continue; - } - caP = new ContextAttribute(ca.name, ca.type, ca.stringValue); - break; - - case orion::NumberDouble: - ca.numberValue = getNumberFieldF(attr, ENT_ATTRS_VALUE); - caP = new ContextAttribute(ca.name, ca.type, ca.numberValue); - break; - - case orion::NumberInt: - ca.numberValue = (double) getIntFieldF(attr, ENT_ATTRS_VALUE); - caP = new ContextAttribute(ca.name, ca.type, ca.numberValue); - break; - - case orion::Bool: - ca.boolValue = getBoolFieldF(attr, ENT_ATTRS_VALUE); - caP = new ContextAttribute(ca.name, ca.type, ca.boolValue); - break; - - case orion::jstNULL: - caP = new ContextAttribute(ca.name, ca.type, ""); - caP->valueType = orion::ValueTypeNull; - break; - - case orion::Object: - caP = new ContextAttribute(ca.name, ca.type, ""); - caP->compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); - caP->valueType = orion::ValueTypeObject; - compoundObjectResponse(caP->compoundValueP, getFieldF(attr, ENT_ATTRS_VALUE)); - break; - - case orion::Array: - caP = new ContextAttribute(ca.name, ca.type, ""); - caP->compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); - caP->valueType = orion::ValueTypeVector; - compoundVectorResponse(caP->compoundValueP, getFieldF(attr, ENT_ATTRS_VALUE)); - break; - - default: - LM_E(("Runtime Error (unknown attribute value type in DB: %d)", getFieldF(attr, ENT_ATTRS_VALUE).type())); - } - } - - // dateExpires is managed like a regular attribute in DB, but it is a builtin and it is shadowed - if (caP->name == DATE_EXPIRES) - { - caP->shadowed = true; - } - - // Setting custom metadata (if any) - if (attr.hasField(ENT_ATTRS_MD)) - { - orion::BSONObj mds = getObjectFieldF(attr, ENT_ATTRS_MD); - std::set mdsSet; - - mds.getFieldNames(&mdsSet); - for (std::set::iterator i = mdsSet.begin(); i != mdsSet.end(); ++i) - { - std::string currentMd = *i; - Metadata* md = new Metadata(dbDecode(currentMd), getObjectFieldF(mds, currentMd)); - caP->metadataVector.push_back(md); - } - } - - // Set creDate and modDate at attribute level - if (attr.hasField(ENT_ATTRS_CREATION_DATE)) - { - caP->creDate = getNumberFieldF(attr, ENT_ATTRS_CREATION_DATE); - } - - if (attr.hasField(ENT_ATTRS_MODIFICATION_DATE)) - { - caP->modDate = getNumberFieldF(attr, ENT_ATTRS_MODIFICATION_DATE); - } - - entity.attributeVector.push_back(caP); - }*/ /* Set creDate and modDate at entity level */ if (entityDoc.hasField(ENT_CREATION_DATE)) From 59bdb402fc735d3b7167180e0c0b016d733a3a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Jan 2024 11:56:19 +0100 Subject: [PATCH 120/390] FIX phony modification --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 70ad7e6957..f6c19777d7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Orion Context Broker + # Orion Context Broker From 6cd1fd7d397899663658f0716e56554882b9c7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Jan 2024 08:54:56 +0100 Subject: [PATCH 121/390] Revert "FIX phony modification" This reverts commit 59bdb402fc735d3b7167180e0c0b016d733a3a27. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f6c19777d7..70ad7e6957 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - # Orion Context Broker +# Orion Context Broker From 340b4eeb35f954e3e5c00e52a7594f6c14b9a96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Jan 2024 08:55:21 +0100 Subject: [PATCH 122/390] FIX initialize variable --- src/lib/apiTypesV2/Subscription.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/apiTypesV2/Subscription.h b/src/lib/apiTypesV2/Subscription.h index 705721618b..1212226096 100644 --- a/src/lib/apiTypesV2/Subscription.h +++ b/src/lib/apiTypesV2/Subscription.h @@ -149,8 +149,9 @@ struct Subject * * Subscription - */ -struct Subscription +class Subscription { +public: std::string id; std::string description; bool descriptionProvided; @@ -164,6 +165,10 @@ struct Subscription std::string toJson(); void release(); + Subscription(): + attrsFormat(NGSI_V2_NORMALIZED) + {} + ~Subscription(); }; From 3b41718fc37f9312cc2bb8c070fbc6ba7f30cacf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Jan 2024 14:57:47 +0100 Subject: [PATCH 123/390] FIX impreove entities consisency script --- .../entities_consistency.py | 123 ++++++--- .../test_entities_consistency.py | 2 +- .../entities_consistency/validation_data.js | 245 ++++++++++++++++++ 3 files changed, 333 insertions(+), 37 deletions(-) diff --git a/scripts/entities_consistency/entities_consistency.py b/scripts/entities_consistency/entities_consistency.py index fd375cbccc..4da9541710 100644 --- a/scripts/entities_consistency/entities_consistency.py +++ b/scripts/entities_consistency/entities_consistency.py @@ -25,6 +25,7 @@ from pymongo import MongoClient from deepdiff import DeepDiff +from datetime import datetime import argparse import logging import json @@ -34,6 +35,7 @@ # Helper functions + def is_geo_type(attr_type): """ Return True if attr type passed as argument is a geo type @@ -52,11 +54,23 @@ def ignore_type(attr): return 'md' in attr and 'ignoreType' in attr['md'] +def is_geo_attr(attr): + """ + Check if a given attr is of geo type, i.e. following conditions are true: + * It has a geo type (geo:json, et.c9 + * It doesn't use the ignoreType metadata + * Its value is not null + """ + return is_geo_type(attr['type']) and not ignore_type(attr) and attr['value'] is not None + + def to_geo_json(attr): """ Return the GeoJSON corresponding to an attribute location, taking into account the type Useful ref: https://github.com/telefonicaid/fiware-orion/blob/3.9.0/doc/manuals/orion-api.md + + :return: either a dict (containing a GeoJSON) or a string (in the case of error) """ if attr['type'] == 'geo:point': @@ -116,10 +130,29 @@ def to_geo_json(attr): 'coordinates': [coordinates] } elif attr['type'] == 'geo:json': - return attr['value'] + # Feature is a special type, see https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#geojson + if attr['value']['type'] == 'Feature': + if 'geometry' in attr['value']: + return attr['value']['geometry'] + else: + return "geo:json Feature does not have geometry element" + # FeatureCollection is a special type, see https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#geojson + if attr['value']['type'] == 'FeatureCollection': + if 'features' not in attr['value']: + return "geo:json FeatureCollection does not have features element" + if len(attr['value']['features']) == 0: + return "geo:json FeatureCollection features has no elements" + if len(attr['value']['features']) > 1: + return "geo:json FeatureCollection features has more than one element" + if 'geometry' in attr['value']['features'][0]: + return attr['value']['features'][0]['geometry'] + else: + return "geo:json FeatureCollection feature does not have geometry element" + else: + return attr['value'] + else: - logger.error(f"unknown geo location type: {attr['type']}") - return None + return f"unknown geo location type: {attr['type']}" def convert_strings_to_numbers(data): @@ -351,7 +384,7 @@ def rule16(entity): geo_attrs = [] for attr in entity['attrs']: # type existence in attribute is checked by another rule - if 'type' in entity['attrs'][attr] and is_geo_type(entity['attrs'][attr]['type']) and not ignore_type(entity['attrs'][attr]): + if 'type' in entity['attrs'][attr] and is_geo_attr(entity['attrs'][attr]): geo_attrs.append(attr) if len(geo_attrs) > 1: @@ -361,31 +394,28 @@ def rule16(entity): # If geo attr found, then check that there is consistent location field geo_attr = geo_attrs[0] geo_type = entity['attrs'][geo_attr]['type'] - if entity['attrs'][geo_attr]['value'] is None: - # if null value in geolocation attribute, then location field must not be present - if 'location' in entity: - return f"geo location '{geo_attr}' ({geo_type}) with null value and location field found" - else: - # not null value in geo location attribute case - if 'location' not in entity: - return f"geo location '{geo_attr}' ({geo_type}) not null but location field not found in entity" - if entity['location']['attrName'] != geo_attr: - return f"location.attrName ({entity['location']['attrName']}) differs from '{geo_attr}'" - - geo_json = to_geo_json(entity['attrs'][geo_attr]) - - # https://www.testcult.com/deep-comparison-of-json-in-python/ - diff = DeepDiff(geo_json, entity['location']['coords'], ignore_order=True) - if diff: - # A typical difference is that attribute value uses strings and location uses numbers - # (this happens when the location was created/updated using NGSIv1). We try to identify that case - geo_json = convert_strings_to_numbers(geo_json) - if not DeepDiff(geo_json, entity['location']['coords'], ignore_order=True): - return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) is consistent, but value " \ - f"should use numbers for coordinates instead of strings" - else: - # Other causes - return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) value: {diff}" + + if 'location' not in entity: + return f"geo location '{geo_attr}' ({geo_type}) not null but location field not found in entity" + if entity['location']['attrName'] != geo_attr: + return f"location.attrName ({entity['location']['attrName']}) differs from '{geo_attr}'" + + geo_json = to_geo_json(entity['attrs'][geo_attr]) + if type(geo_json) == str: + return geo_json + + # https://www.testcult.com/deep-comparison-of-json-in-python/ + diff = DeepDiff(geo_json, entity['location']['coords'], ignore_order=True) + if diff: + # A typical difference is that attribute value uses strings and location uses numbers + # (this happens when the location was created/updated using NGSIv1). We try to identify that case + geo_json = convert_strings_to_numbers(geo_json) + if not DeepDiff(geo_json, entity['location']['coords'], ignore_order=True): + return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) is consistent, but value " \ + f"should use numbers for coordinates instead of strings" + else: + # Other causes + return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) value: {diff}" else: # len(geo_attrs) == 0 # If no geo attr found, check there isn't a location field if 'location' in entity: @@ -628,7 +658,8 @@ def rule92(entity): if 'location' in entity['attrs'][attr]['md']: location_value = entity['attrs'][attr]['md']['location']['value'] if location_value != 'WGS84' and location_value != 'WSG84': - s.append(f"in attribute '{attr}' location metadata value is {location_value} (should be WGS84 or WSG84)") + s.append( + f"in attribute '{attr}' location metadata value is {location_value} (should be WGS84 or WSG84)") if len(s) > 0: return ', '.join(s) @@ -773,20 +804,22 @@ def rule94(entity): ] -def process_db(logger, db_name, db_conn, query, rules_exp): +def process_db(logger, db_name, db_conn, include_entity_date, query, rules_exp): """ Process an individual DB :param logger: logger object :param db_name: the name of the DB to process :param db_conn: connection to MongoDB + :param include_entity_date: if True, include entity modification date in log traces :param query: query to filter entities to be processed :param rules_exp: regular expression to filter rules to apply :return: fails """ - logger.info(f'Processing {db_name}') + logger.info(f'processing {db_name}') n = 0 + failed_entities = 0 fails = 0 # check collection existence @@ -808,7 +841,15 @@ def process_db(logger, db_name, db_conn, query, rules_exp): # apply per-entity rules for entity in db_conn[db_name]['entities'].find(query): n += 1 + entity_fail = False + id_string = json.dumps(entity['_id']) + if include_entity_date: + if 'modDate' in entity: + id_string = f"({datetime.fromtimestamp(entity['modDate']).strftime('%Y-%m-%dT%H:%M:%SZ')}) {id_string}" + else: + id_string = f"()) {id_string}" + logger.debug(f'* processing entity {id_string}') for rule in rules: if rules_exp is not None and not re.search(rules_exp, rule['label']): @@ -818,9 +859,17 @@ def process_db(logger, db_name, db_conn, query, rules_exp): s = rule['func'](entity) if s is not None: logger.warning(f'DB {db_name} {rule["label"]} violation for entity {id_string}: {s}') + entity_fail = True fails += 1 - logger.info(f'processed {n} entities ({fails} rule violations)') + if entity_fail: + failed_entities += 1 + + if n > 0: + logger.info( + f'processed {db_name}: {failed_entities}/{n} ({round(failed_entities / n * 100, 2)}%) failed entities with {fails} rule violations') + else: + logger.warning(f'{db_name} has 0 entities (maybe it should be cleaned up?)') return fails @@ -834,10 +883,12 @@ def process_db(logger, db_name, db_conn, query, rules_exp): help='MongoDB URI. Default is mongodb://localhost:27017') parser.add_argument('--db', dest='db', help='DB name to check. If omitted all DBs starting with "orion" will be checked.') + parser.add_argument('--include-entities-date', dest='include_entities_date', default=False, action='store_true', + help='include entity modification time in log traces') parser.add_argument('--query', dest='query', default='{}', help='query to filter entities to check, in JSON MongoDB query language. By default, ' 'all entities in the collection will be checked.') - parser.add_argument('--rulesExp', dest='rules_exp', + parser.add_argument('--rules-exp', dest='rules_exp', help='Specifies the rules to apply, as a regular expression. By default all rules are applied.') parser.add_argument('--logLevel', dest='log_level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'], default='INFO', help='log level. Default is INFO') @@ -863,7 +914,7 @@ def process_db(logger, db_name, db_conn, query, rules_exp): fails = 0 if args.db is not None: if args.db in db_names: - fails += process_db(logger, args.db, mongo_client, query, args.rules_exp) + fails += process_db(logger, args.db, mongo_client, args.include_entities_date, query, args.rules_exp) else: logger.fatal(f'database {args.db} does not exist') sys.exit(1) @@ -871,6 +922,6 @@ def process_db(logger, db_name, db_conn, query, rules_exp): # Process all Orion databases for db_name in db_names: if db_name.startswith('orion-'): - fails += process_db(logger, db_name, mongo_client, query, args.rules_exp) + fails += process_db(logger, db_name, mongo_client, args.include_entities_date, query, args.rules_exp) logger.info(f'total rule violations: {fails}') diff --git a/scripts/entities_consistency/test_entities_consistency.py b/scripts/entities_consistency/test_entities_consistency.py index eb0319c901..f0ab04e399 100644 --- a/scripts/entities_consistency/test_entities_consistency.py +++ b/scripts/entities_consistency/test_entities_consistency.py @@ -52,4 +52,4 @@ def test_process_db(self): # connect to MongoDB and process validation DB mongo_client = MongoClient('mongodb://localhost:27017') - process_db(logger, 'orion-validation', mongo_client, {}, None) + process_db(logger, 'orion-validation', mongo_client, False, {}, None) diff --git a/scripts/entities_consistency/validation_data.js b/scripts/entities_consistency/validation_data.js index dde81aa544..a602e41ff0 100644 --- a/scripts/entities_consistency/validation_data.js +++ b/scripts/entities_consistency/validation_data.js @@ -1780,6 +1780,251 @@ db.getSiblingDB("orion-validation").entities.insertMany([ }, "lastCorrelator": "fa02fa68-b932-11ee-a0fb-080027cd35f1" }, + { + "_id": { + "id": "Rule16.12", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.12: Feature without geometry", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "Feature", + "properties": { + "label": "-3.6127119138731127, 40.53901978067972" + } + }, + "type": "geo:json", + "creDate": 1706274323.1297336, + "modDate": 1706274323.1297336, + "mdNames": [] + } + }, + "creDate": 1706274323.1297336, + "modDate": 1706274323.1297336, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + -3.6127119138731127, + 40.53901978067972 + ] + } + }, + "lastCorrelator": "90b31496-bc4b-11ee-ab64-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.13", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.13: FeatureCollection without features element", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "FeatureCollection" + }, + "type": "geo:json", + "creDate": 1706274323.130671, + "modDate": 1706274323.130671, + "mdNames": [] + } + }, + "creDate": 1706274323.130671, + "modDate": 1706274323.130671, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + -3.6127119138731127, + 40.53901978067972 + ] + } + }, + "lastCorrelator": "90b31496-bc4b-11ee-ab64-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.14", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.14: FeatureCollection with 0 Feature", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "FeatureCollection", + "features": [ + ] + }, + "type": "geo:json", + "creDate": 1706274323.130671, + "modDate": 1706274323.130671, + "mdNames": [] + } + }, + "creDate": 1706274323.130671, + "modDate": 1706274323.130671, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + -3.6127119138731127, + 40.53901978067972 + ] + } + }, + "lastCorrelator": "90b31496-bc4b-11ee-ab64-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.15", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.15: FeatureCollection with more than one Feature", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Point", + "coordinates": [ + -3.6127119138731127, + 40.53901978067972 + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Point", + "coordinates": [ + -3.6127119138731127, + 40.53901978067972 + ] + } + } + ] + }, + "type": "geo:json", + "creDate": 1706274323.130671, + "modDate": 1706274323.130671, + "mdNames": [] + } + }, + "creDate": 1706274323.130671, + "modDate": 1706274323.130671, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + -3.6127119138731127, + 40.53901978067972 + ] + } + }, + "lastCorrelator": "90b31496-bc4b-11ee-ab64-080027cd35f1" + }, + { + "_id": { + "id": "Rule16.16", + "type": "T", + "servicePath": "/" + }, + "attrNames": [ + "desc", + "location" + ], + "attrs": { + "desc": { + "value": "Rule16.16: FeatureCollection with Feature without geometry", + "type": "Text", + "creDate": 1705931202.187858, + "modDate": 1705931202.187858, + "mdNames": [] + }, + "location": { + "value": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {} + } + ] + }, + "type": "geo:json", + "creDate": 1706274323.130671, + "modDate": 1706274323.130671, + "mdNames": [] + } + }, + "creDate": 1706274323.130671, + "modDate": 1706274323.130671, + "location": { + "attrName": "location", + "coords": { + "type": "Point", + "coordinates": [ + -3.6127119138731127, + 40.53901978067972 + ] + } + }, + "lastCorrelator": "90b31496-bc4b-11ee-ab64-080027cd35f1" + }, { "_id": { "id": "Rule17.1", From c505fdc0616ce559c01e19b7130b019d85c4feba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 29 Jan 2024 09:08:55 +0100 Subject: [PATCH 124/390] Step: 3.10.0-next -> 3.11.0 --- CHANGES_NEXT_RELEASE | 28 ---------------------------- Changelog | 31 +++++++++++++++++++++++++++++++ README.md | 2 +- src/app/contextBroker/version.h | 2 +- src/lib/common/defaultValues.h | 2 +- 5 files changed, 34 insertions(+), 31 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 30bf0cd345..e69de29bb2 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,28 +0,0 @@ -- Add: servicePath field to builtin attributes (#2877) -- Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388) -- Add: notification payload in INFO log traces (#4449) -- Add: log deprecation traces for usage of legacyForwarding mode in registrations -- Add: log deprecation traces for usage of attrsFormat legacy in subscriptions -- Add: deprecatedFeatures counters block in GET /v2/statistics -- Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) -- Fix: builtin attributes alterationType, dateCreated and dateModified included in notifications even when onlyChangedAttrs is true -- Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) -- Fix: update forwarding was not working when entity type is not included in the request (#4460) -- Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) -- Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) -- Fix: in custom notification NGSI payload patching macro subscription covering completely the string where is used, always cast to string for id/type (#4462) -- Fix: improve error traces (#4387) -- Fix: on delete attributes operation, the lack of attribute in entity doesn't preclude the deletion of attributes which do exist (this way behaving same way as update attributes operation) -- Fix: POST /v2/entities/E/attrs?options=append was wrongly creating entities -- Fix: provide more informative error description in some error responses in update/delete operations -- Fix: proper use of "PartialUpdate" (instead of "Unprocessed") in responses in the case of partial updates/deletions (#3499) -- Fix: response 404 Not Found "NotFound" errors instead of 422 Unprocessable Content "Unprocessed" in the case of missing attribute in existing entity in attribute update operations -- Fix: "Internal Server Error" changed to "InternalServerError" in error responses -- Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) -- Fix: improve logs in MongoDB query logic -- Fix: false positive in log deprecation logic when entity name (or other literal) includes the token "v1" (#4454) -- Remove: deprecated NGSIv1 location metadata to specify entity geo-location -- Upgrade Debian version from 11.6 to 12.4 in Dockerfile -- Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3 -- Reference MongoDB version changed from 4.4 to 6.0 -- Reference distribution changed from Debian 11 to Debian 12 diff --git a/Changelog b/Changelog index 016b846508..dd05d8f08c 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,34 @@ +3.11.0 (January 29th, 2024) + +- Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388) +- Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794) +- Add: servicePath field to builtin attributes (#2877) +- Add: notification payload in INFO log traces (#4449) +- Add: log deprecation traces for usage of legacyForwarding mode in registrations +- Add: log deprecation traces for usage of attrsFormat legacy in subscriptions +- Add: deprecatedFeatures counters block in GET /v2/statistics +- Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643) +- Fix: do not show dateExpires built-in attribute in GET /v2/types and GET /v2/types/{type} operations (#4451) +- Fix: builtin attributes alterationType, dateCreated and dateModified included in notifications even when onlyChangedAttrs is true +- Fix: update forwarding was not working when entity type is not included in the request (#4460) +- Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435) +- Fix: in custom notification NGSI payload patching macro subscription covering completely the string where is used, always cast to string for id/type (#4462) +- Fix: on delete attributes operation, the lack of attribute in entity doesn't preclude the deletion of attributes which do exist (this way behaving same way as update attributes operation) +- Fix: POST /v2/entities/E/attrs?options=append was wrongly creating entities +- Fix: provide more informative error description in some error responses in update/delete operations +- Fix: proper use of "PartialUpdate" (instead of "Unprocessed") in responses in the case of partial updates/deletions (#3499) +- Fix: response 404 Not Found "NotFound" errors instead of 422 Unprocessable Content "Unprocessed" in the case of missing attribute in existing entity in attribute update operations +- Fix: "Internal Server Error" changed to "InternalServerError" in error responses +- Fix: improve logs in MongoDB query logic +- Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) +- Fix: false positive in log deprecation logic when entity name (or other literal) includes the token "v1" (#4454) +- Fix: improve error traces (#4387) +- Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3 +- Reference MongoDB version changed from 4.4 to 6.0 +- Reference distribution changed from Debian 11 to Debian 12 +- Upgrade Debian version from 11.6 to 12.4 in Dockerfile +- Remove: deprecated NGSIv1 location metadata to specify entity geo-location + 3.10.0 (June 12th, 2023) - Add: option to change deprecate log setting in the REST management API diff --git a/README.md b/README.md index 70ad7e6957..6c6101f61f 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![Support badge](https://img.shields.io/badge/tag-fiware--orion-orange.svg?logo=stackoverflow)](http://stackoverflow.com/questions/tagged/fiware-orion) [![NGSI v2](https://img.shields.io/badge/NGSI-V2-red.svg)](doc/manuals/orion-api.md)
-[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion.svg)](https://fiware-orion.rtfd.io) +[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion/3.11.0.svg)](https://fiware-orion.rtfd.io/en/3.11.0/) ![Compliance Tests](https://github.com/telefonicaid/fiware-orion/workflows/Compliance%20Tests/badge.svg) ![Unit Tests](https://github.com/telefonicaid/fiware-orion/workflows/Unit%20Tests/badge.svg) ![Functional Tests](https://github.com/telefonicaid/fiware-orion/workflows/Functional%20Tests/badge.svg) diff --git a/src/app/contextBroker/version.h b/src/app/contextBroker/version.h index 5937a761b9..a8acd507be 100644 --- a/src/app/contextBroker/version.h +++ b/src/app/contextBroker/version.h @@ -28,6 +28,6 @@ -#define ORION_VERSION "3.10.0-next" +#define ORION_VERSION "3.11.0" #endif // SRC_APP_CONTEXTBROKER_VERSION_H_ diff --git a/src/lib/common/defaultValues.h b/src/lib/common/defaultValues.h index 95998c82df..fe9a2e0b32 100644 --- a/src/lib/common/defaultValues.h +++ b/src/lib/common/defaultValues.h @@ -41,6 +41,6 @@ * * API Documentation - The link to the the GEri documentation, either in the gh-pages (.github.io/) inside the fiware organization in GitHub or ReadTheDocs manual. */ -#define API_DOC "https://fiware-orion.rtfd.io/" +#define API_DOC "https://fiware-orion.rtfd.io/en/3.11.0/" #endif // SRC_LIB_COMMON_DEFAULTVALUES_H_ From 4e4d96fcef96e50327a802da011d6b7da110926d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 29 Jan 2024 10:12:16 +0100 Subject: [PATCH 125/390] Step: 3.11.0 -> 3.11.0-next --- README.md | 2 +- src/app/contextBroker/version.h | 2 +- src/lib/common/defaultValues.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6c6101f61f..70ad7e6957 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![Support badge](https://img.shields.io/badge/tag-fiware--orion-orange.svg?logo=stackoverflow)](http://stackoverflow.com/questions/tagged/fiware-orion) [![NGSI v2](https://img.shields.io/badge/NGSI-V2-red.svg)](doc/manuals/orion-api.md)
-[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion/3.11.0.svg)](https://fiware-orion.rtfd.io/en/3.11.0/) +[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion.svg)](https://fiware-orion.rtfd.io) ![Compliance Tests](https://github.com/telefonicaid/fiware-orion/workflows/Compliance%20Tests/badge.svg) ![Unit Tests](https://github.com/telefonicaid/fiware-orion/workflows/Unit%20Tests/badge.svg) ![Functional Tests](https://github.com/telefonicaid/fiware-orion/workflows/Functional%20Tests/badge.svg) diff --git a/src/app/contextBroker/version.h b/src/app/contextBroker/version.h index a8acd507be..9e3680ba6d 100644 --- a/src/app/contextBroker/version.h +++ b/src/app/contextBroker/version.h @@ -28,6 +28,6 @@ -#define ORION_VERSION "3.11.0" +#define ORION_VERSION "3.11.0-next" #endif // SRC_APP_CONTEXTBROKER_VERSION_H_ diff --git a/src/lib/common/defaultValues.h b/src/lib/common/defaultValues.h index fe9a2e0b32..95998c82df 100644 --- a/src/lib/common/defaultValues.h +++ b/src/lib/common/defaultValues.h @@ -41,6 +41,6 @@ * * API Documentation - The link to the the GEri documentation, either in the gh-pages (.github.io/) inside the fiware organization in GitHub or ReadTheDocs manual. */ -#define API_DOC "https://fiware-orion.rtfd.io/en/3.11.0/" +#define API_DOC "https://fiware-orion.rtfd.io/" #endif // SRC_LIB_COMMON_DEFAULTVALUES_H_ From 6e7607e384a791c4ede2c5c1993a3a5b5f32afd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 31 Jan 2024 13:28:07 +0100 Subject: [PATCH 126/390] FIX simplify init methods in managers --- src/lib/alarmMgr/AlarmManager.cpp | 11 ++++------- src/lib/alarmMgr/AlarmManager.h | 4 ++-- src/lib/common/sem.cpp | 15 +++++---------- src/lib/common/sem.h | 2 +- src/lib/metricsMgr/MetricsManager.cpp | 7 ++----- src/lib/metricsMgr/MetricsManager.h | 2 +- src/lib/mqtt/MqttConnectionManager.cpp | 11 ++++------- src/lib/mqtt/MqttConnectionManager.h | 4 ++-- 8 files changed, 21 insertions(+), 35 deletions(-) diff --git a/src/lib/alarmMgr/AlarmManager.cpp b/src/lib/alarmMgr/AlarmManager.cpp index da84e66b79..171c68e4fd 100644 --- a/src/lib/alarmMgr/AlarmManager.cpp +++ b/src/lib/alarmMgr/AlarmManager.cpp @@ -79,7 +79,7 @@ AlarmManager::AlarmManager() * * AlarmManager::init - */ -int AlarmManager::init(bool logAlreadyRaisedAlarms) +void AlarmManager::init(bool logAlreadyRaisedAlarms) { notificationErrorLogAlways = logAlreadyRaisedAlarms; badInputLogAlways = logAlreadyRaisedAlarms; @@ -87,7 +87,7 @@ int AlarmManager::init(bool logAlreadyRaisedAlarms) mqttConnectionErrorLogAlways = logAlreadyRaisedAlarms; forwardingErrorLogAlways = logAlreadyRaisedAlarms; - return semInit(); + semInit(); } @@ -96,15 +96,12 @@ int AlarmManager::init(bool logAlreadyRaisedAlarms) * * AlarmManager::semInit - */ -int AlarmManager::semInit(void) +void AlarmManager::semInit(void) { if (sem_init(&sem, 0, 1) == -1) { - LM_E(("Runtime Error (error initializing 'alarm mgr' semaphore: %s)", strerror(errno))); - return -1; + LM_X(1, ("Fatal Error (error initializing 'alarm mgr' semaphore: %s)", strerror(errno))); } - - return 0; } diff --git a/src/lib/alarmMgr/AlarmManager.h b/src/lib/alarmMgr/AlarmManager.h index 685eba17ed..2b55387a5c 100644 --- a/src/lib/alarmMgr/AlarmManager.h +++ b/src/lib/alarmMgr/AlarmManager.h @@ -68,7 +68,7 @@ class AlarmManager public: AlarmManager(); - int init(bool logAlreadyRaisedAlarms); + void init(bool logAlreadyRaisedAlarms); void semTake(void); void semGive(void); @@ -97,7 +97,7 @@ class AlarmManager void mqttConnectionErrorGet(int64_t* active, int64_t* raised, int64_t* released); private: - int semInit(void); + void semInit(void); }; #endif // SRC_LIB_ALARMMGR_ALARMMANAGER_H_ diff --git a/src/lib/common/sem.cpp b/src/lib/common/sem.cpp index a629447692..8b1673ef3c 100644 --- a/src/lib/common/sem.cpp +++ b/src/lib/common/sem.cpp @@ -69,37 +69,32 @@ static struct timespec accTimeStatSemTime = { 0, 0 }; * -1 on failure * */ -int semInit(SemOpType _reqPolicy, bool semTimeStat, int shared, int takenInitially) +void semInit(SemOpType _reqPolicy, bool semTimeStat, int shared, int takenInitially) { if (sem_init(&reqSem, shared, takenInitially) == -1) { - LM_E(("Runtime Error (error initializing 'req' semaphore: %s)", strerror(errno))); - return -1; + LM_X(1, ("Fatal Error (error initializing 'req' semaphore: %s)", strerror(errno))); } if (sem_init(&transSem, shared, takenInitially) == -1) { - LM_E(("Runtime Error (error initializing 'transactionId' semaphore: %s)", strerror(errno))); - return -1; + LM_X(1, ("Fatal Error (error initializing 'transactionId' semaphore: %s)", strerror(errno))); } if (sem_init(&cacheSem, shared, takenInitially) == -1) { - LM_E(("Runtime Error (error initializing 'cache' semaphore: %s)", strerror(errno))); - return -1; + LM_X(1, ("Fatal Error (error initializing 'cache' semaphore: %s)", strerror(errno))); } if (sem_init(&timeStatSem, shared, takenInitially) == -1) { - LM_E(("Runtime Error (error initializing 'timeStat' semaphore: %s)", strerror(errno))); - return -1; + LM_X(1, ("Fatal Error (error initializing 'timeStat' semaphore: %s)", strerror(errno))); } reqPolicy = _reqPolicy; // Measure accumulated semaphore waiting time? semWaitStatistics = semTimeStat; - return 0; } diff --git a/src/lib/common/sem.h b/src/lib/common/sem.h index e106695d7c..04b10599e6 100644 --- a/src/lib/common/sem.h +++ b/src/lib/common/sem.h @@ -47,7 +47,7 @@ typedef enum SemOpType * * semInit - */ -extern int semInit +extern void semInit ( SemOpType _reqPolicy = SemReadWriteOp, bool semTimeStat = false, diff --git a/src/lib/metricsMgr/MetricsManager.cpp b/src/lib/metricsMgr/MetricsManager.cpp index 46014cfa66..595d10e3c7 100644 --- a/src/lib/metricsMgr/MetricsManager.cpp +++ b/src/lib/metricsMgr/MetricsManager.cpp @@ -103,18 +103,15 @@ bool MetricsManager::subServiceValid(const std::string& subsrv) * It's only one sys-call, and this way, the broker is prepared to receive 'on/off' * via REST. */ -bool MetricsManager::init(bool _on, bool _semWaitStatistics) +void MetricsManager::init(bool _on, bool _semWaitStatistics) { on = _on; semWaitStatistics = _semWaitStatistics; if (sem_init(&sem, 0, 1) == -1) { - LM_E(("Runtime Error (error initializing 'metrics mgr' semaphore: %s)", strerror(errno))); - return false; + LM_X(1, ("Fatal Error (error initializing 'metrics mgr' semaphore: %s)", strerror(errno))); } - - return true; } diff --git a/src/lib/metricsMgr/MetricsManager.h b/src/lib/metricsMgr/MetricsManager.h index c6a3cde1e0..a8b1aac374 100644 --- a/src/lib/metricsMgr/MetricsManager.h +++ b/src/lib/metricsMgr/MetricsManager.h @@ -138,7 +138,7 @@ class MetricsManager public: MetricsManager(); - bool init(bool _on, bool _semWaitStatistics); + void init(bool _on, bool _semWaitStatistics); void add(const std::string& srv, const std::string& subServ, const std::string& metric, uint64_t value); void reset(void); std::string toJson(bool doReset); diff --git a/src/lib/mqtt/MqttConnectionManager.cpp b/src/lib/mqtt/MqttConnectionManager.cpp index 00ef88b9a0..554fd936ba 100644 --- a/src/lib/mqtt/MqttConnectionManager.cpp +++ b/src/lib/mqtt/MqttConnectionManager.cpp @@ -105,7 +105,7 @@ MqttConnectionManager::MqttConnectionManager(void) * * MqttConnectionManager::init - */ -int MqttConnectionManager::init(long _timeout) +void MqttConnectionManager::init(long _timeout) { LM_T(LmtMqttNotif, ("Initializing MQTT library")); mosquitto_lib_init(); @@ -119,7 +119,7 @@ int MqttConnectionManager::init(long _timeout) timeout = DEFAULT_TIMEOUT; } - return semInit(); + semInit(); } @@ -152,15 +152,12 @@ void MqttConnectionManager::teardown(void) * * MqttConnectionManager::semInit - */ -int MqttConnectionManager::semInit(void) +void MqttConnectionManager::semInit(void) { if (sem_init(&sem, 0, 1) == -1) { - LM_E(("Runtime Error (error initializing 'mqtt connection mgr' semaphore: %s)", strerror(errno))); - return -1; + LM_X(1, ("Fatal Error (error initializing 'mqtt connection mgr' semaphore: %s)", strerror(errno))); } - - return 0; } diff --git a/src/lib/mqtt/MqttConnectionManager.h b/src/lib/mqtt/MqttConnectionManager.h index f80e1c25f6..012d2caa24 100644 --- a/src/lib/mqtt/MqttConnectionManager.h +++ b/src/lib/mqtt/MqttConnectionManager.h @@ -64,7 +64,7 @@ class MqttConnectionManager public: MqttConnectionManager(); - int init(long _timeout); + void init(long _timeout); void teardown(void); const char* semGet(void); @@ -74,7 +74,7 @@ class MqttConnectionManager private: void disconnect(struct mosquitto* mosq, const std::string& endpoint); - int semInit(void); + void semInit(void); void semTake(void); void semGive(void); From 2bba4c3c6c5774c1254041186a480e0273e55b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 31 Jan 2024 13:44:55 +0100 Subject: [PATCH 127/390] FIX broken utest --- test/unittests/common/commonSem_test.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/unittests/common/commonSem_test.cpp b/test/unittests/common/commonSem_test.cpp index 1a1442aba4..f389fbd449 100644 --- a/test/unittests/common/commonSem_test.cpp +++ b/test/unittests/common/commonSem_test.cpp @@ -45,8 +45,7 @@ TEST(commonSem, unique) { int s; - s = semInit(); - EXPECT_EQ(0, s); + semInit(); s = reqSemGive(__FUNCTION__, "test"); EXPECT_EQ(0, s); From 0005522391830da6b1233a7270a13fd4778c7dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 31 Jan 2024 13:06:50 +0100 Subject: [PATCH 128/390] ADD first version of jexlManager with init method to load Python interpreter --- CMakeLists.txt | 4 +++ src/app/contextBroker/contextBroker.cpp | 2 ++ src/lib/jexl/CMakeLists.txt | 42 +++++++++++++++++++++++ src/lib/jexl/JexlManager.cpp | 45 +++++++++++++++++++++++++ src/lib/jexl/JexlManager.h | 44 ++++++++++++++++++++++++ src/lib/jexl/jexlMgr.cpp | 33 ++++++++++++++++++ src/lib/jexl/jexlMgr.h | 38 +++++++++++++++++++++ src/lib/logMsg/traceLevels.h | 1 + 8 files changed, 209 insertions(+) create mode 100644 src/lib/jexl/CMakeLists.txt create mode 100644 src/lib/jexl/JexlManager.cpp create mode 100644 src/lib/jexl/JexlManager.h create mode 100644 src/lib/jexl/jexlMgr.cpp create mode 100644 src/lib/jexl/jexlMgr.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 42a046ec8d..4cae22ff0b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,6 +204,7 @@ SET (ORION_LIBS common alarmMgr metricsMgr + jexl logSummary lm pa @@ -245,6 +246,7 @@ SET (DYNAMIC_LIBS uuid crypto sasl2 + python3.11 ) # @@ -263,6 +265,7 @@ endif (UNIT_TEST) # Common include # include_directories("/usr/include") +include_directories("/usr/include/python3.11/") # Needed for the new C driver @@ -319,6 +322,7 @@ if (error EQUAL 0) ADD_SUBDIRECTORY(src/lib/cache) ADD_SUBDIRECTORY(src/lib/alarmMgr) ADD_SUBDIRECTORY(src/lib/metricsMgr) + ADD_SUBDIRECTORY(src/lib/jexl) ADD_SUBDIRECTORY(src/lib/logSummary) ADD_SUBDIRECTORY(src/lib/mqtt) ADD_SUBDIRECTORY(src/app/contextBroker) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index bf81abec8f..ccade9df1c 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -105,6 +105,7 @@ #include "alarmMgr/alarmMgr.h" #include "mqtt/mqttMgr.h" #include "metricsMgr/metricsMgr.h" +#include "jexl/jexlMgr.h" #include "logSummary/logSummary.h" #include "contextBroker/orionRestServices.h" @@ -1215,6 +1216,7 @@ int main(int argC, char* argV[]) SemOpType policy = policyGet(reqMutexPolicy); alarmMgr.init(relogAlarms); mqttMgr.init(mqttTimeout); + jexlMgr.init(); orionInit(orionExit, ORION_VERSION, policy, statCounters, statSemWait, statTiming, statNotifQueue, strictIdv1); mongoInit(dbURI, dbHost, rplSet, dbName, user, pwd, authMech, authDb, dbSSL, dbDisableRetryWrites, mtenant, dbTimeout, writeConcern, dbPoolSize, statSemWait); metricsMgr.init(!disableMetrics, statSemWait); diff --git a/src/lib/jexl/CMakeLists.txt b/src/lib/jexl/CMakeLists.txt new file mode 100644 index 0000000000..67a7a10e24 --- /dev/null +++ b/src/lib/jexl/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +CMAKE_MINIMUM_REQUIRED(VERSION 2.6) + +SET (SOURCES + JexlManager.cpp + jexlMgr.cpp +) + +SET (HEADERS + JexlManager.h + jexlMgr.h +) + + + +# Include directories +# ----------------------------------------------------------------- +include_directories("${PROJECT_SOURCE_DIR}/src/lib") + + +# Library declaration +# ----------------------------------------------------------------- +ADD_LIBRARY(jexl STATIC ${SOURCES} ${HEADERS}) diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp new file mode 100644 index 0000000000..61725b309d --- /dev/null +++ b/src/lib/jexl/JexlManager.cpp @@ -0,0 +1,45 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include + +#include "jexl/JexlManager.h" +#include "logMsg/logMsg.h" + +/* **************************************************************************** +* +* JexlManager::init - +*/ +void JexlManager::init(void) +{ + if (sem_init(&sem, 0, 1) == -1) + { + LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); + } + + // FIXME PR: Py_Finalize() has to be be called in the proper place + Py_Initialize(); + LM_T(LmtJexl, ("Python interpreter has been initialized")); +} \ No newline at end of file diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h new file mode 100644 index 0000000000..d22f442392 --- /dev/null +++ b/src/lib/jexl/JexlManager.h @@ -0,0 +1,44 @@ +#ifndef SRC_LIB_JEXL_JEXLMANAGER_H_ +#define SRC_LIB_JEXL_JEXLMANAGER_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include + +/* **************************************************************************** +* +* JexlManager - +*/ +class JexlManager +{ +private: + sem_t sem; + +public: + void init(void); +}; + +#endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file diff --git a/src/lib/jexl/jexlMgr.cpp b/src/lib/jexl/jexlMgr.cpp new file mode 100644 index 0000000000..a5e8e2cd85 --- /dev/null +++ b/src/lib/jexl/jexlMgr.cpp @@ -0,0 +1,33 @@ +/* +* +* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Ken Zangelin +*/ +#include "jexl/JexlManager.h" + + + +/* **************************************************************************** +* +* jexlMgr - +*/ +JexlManager jexlMgr; diff --git a/src/lib/jexl/jexlMgr.h b/src/lib/jexl/jexlMgr.h new file mode 100644 index 0000000000..75b22a66a8 --- /dev/null +++ b/src/lib/jexl/jexlMgr.h @@ -0,0 +1,38 @@ +#ifndef SRC_LIB_JEXL_JEXLMGR_H_ +#define SRC_LIB_JEXL_JEXLMGR_H_ + +/* +* +* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ +#include "jexl/JexlManager.h" + + + +/* **************************************************************************** +* +* metricsMgr - +*/ +extern JexlManager jexlMgr; + +#endif // SRC_LIB_JEXL_JEXLMGR_H_ diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index d0df21e423..8a5ca6a412 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -131,6 +131,7 @@ typedef enum TraceLevels LmtNotImplemented, LmtCurlContext, LmtThreadpool, + LmtJexl, LmtOldInfo = 240, // old INFO traces moved to DEBUG in Orion 2.5.0 From ef79992b9dc6c12912cbd9062670d42ee2bf64e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Feb 2024 09:14:00 +0100 Subject: [PATCH 129/390] FIX typo in orion-api.md --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index ae67dc2f51..7b66992569 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -812,7 +812,7 @@ i.e. `GET /v2/entities?q=T>2021-04-21`. { "timestamp": { "value": "2017-06-17T07:21:24.238Z", - "type: "DateTime" + "type": "DateTime" } } ``` From 42f215efc30fc2b5faf1c8ad53051426704c1edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 5 Feb 2024 16:51:59 +0100 Subject: [PATCH 130/390] FIX PoC to measure times --- src/lib/jexl/JexlManager.cpp | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 61725b309d..e14017073c 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -40,6 +40,44 @@ void JexlManager::init(void) } // FIXME PR: Py_Finalize() has to be be called in the proper place + LM_T(LmtJexl, ("Python interpreter is about to be initialized")); Py_Initialize(); LM_T(LmtJexl, ("Python interpreter has been initialized")); + + // FIXME PR: proper Py_XDECREF() has to be called in the proper place + // FIXME PR: capture error and move it to char* to print in LM_X + PyObject* jexl_module = PyImport_ImportModule("pyjexl"); + LM_T(LmtJexl, ("jexl module has been loaded")); + + // Create JEXL engine + PyObject* jexl_engine = PyObject_CallMethod(jexl_module, "JEXL", NULL); + LM_T(LmtJexl, ("jexl engine has been created")); + + // Create expression and context + PyObject* expression = Py_BuildValue("s", "x + y"); + LM_T(LmtJexl, ("jexl expression has been built")); + PyObject* context = Py_BuildValue("{s:i, s:i}", "x", 4, "y", 11); + LM_T(LmtJexl, ("jexl context has been built")); + + // Call evaluate method + PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, context); + LM_T(LmtJexl, ("jexl evaluation is done")); + + // Print result + PyObject* repr = PyObject_Repr(result); + const char* result_str = PyUnicode_AsUTF8(repr); + LM_T(LmtJexl, ("jexl evaluation result is obtainted")); + LM_T(LmtJexl, ("jexl result: %s", result_str)); + + // Free resources + Py_XDECREF(repr); + Py_XDECREF(result); + Py_XDECREF(context); + Py_XDECREF(expression); + Py_XDECREF(jexl_engine); + Py_XDECREF(jexl_module); + LM_T(LmtJexl, ("all has been freed")); + + // Finalize the Python interpreter + Py_Finalize(); } \ No newline at end of file From 2aa814d117d876de35adccb874c3a38a8c4153de Mon Sep 17 00:00:00 2001 From: George Alexiou Date: Wed, 7 Feb 2024 09:10:45 +0000 Subject: [PATCH 131/390] FIX -dbTimeout conflict with -dbURI (#4496) (#4497) * FIX -dbTimeout conflict with -dbURI * Adjust timeout handling and fixing syntax error * Simplify dbTimeout Handling by Setting Default to 0 * Update CHANGES_NEXT_RELEASE for -dbTimeout Default Value Fix * Adjust Test Expectations for New -dbTimeout Default --- CHANGES_NEXT_RELEASE | 1 + src/app/contextBroker/contextBroker.cpp | 2 +- src/lib/mongoDriver/mongoConnectionPool.cpp | 2 +- test/functionalTest/cases/3658_env_vars/env_vars.test | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e69de29bb2..41e24c99bf 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -0,0 +1 @@ +- Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) \ No newline at end of file diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index bf81abec8f..71d2a6669f 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -309,7 +309,7 @@ PaArgument paArgs[] = { "-dbDisableRetryWrites", &dbDisableRetryWrites, "MONGO_DISABLE_RETRY_WRITES", PaBool, PaOpt, false, false, true, DBDISABLERETRYWRITES_DESC }, { "-db", dbName, "MONGO_DB", PaString, PaOpt, _i "orion", PaNL, PaNL, DB_DESC }, - { "-dbTimeout", &dbTimeout, "MONGO_TIMEOUT", PaULong, PaOpt, 10000, 0, UINT_MAX, DB_TMO_DESC }, + { "-dbTimeout", &dbTimeout, "MONGO_TIMEOUT", PaULong, PaOpt, 0, 0, UINT_MAX, DB_TMO_DESC }, { "-dbPoolSize", &dbPoolSize, "MONGO_POOL_SIZE", PaInt, PaOpt, 10, 1, 10000, DBPS_DESC }, { "-ipv4", &useOnlyIPv4, "USEIPV4", PaBool, PaOpt, false, false, true, USEIPV4_DESC }, diff --git a/src/lib/mongoDriver/mongoConnectionPool.cpp b/src/lib/mongoDriver/mongoConnectionPool.cpp index ed0d1db07c..855896753e 100644 --- a/src/lib/mongoDriver/mongoConnectionPool.cpp +++ b/src/lib/mongoDriver/mongoConnectionPool.cpp @@ -407,7 +407,7 @@ static std::string composeMongoUri optionPrefix = "&"; } } - + LM_T(LmtMongo, ("MongoDB connection URI: '%s'", offuscatePassword(uri, passwd).c_str())); return uri; diff --git a/test/functionalTest/cases/3658_env_vars/env_vars.test b/test/functionalTest/cases/3658_env_vars/env_vars.test index 495c359dec..2ede8f83af 100644 --- a/test/functionalTest/cases/3658_env_vars/env_vars.test +++ b/test/functionalTest/cases/3658_env_vars/env_vars.test @@ -95,7 +95,7 @@ Extended Usage: contextBroker [option '-U' (extended usage)] [option '-dbSSL' (enable SSL connection to DB)] ORION_MONGO_SSL FALSE /FALSE/ [option '-dbDisableRetryWrites' (set retryWrite parameter to false in DB connect] ORION_MONGO_DISABLE_RETRY_WRITES FALSE /FALSE/ [option '-db' ] ORION_MONGO_DB 'orion' /'orion'/ - [option '-dbTimeout' ] ORION_MONGO_POOL_SIZE 1 <= 10 /10/ <= 10000 [option '-ipv4' (use ip v4 only)] ORION_USEIPV4 FALSE /FALSE/ [option '-ipv6' (use ip v6 only)] ORION_USEIPV6 FALSE /FALSE/ From 8d8d954653ee2d5e55d73dbe7ae3685465fdab10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 7 Feb 2024 11:05:56 +0100 Subject: [PATCH 132/390] FIX wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4501) --- CHANGES_NEXT_RELEASE | 3 ++- src/app/contextBroker/contextBroker.cpp | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 41e24c99bf..1e9166cf9c 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ -- Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) \ No newline at end of file +- Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) +- Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index 71d2a6669f..d1cc8a57b2 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -979,13 +979,17 @@ static void logEnvVars(void) { LM_I(("env var ORION_%s (%s): %d", aP->envName, aP->option, *((int*) aP->varP))); } + else if (aP->type == PaULong) + { + LM_I(("env var ORION_%s (%s): %d", aP->envName, aP->option, *((unsigned long*) aP->varP))); + } else if (aP->type == PaDouble) { LM_I(("env var ORION_%s (%s): %d", aP->envName, aP->option, *((double*) aP->varP))); } else { - LM_I(("env var ORION_%s (%s): %d", aP->envName, aP->option)); + LM_E(("cannot show env var ORION_%s (%s) due to unrecognized type: %d", aP->envName, aP->option, aP->type)); } } } From fd572d6f5bbd5fc1e79446c3f4c5f5a2cdc0f4d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 7 Feb 2024 12:34:37 +0100 Subject: [PATCH 133/390] FIX return 400 Bad Request when subject.entities exists but it is an empty array (#4502) --- CHANGES_NEXT_RELEASE | 1 + src/lib/jsonParseV2/parseSubscription.cpp | 5 + .../sub_without_entities_return_error.test | 193 ++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 test/functionalTest/cases/4499_subs_without_entities_return_error/sub_without_entities_return_error.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 1e9166cf9c..fdb575ad63 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ - Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) +- Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index 3724528ba1..4207dfa023 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -278,6 +278,11 @@ static std::string parseSubject(ConnectionInfo* ciP, SubscriptionUpdate* subsP, return badInput(ciP, errorString); } + if (subsP->subject.entities.size() == 0) + { + return badInput(ciP, "subject entities is empty"); + } + // Condition if (subject.HasMember("condition")) { diff --git a/test/functionalTest/cases/4499_subs_without_entities_return_error/sub_without_entities_return_error.test b/test/functionalTest/cases/4499_subs_without_entities_return_error/sub_without_entities_return_error.test new file mode 100644 index 0000000000..c1f71e8d51 --- /dev/null +++ b/test/functionalTest/cases/4499_subs_without_entities_return_error/sub_without_entities_return_error.test @@ -0,0 +1,193 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Subscription without entities return error + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create subscription without subject.entities field, see error +# 02. Create subscription with subject.entities field with 0 items, see error +# 03. Create subscription +# 04. Update subscription without subject.entities field, see error +# 05. Update subscription with subject.entities field with 0 items, see error +# + + +echo "01. Create subscription without subject.entities field, see error" +echo "=================================================================" +payload='{ + "subject": { + }, + "notification": { + "http": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create subscription with subject.entities field with 0 items, see error" +echo "===========================================================================" +payload='{ + "subject": { + "entities": [ + ] + }, + "notification": { + "http": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create subscription" +echo "=======================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "04. Update subscription without subject.entities field, see error" +echo "=================================================================" +payload='{ + "subject": { + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID -X PATCH --payload "$payload" +echo +echo + + +echo "05. Update subscription with subject.entities field with 0 items, see error" +echo "===========================================================================" +payload='{ + "subject": { + "entities": [ + ] + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID -X PATCH --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. Create subscription without subject.entities field, see error +================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "no subject entities specified", + "error": "BadRequest" +} + + +02. Create subscription with subject.entities field with 0 items, see error +=========================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 64 + +{ + "description": "subject entities is empty", + "error": "BadRequest" +} + + +03. Create subscription +======================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Update subscription without subject.entities field, see error +================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "no subject entities specified", + "error": "BadRequest" +} + + +05. Update subscription with subject.entities field with 0 items, see error +=========================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 64 + +{ + "description": "subject entities is empty", + "error": "BadRequest" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB From 1d8a75848aa55f2819ca7ebe59512b7da2621fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 7 Feb 2024 14:17:00 +0100 Subject: [PATCH 134/390] FIX deprecated old db CLI --- CHANGES_NEXT_RELEASE | 1 + ci/deb/build.sh | 4 +-- ci/deb/makefile | 2 +- doc/manuals/admin/cli.md | 36 +------------------ doc/manuals/admin/database_admin.md | 25 +++++++------ doc/manuals/admin/diagnosis.md | 2 +- doc/manuals/admin/sanity_check.md | 15 +++----- doc/manuals/admin/watchdog.md | 2 +- doc/manuals/deprecated.md | 28 +++++++++++++++ doc/manuals/devel/cookbook.md | 2 +- docker/README.md | 10 +++--- docker/docker-compose.yml | 2 +- docker/docker_swarm.md | 2 +- docker/raspberry_pi.md | 2 +- etc/init.d/contextBroker.centos | 5 ++- etc/init.d/contextBroker.ubuntu | 2 +- makefile | 4 +-- .../components/common_steps/initial_steps.py | 2 +- test/functionalTest/harnessFunctions.sh | 14 ++++---- 19 files changed, 77 insertions(+), 83 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index fdb575ad63..e628230f8f 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ - Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) - Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) +- Deprecate: `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` CLI parameters along with associated env vars (use `-dbURI` instead`) \ No newline at end of file diff --git a/ci/deb/build.sh b/ci/deb/build.sh index f78d96371a..aea9ed627d 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -93,8 +93,8 @@ function _execute() sleep 3 - contextBroker -port 30001 -dbhost localhost:20001 -pidpath cb1.pid & - contextBroker -port 30002 -dbhost localhost:20002 -pidpath cb2.pid & + contextBroker -port 30001 -dbURI mongodb://localhost:20001 -pidpath cb1.pid & + contextBroker -port 30002 -dbURI mongodb://localhost:20002 -pidpath cb2.pid & } [ $# = 0 ] && _usage diff --git a/ci/deb/makefile b/ci/deb/makefile index 4cbc001c5d..be524b592f 100644 --- a/ci/deb/makefile +++ b/ci/deb/makefile @@ -69,7 +69,7 @@ install_unit: install unit: @echo '------------------------------------- make unit ----------------------------------------' - BUILD_UNIT/test/unittests/unitTest -t 0-255 -dbhost ${MONGO_HOST} --gtest_output=xml:/tmp/builder/logs/unit.xml + BUILD_UNIT/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:/tmp/builder/logs/unit.xml build_functional: prepare @echo '------------------------------------- make build_functional ----------------------------' diff --git a/doc/manuals/admin/cli.md b/doc/manuals/admin/cli.md index 0899b45255..0159a4e1a8 100644 --- a/doc/manuals/admin/cli.md +++ b/doc/manuals/admin/cli.md @@ -49,36 +49,9 @@ The list of available options is the following: - **-dbURI ** : The URI to use the MongoDB. If the URI contains the string `${PWD}`, it will be replaced with the password specified in `-dbpwd` or the environment variable `ORION_MONGO_PASSWORD`. - This option cannot be combined with `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, - `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` (if you attempt to do that - Orion will exit with an error on startup). -- **-dbhost **. The MongoDB host and port to use, e.g. `-dbhost - localhost:12345`. -- **-rplSet **. If used, Orion CB connnects to a - MongoDB replica set (instead of a stand-alone MongoDB instance). - The name of the replica set to use is the value of the parameter. In - this case, the -dbhost parameter can be a list of hosts (separated - by ",") which are used as seed for the replica set. -- **-dbTimeout **. Only used in the case of using replica - set (-rplSet), ignored otherwise. It specifies the timeout in - milliseconds for connections to the replica set. -- **-dbuser **. The MongoDB user to use. If your MongoDB doesn't - use authorization then this option must be avoided. See [database - authorization section](database_admin.md#database-authorization). - **-dbpwd **. The MongoDB password to use. If your MongoDB doesn't use authorization then this option must be avoided. See [database - authorization section]( database_admin.md#database-authorization). -- **-dbAuthMech **. The MongoDB authentication mechanism to use in the case - of providing `-dbuser` and `-dbpwd`. Alternatives are SCRAM-SHA-1 or SCRAM-SHA-256. -- **-dbAuthDb **. Specifies the database to use for authentication in the case - of providing `-dbuser` and `-dbpwd`. -- **-dbSSL**. Enable SSL in the connection to MongoDB. You have to use this option if your - MongoDB server or replica set is using SSL (or, the other way around, you have not to use - this option if your MongoDB server or replicat set is not using SSL). Note there is - currently a limitation: Orion uses `tlsAllowInvalidCertificates=true` in this case, - so the certificate used by MongoDB server is not being validated. -- **-dbDisableRetryWrites**. Set retryWrite parameter to false in DB connections (not - recommended, only to keep compatibility with old MongoDB instances) + authorization section](database_admin.md#database-authorization). - **-dbPoolSize **. Database connection pool. Default size of the pool is 10 connections. - **-writeConcern <0|1>**. Write concern for MongoDB write operations: @@ -218,15 +191,8 @@ Two facts have to be taken into account: | ORION_PORT | port | | ORION_PID_PATH | pidpath | | ORION_MONGO_URI | dbURI | -| ORION_MONGO_HOST | dbhost | -| ORION_MONGO_REPLICA_SET | rplSet | -| ORION_MONGO_USER | dbuser | | ORION_MONGO_PASSWORD | dbpwd | -| ORION_MONGO_AUTH_MECH | dbAuthMech | -| ORION_MONGO_AUTH_SOURCE | dbAuthDb | -| ORION_MONGO_SSL | dbSSL | | ORION_MONGO_DB | db | -| ORION_MONGO_TIMEOUT | dbTimeout | | ORION_MONGO_POOL_SIZE | dbPoolSize | | ORION_USEIPV4 | ipv4 | | ORION_USEIPV6 | ipv6 | diff --git a/doc/manuals/admin/database_admin.md b/doc/manuals/admin/database_admin.md index 7f0e868a8c..f521469252 100644 --- a/doc/manuals/admin/database_admin.md +++ b/doc/manuals/admin/database_admin.md @@ -65,41 +65,44 @@ you need to apply the procedures to each per-tenant/service database. ## Database authorization -MongoDB authorization is configured with the `-db`, `-dbuser` and `-dbpwd` -options ([see section on command line -options](cli.md)). There are a few different cases +MongoDB authorization is configured using `-dbURI` and `-dbpwd` +options ([see section on command line options](cli.md)). There are a few different cases to take into account: - If your MongoDB instance/cluster doesn't use authorization, - then do not use the `-dbuser` and `-dbpwd` options. -- You can specify authentication mechanism with `-dbAuthMech`. + then do not use the `-dbpwd` and use `-dbURI` without the `username:${PWD}@` part. +- You can specify authentication mechanism within the `-dbURI`, using the `authMechanism` option. - If your MongoDB instance/cluster uses authorization , then: + - In the `-dbURI` you have to use the `username:${PWD}@` part. The `${PWD}` will be replaced + by the value of the `dbpwd` parameter. - If you run Orion in single service/tenant mode (i.e. without `-multiservice`) then you are using only one database - (the one specified by the -db option) and the authorization is - done with `-dbuser` and `-dbpwd` in that database. + (the one specified by the `-db` option) and the authorization is + done in that database with the username specified in the `-dbURI` and `-dbpwd`. - If you run Orion in multi service/tenant mode (i.e. with `-multiservice`) then the authorization is done at `admin` - database using `-dbuser` and `-dbpwd`. As described [later in this + database using the username specified in the `-dbURI` and `-dbpwd`. As described [later in this document](#multiservicemultitenant-database-separation), in multi service/tenant mode, Orion uses several databases (which in addition can potentially be created on the fly), thus authorizing on `admin` DB ensures permissions in all of them. - - Anyway, you can override the above default with `-dbAuthDb` and + - Anyway, you can override the above default with `defaultauthdb` in the `-dbURI` and specify the authentication DB you want. +Check the [MongoURI documentation](https://www.mongodb.com/docs/manual/reference/connection-string) for additional information. + Let's consider the following example. If your MongoDB configuration is so you typically access to it using: ``` -mongo "mongodb://example1.net:27017,example2.net:27017,example3.net:27017/orion?replicaSet=rs0" --ssl --authenticationDatabase admin --username orion --password orionrules +mongosh mongodb://orion@orionrules:example1.net:27017,example2.net:27017,example3.net:27017/admin?replicaSet=rs0&tls=true&tlsAllowInvalidCertificates=true ``` Then the equivalent connection in Context Broker CLI parameters will be: ``` --dbhost examples1.net:27017,example2.net:27017,example3.net:27017 -rplSet rs0 -dbSSL -dbAuthDb admin -dbuser orion -dbpwd orionrules +-dbURI mongodb://orion@${PWD}:example1.net:27017,example2.net:27017,example3.net:27017/admin?replicaSet=rs0&tls=true&tlsAllowInvalidCertificates=true -dbpwd orionrules ``` diff --git a/doc/manuals/admin/diagnosis.md b/doc/manuals/admin/diagnosis.md index f4a0117b77..b59ce9de36 100644 --- a/doc/manuals/admin/diagnosis.md +++ b/doc/manuals/admin/diagnosis.md @@ -288,7 +288,7 @@ The symptoms of a database connection problem are the following: ``` In both cases, check that the connection to MonogDB is correctly -configured (in particular, the "-dbhost" option [from the command +configured (in particular, the `-dbURI` option [from the command line](cli.md)) and that the mongod/mongos process (depending if you are using sharding or not) is up and running. diff --git a/doc/manuals/admin/sanity_check.md b/doc/manuals/admin/sanity_check.md index b52616cbdf..2657aff67e 100644 --- a/doc/manuals/admin/sanity_check.md +++ b/doc/manuals/admin/sanity_check.md @@ -46,7 +46,7 @@ A process named "contextBroker" should be up and running, e.g.: ``` $ ps ax | grep contextBroker - 8517 ? Ssl 8:58 /usr/bin/contextBroker -port 1026 -logDir /var/log/contextBroker -pidpath /var/log/contextBroker/contextBroker.pid -dbhost localhost -db orion + 8517 ? Ssl 8:58 /usr/bin/contextBroker -port 1026 -logDir /var/log/contextBroker -pidpath /var/log/contextBroker/contextBroker.pid -dbURI mongodb://localhost/ -db orion ``` [Top](#top) @@ -63,25 +63,18 @@ changed using the -port command line option. The Orion Context Broker uses a MongoDB database, whose parameters are provided using the command line options: -* `-dbhost` +* `-dbUri` * `-db` -* `-dbuser` * `-dbpwd` -* `-dbAuthMech` -* `-dbAuthDb` -* `-dbSSL` -* `-dbDisableRetryWrites` -* `-dbTimeout` * `-dbPoolSize` * `-writeConcern` -Note that `-dbuser`, `-dbpwd`, `-dbAuthMech` and `-dbAuthDb`. -are only used if MongoDB runs using authentication, i.e. with `--auth`. +Note that `-dbpwd` is only used if MongoDB runs using authentication, i.e. with `--auth`. You can check that the database is working using the mongo console: ``` -mongo / +mongosh / ``` You can check the different collections used by the broker using the diff --git a/doc/manuals/admin/watchdog.md b/doc/manuals/admin/watchdog.md index 88071dbbe9..27ffd197da 100644 --- a/doc/manuals/admin/watchdog.md +++ b/doc/manuals/admin/watchdog.md @@ -90,7 +90,7 @@ e.g.: # ps -ef | grep contextBroker 500 27175 1 0 21:06 ? 00:00:00 monit -v -c /home/localadmin/monit_CB/monitBROKER.conf -d 10 -p /var/log/contextBroker/monit.pid - 500 27205 1 0 21:06 ? 00:00:00 /usr/bin/contextBroker -port 1026 -logDir /var/log/contextBroker -pidpath /var/log/contextBroker/contextBroker.pid -dbhost localhost -db orion; + 500 27205 1 0 21:06 ? 00:00:00 /usr/bin/contextBroker -port 1026 -logDir /var/log/contextBroker -pidpath /var/log/contextBroker/contextBroker.pid -dbURI mongodb://localhost/ -db orion; Then, kill contextBroker, e.g.: diff --git a/doc/manuals/deprecated.md b/doc/manuals/deprecated.md index 141ab0d6d5..9b289331b4 100644 --- a/doc/manuals/deprecated.md +++ b/doc/manuals/deprecated.md @@ -15,6 +15,9 @@ not maintained or evolved any longer. In particular: A list of deprecated features and the version in which they were deprecated follows: +* CLI parameters (and associated env vars): `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, + `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` in Orion 3.12.0. Use `dbURI` instead, + checking [this section](#mapping-to-mongouri-from-old-cli-parameters) if you need to know hot to build the MongoDB URI. * `geo:point`, `geo:line`, `geo:box` and `geo:polygon` attribute types in Orion 3.10.0. Use `geo:json` instead. * `GET /v2` operation in Orion 3.8.0. This operation is pretty useless and not actually used. * Initial notification in subscriptions (along with `skipInitialNotification` option) in Orion 3.1.0. @@ -77,6 +80,30 @@ A list of deprecated features and the version in which they were deprecated foll * Configuration Manager role (deprecated in 0.21.0, removed in 0.25.0) * Associations (deprecated in 0.21.0, removed in 0.25.0). +### Mapping to MongoURI from old CLI parameters + +Considering we have the following CLI parameters: + +* `-dbhost HOST` +* `-rplSet RPLSET` +* `-dbTimeout TIMEOUT` +* `-dbuser USER` +* `-dbpass PASS` +* `-dbAuthMech AUTHMECH` +* `-dbAuthDb AUTHDB` +* `-dbSSL` +* `-dbDisableRetryWrites` + +The resulting MongoURI (i.e. the value for `-dbURI`) should be: + +> mongodb://[USER:PASS@]HOST/[AUTHDB][?replicaSet=RPLSET[&authMechanism=AUTHMECH][&tls=true&tlsAllowInvalidCertificates=true][&retryWrites=false][&connectTimeoutMS=TIMEOUT] + +Notes: + +* The `&tls=true&tlsAllowInvalidCertificates=true` token is added if `-dbSSL` is used +* The `&retryWrites=false` token is added if `-dbDisableRetryWrites` is used +* Other `[...]` mean optional tokens, depending on if the corresponding parameter is used or not. + ## Log deprecation warnings Some (not all) usages of deprecated features can be logged using the `-logDeprecate` [CLI flag](admin/cli.md) @@ -107,6 +134,7 @@ The following table provides information about the last Orion version supporting | **Removed feature** | **Last Orion version supporting feature** | **That version release date** | |----------------------------------------------------------------------------|-------------------------------------------|---------------------------------| +| CLI `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` (and associated env vars) | Not yet defined | Not yet defined | | `attributes` field in `POST /v2/entities` operation | Not yet defined | Not yet defined | | `APPEND`, `UPDATE`, etc. action types in `POST /v2/op/update` | Not yet defined | Not yet defined | | `dateCreated` and `dateModified` in `options` URI parameter | Not yet defined | Not yet defined | diff --git a/doc/manuals/devel/cookbook.md b/doc/manuals/devel/cookbook.md index 6af4ee7b8f..323adc39ac 100644 --- a/doc/manuals/devel/cookbook.md +++ b/doc/manuals/devel/cookbook.md @@ -20,7 +20,7 @@ You basically need to implement two things: * a **new item** in the `PaArgument` vector `paArgs` If the new CLI parameter is a boolean one, like `-v` (verbose), a `bool` variable is needed, -if it's a text parameter, like `-dbHost `, a char-vector is used, and so on. +if it's a text parameter, like `-dbURI `, a char-vector is used, and so on. The easiest way is to simply copy an older CLI parameter of the same type. diff --git a/docker/README.md b/docker/README.md index e8cc578dce..31f7ccf9a1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -35,7 +35,7 @@ Follow these steps: - "1026:1026" depends_on: - mongo - command: -dbhost mongo + command: -dbURI mongodb://mongo mongo: image: mongo:6.0 @@ -81,7 +81,7 @@ In case you want to run MongoDB on another container you can launch it like this And then run Orion with this command - sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 fiware/orion -dbhost mongodb + sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 fiware/orion -dbURI mongodb://mongodb Check that everything works with @@ -93,7 +93,7 @@ This method is functionally equivalent as the one described in section 1, but do If you want to connect to a different MongoDB instance do the following command **instead of** the previous one - sudo docker run -d --name orion1 -p 1026:1026 fiware/orion -dbhost + sudo docker run -d --name orion1 -p 1026:1026 fiware/orion -dbURI mongodb:// Check that everything works with @@ -113,10 +113,10 @@ Steps: * Manually, running MongoDB on another container: 1. `sudo docker run --name mongodb -d mongo:6.0` 2. `sudo docker build -t orion .` - 3. `sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 orion -dbhost mongodb`. + 3. `sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 orion -dbURI mongodb://mongodb`. * Manually, specifying where to find your MongoDB host: 1. `sudo docker build -t orion .` - 2. `sudo docker run -d --name orion1 -p 1026:1026 orion -dbhost `. + 2. `sudo docker run -d --name orion1 -p 1026:1026 orion -dbURI mongodb://`. Check that everything works with diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4587af3990..81b78922e7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,7 +7,7 @@ services: - "1026:1026" depends_on: - mongo - command: -dbhost mongo + command: -dbURI mongodb://mongo mongo: image: mongo:6.0 diff --git a/docker/docker_swarm.md b/docker/docker_swarm.md index ef30a5b5a4..d6f23ee16c 100644 --- a/docker/docker_swarm.md +++ b/docker/docker_swarm.md @@ -185,7 +185,7 @@ Details on how to deploy a MongoDB ReplicaSet in Docker Swarm are available image: fiware/orion:latest ports: - "1026:1026" - command: -logLevel DEBUG -dbhost mongo_mongo -rplSet rs -dbTimeout 10000 + command: -logLevel DEBUG -dbURI mongodb://mongo_mongo/?replicaSet=rs&connectTimeoutMS=10000 deploy: replicas: 2 networks: diff --git a/docker/raspberry_pi.md b/docker/raspberry_pi.md index bbc3615ac6..cb7baecd07 100644 --- a/docker/raspberry_pi.md +++ b/docker/raspberry_pi.md @@ -46,7 +46,7 @@ services: - "1026:1026" depends_on: - mongo - command: -dbhost mongo + command: -dbURI mongodb://mongo mongo: image: mongo:6.0 diff --git a/etc/init.d/contextBroker.centos b/etc/init.d/contextBroker.centos index e507ef4046..cf9d2df29f 100644 --- a/etc/init.d/contextBroker.centos +++ b/etc/init.d/contextBroker.centos @@ -79,19 +79,22 @@ BROKER_OPTS="-port ${BROKER_PORT} \ -logDir ${BROKER_LOG_DIR} \ -logLevel ${BROKER_LOG_LEVEL} \ -pidpath ${BROKER_PID_FILE} \ - -dbhost ${BROKER_DATABASE_HOST} \ + -dbURI mongodb://${BROKER_DATABASE_HOST} \ -db ${BROKER_DATABASE_NAME} \ ${BROKER_EXTRA_OPS}" # Optional parameters +# FIXME: -dbuser option is deprecated. This needs to be migrated to -dbURI usage if [[ ! -z "${BROKER_DATABASE_USER}" ]]; then BROKER_OPTS="${BROKER_OPTS} -dbuser ${BROKER_DATABASE_USER}" fi +# FIXME: -dbpwd needs ${PWD} in the -dbURI CLI. This needs to be adapted if [[ ! -z "${BROKER_DATABASE_PASSWORD}" ]]; then BROKER_OPTS="${BROKER_OPTS} -dbpwd ${BROKER_DATABASE_PASSWORD}" fi +# FIXME: -rplsSet option is deprecated. This needs to be migrated to -dbURI usage if [[ ! -z "${BROKER_DATABASE_RPLSET}" ]]; then BROKER_OPTS="${BROKER_OPTS} -rplSet ${BROKER_DATABASE_RPLSET}" fi diff --git a/etc/init.d/contextBroker.ubuntu b/etc/init.d/contextBroker.ubuntu index d3cd38e1e2..d86fab69ef 100755 --- a/etc/init.d/contextBroker.ubuntu +++ b/etc/init.d/contextBroker.ubuntu @@ -58,7 +58,7 @@ SCRIPTNAME=/etc/init.d/$NAME DAEMON_ARGS="-port ${BROKER_PORT} \ -logDir ${BROKER_LOG_DIR} \ -pidpath ${BROKER_PID_FILE} \ - -dbhost ${BROKER_DATABASE_HOST} \ + -dbURI mongodb://${BROKER_DATABASE_HOST} \ -db ${BROKER_DATABASE_NAME} \ ${BROKER_EXTRA_OPS}" diff --git a/makefile b/makefile index 594f21f2d3..8e940023ae 100644 --- a/makefile +++ b/makefile @@ -216,9 +216,9 @@ build_unit_test: prepare_unit_test unit_test: build_unit_test @echo '------------------------------- unit_test starts ---------------------------------' if [ -z "${TEST_FILTER}" ]; then \ - BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbhost ${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml; \ + BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml; \ else \ - BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbhost ${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml --gtest_filter=${TEST_FILTER}; \ + BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml --gtest_filter=${TEST_FILTER}; \ fi @echo '------------------------------- unit_test ended ---------------------------------' diff --git a/test/acceptance/behave/components/common_steps/initial_steps.py b/test/acceptance/behave/components/common_steps/initial_steps.py index e033cf6d98..f662dd17e3 100644 --- a/test/acceptance/behave/components/common_steps/initial_steps.py +++ b/test/acceptance/behave/components/common_steps/initial_steps.py @@ -82,7 +82,7 @@ def start_context_broker(context): __logger__.debug("Starting contextBroker per command line interface...") props_cb["CB_EXTRA_OPS"] = props_cb["CB_EXTRA_OPS"].replace('"', "") # hint: the -harakiri option is used to kill contextBroker (must be compiled in DEBUG mode) - command = "contextBroker -port %s -logDir %s -pidpath %s -dbhost %s -db %s %s -harakiri" %\ + command = "contextBroker -port %s -logDir %s -pidpath %s -dbURI mondodb://%s -db %s %s -harakiri" %\ (props_cb["CB_PORT"], props_cb["CB_LOG_FILE"], props_cb["CB_PID_FILE"], props_mongo["MONGO_HOST"], props_mongo["MONGO_DATABASE"], props_cb["CB_EXTRA_OPS"]) __logger__.debug("command: %s" % command) diff --git a/test/functionalTest/harnessFunctions.sh b/test/functionalTest/harnessFunctions.sh index d1de8a684d..6a9886df9f 100644 --- a/test/functionalTest/harnessFunctions.sh +++ b/test/functionalTest/harnessFunctions.sh @@ -377,38 +377,38 @@ function localBrokerStart() if [ "$role" == "CB" ] then port=$CB_PORT - CB_START_CMD="$CB_START_CMD_PREFIX -port $CB_PORT -pidpath $CB_PID_FILE -dbhost $dbHost:$dbPort -db $CB_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption $extraParams" + CB_START_CMD="$CB_START_CMD_PREFIX -port $CB_PORT -pidpath $CB_PID_FILE -dbURI mongodb://$dbHost:$dbPort -db $CB_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption $extraParams" elif [ "$role" == "CBHA" ] then # CB-HA broker uses same database than main CB mkdir -p $CBHA_LOG_DIR port=$CBHA_PORT - CB_START_CMD="$CB_START_CMD_PREFIX -port $CBHA_PORT -pidpath $CBHA_PID_FILE -dbhost $dbHost:$dbPort -db $CB_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CBHA_LOG_DIR $extraParams" + CB_START_CMD="$CB_START_CMD_PREFIX -port $CBHA_PORT -pidpath $CBHA_PID_FILE -dbURI mongodb://$dbHost:$dbPort -db $CB_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CBHA_LOG_DIR $extraParams" elif [ "$role" == "CP1" ] then mkdir -p $CP1_LOG_DIR port=$CP1_PORT - CB_START_CMD="$CB_START_CMD_PREFIX -port $CP1_PORT -pidpath $CP1_PID_FILE -dbhost $dbHost:$dbPort -db $CP1_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP1_LOG_DIR $extraParams" + CB_START_CMD="$CB_START_CMD_PREFIX -port $CP1_PORT -pidpath $CP1_PID_FILE -dbURI mongodb://$dbHost:$dbPort -db $CP1_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP1_LOG_DIR $extraParams" elif [ "$role" == "CP2" ] then mkdir -p $CP2_LOG_DIR port=$CP2_PORT - CB_START_CMD="$CB_START_CMD_PREFIX -port $CP2_PORT -pidpath $CP2_PID_FILE -dbhost $dbHost:$dbPort -db $CP2_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP2_LOG_DIR $extraParams" + CB_START_CMD="$CB_START_CMD_PREFIX -port $CP2_PORT -pidpath $CP2_PID_FILE -dbURI mongodb://$dbHost:$dbPort -db $CP2_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP2_LOG_DIR $extraParams" elif [ "$role" == "CP3" ] then mkdir -p $CP3_LOG_DIR port=$CP3_PORT - CB_START_CMD="$CB_START_CMD_PREFIX -port $CP3_PORT -pidpath $CP3_PID_FILE -dbhost $dbHost:$dbPort -db $CP3_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP3_LOG_DIR $extraParams" + CB_START_CMD="$CB_START_CMD_PREFIX -port $CP3_PORT -pidpath $CP3_PID_FILE -dbURI mongodb://$dbHost:$dbPort -db $CP3_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP3_LOG_DIR $extraParams" elif [ "$role" == "CP4" ] then mkdir -p $CP4_LOG_DIR port=$CP4_PORT - CB_START_CMD="$CB_START_CMD_PREFIX -port $CP4_PORT -pidpath $CP4_PID_FILE -dbhost $dbHost:$dbPort -db $CP4_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP4_LOG_DIR $extraParams" + CB_START_CMD="$CB_START_CMD_PREFIX -port $CP4_PORT -pidpath $CP4_PID_FILE -dbURI mongodb://$dbHost:$dbPort -db $CP4_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP4_LOG_DIR $extraParams" elif [ "$role" == "CP5" ] then mkdir -p $CP5_LOG_DIR port=$CP5_PORT - CB_START_CMD="$CB_START_CMD_PREFIX -port $CP5_PORT -pidpath $CP5_PID_FILE -dbhost $dbHost:$dbPort -db $CP5_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP5_LOG_DIR $extraParams" + CB_START_CMD="$CB_START_CMD_PREFIX -port $CP5_PORT -pidpath $CP5_PID_FILE -dbURI mongodb://$dbHost:$dbPort -db $CP5_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP5_LOG_DIR $extraParams" fi From 2c15d46feba4f1a1b6131458a50a458d02b90b6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 7 Feb 2024 14:33:27 +0100 Subject: [PATCH 135/390] FIX broken utest --- test/unittests/main_UnitTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unittests/main_UnitTest.cpp b/test/unittests/main_UnitTest.cpp index 7d7b9bc991..3b7718b0c1 100644 --- a/test/unittests/main_UnitTest.cpp +++ b/test/unittests/main_UnitTest.cpp @@ -117,7 +117,7 @@ PaArgument paArgs[] = { "-dbAuthDb", authDb, "DB_AUTH_DB", PaString, PaOpt, (int64_t) "", PaNL, PaNL, "" }, { "-dbSSL", &dbSSL, "DB_AUTH_SSL", PaBool, PaOpt, false, false, true, "" }, { "-db", dbName, "DB", PaString, PaOpt, (int64_t) "orion", PaNL, PaNL, "" }, - { "-dbTimeout", &dbTimeout, "DB_TIMEOUT", PaInt64, PaOpt, 10000, PaNL, PaNL, "" }, + { "-dbTimeout", &dbTimeout, "DB_TIMEOUT", PaInt64, PaOpt, 0, PaNL, PaNL, "" }, { "-dbPoolSize", &dbPoolSize, "DB_POOL_SIZE", PaInt, PaOpt, 10, 1, 10000, "" }, { "-writeConcern", &writeConcern, "WRITE_CONCERN", PaInt, PaOpt, 1, 0, 1, "" }, { "--gtest_filter=", gtest_filter, "", PaString, PaOpt, (int64_t) "", PaNL, PaNL, "" }, From e696631327e1b0f820bb00064958aef8f32df537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 7 Feb 2024 16:58:22 +0100 Subject: [PATCH 136/390] FIX JexlManager init() and release() methods --- src/app/contextBroker/contextBroker.cpp | 2 + src/lib/jexl/JexlManager.cpp | 106 ++++++++++++++++-------- src/lib/jexl/JexlManager.h | 6 +- 3 files changed, 80 insertions(+), 34 deletions(-) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index ccade9df1c..992e9264ea 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -598,6 +598,8 @@ void exitFunc(void) curl_context_cleanup(); curl_global_cleanup(); + jexlMgr.release(); + #ifdef DEBUG // valgrind pass is done using DEBUG compilation, so we have to take care with // the cache releasing to avoid false positives. In production this is not really diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index e14017073c..00ba1331b6 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -28,56 +28,96 @@ #include "jexl/JexlManager.h" #include "logMsg/logMsg.h" + + +/* **************************************************************************** +* +* capturePythonError - +*/ +static const char* capturePythonError() +{ + if (PyErr_Occurred()) + { + PyObject* ptype; + PyObject* pvalue; + PyObject* ptraceback; + + // Fetch the exception type, value, and traceback + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + + if (pvalue != NULL) + { + PyObject* str_obj = PyObject_Str(pvalue); + const char* error_message = PyUnicode_AsUTF8(str_obj); + + // Release the Python objects + Py_XDECREF(str_obj); + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + + return error_message; + } + } + + PyErr_Clear(); + return ""; +} + + + /* **************************************************************************** * * JexlManager::init - */ void JexlManager::init(void) { + jexl_module = NULL; + jexl_engine = NULL; + if (sem_init(&sem, 0, 1) == -1) { LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); } - // FIXME PR: Py_Finalize() has to be be called in the proper place - LM_T(LmtJexl, ("Python interpreter is about to be initialized")); Py_Initialize(); LM_T(LmtJexl, ("Python interpreter has been initialized")); - // FIXME PR: proper Py_XDECREF() has to be called in the proper place - // FIXME PR: capture error and move it to char* to print in LM_X - PyObject* jexl_module = PyImport_ImportModule("pyjexl"); + jexl_module = PyImport_ImportModule("pyjexl"); + if (jexl_module == NULL) + { + const char* error = capturePythonError(); + LM_X(1, ("Fatal Error (error importing jexl_module: %s)", error)); + } LM_T(LmtJexl, ("jexl module has been loaded")); - // Create JEXL engine - PyObject* jexl_engine = PyObject_CallMethod(jexl_module, "JEXL", NULL); + jexl_engine = PyObject_CallMethod(jexl_module, "JEXL", NULL); + if (jexl_engine == NULL) + { + const char* error = capturePythonError(); + LM_X(1, ("Fatal Error (error creating jexl_engine: %s)", error)); + } LM_T(LmtJexl, ("jexl engine has been created")); +} + +/* **************************************************************************** +* +* JexlManager::release - +*/ +void JexlManager::release(void) +{ + if (jexl_engine != NULL) + { + Py_XDECREF(jexl_engine); + LM_T(LmtJexl, ("jexl engine has been freed")); + } + + if (jexl_module != NULL) + { + Py_XDECREF(jexl_module); + LM_T(LmtJexl, ("jexl module has been freed")); + } - // Create expression and context - PyObject* expression = Py_BuildValue("s", "x + y"); - LM_T(LmtJexl, ("jexl expression has been built")); - PyObject* context = Py_BuildValue("{s:i, s:i}", "x", 4, "y", 11); - LM_T(LmtJexl, ("jexl context has been built")); - - // Call evaluate method - PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, context); - LM_T(LmtJexl, ("jexl evaluation is done")); - - // Print result - PyObject* repr = PyObject_Repr(result); - const char* result_str = PyUnicode_AsUTF8(repr); - LM_T(LmtJexl, ("jexl evaluation result is obtainted")); - LM_T(LmtJexl, ("jexl result: %s", result_str)); - - // Free resources - Py_XDECREF(repr); - Py_XDECREF(result); - Py_XDECREF(context); - Py_XDECREF(expression); - Py_XDECREF(jexl_engine); - Py_XDECREF(jexl_module); - LM_T(LmtJexl, ("all has been freed")); - - // Finalize the Python interpreter Py_Finalize(); + LM_T(LmtJexl, ("Python interpreter has been finalized")); } \ No newline at end of file diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h index d22f442392..6d430154ec 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/jexl/JexlManager.h @@ -26,6 +26,7 @@ * Author: Fermin Galan */ +#include #include /* **************************************************************************** @@ -35,10 +36,13 @@ class JexlManager { private: - sem_t sem; + PyObject* jexl_module; + PyObject* jexl_engine; + sem_t sem; public: void init(void); + void release(void); }; #endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file From 0c5e76fd9561483a9d8098ec2a596a698f6d6972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 7 Feb 2024 17:40:47 +0100 Subject: [PATCH 137/390] FIX reduce buildReplacementsMap invocations --- src/lib/ngsiNotify/Notifier.cpp | 43 ++++++++++----------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index f452b51ac2..353c157d7c 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -121,8 +121,7 @@ static bool setPayload const std::string& notifPayload, const SubscriptionId& subscriptionId, Entity& en, - const std::string& service, - const std::string& token, + std::map* replacementsP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -171,9 +170,7 @@ static bool setPayload } else { - std::map replacements; - buildReplacementsMap(en, service, token, &replacements); - if (!macroSubstitute(payloadP, notifPayload, &replacements, "")) + if (!macroSubstitute(payloadP, notifPayload, replacementsP, "")) { return false; } @@ -197,19 +194,12 @@ static bool setPayload static bool setJsonPayload ( orion::CompoundValueNode* json, - const Entity& en, - const std::string& service, - const std::string& token, + std::map* replacementsP, std::string* payloadP, std::string* mimeTypeP ) { - // Prepare a map for macro replacements. We firstly tried to pass Entity object to - // orion::CompoundValueNode()::toJson(), but the include Entity.h in CompoundValueNode.h - // makes compiler to cry (maybe some kind of circular dependency problem?) - std::map replacements; - buildReplacementsMap(en, service, token, &replacements); - *payloadP = json->toJson(&replacements); + *payloadP = json->toJson(replacementsP); *mimeTypeP = "application/json"; // this can be overriden by headers field return true; } @@ -247,8 +237,7 @@ static bool setNgsiPayload const Entity& ngsi, const SubscriptionId& subscriptionId, Entity& en, - const std::string& service, - const std::string& token, + std::map* replacementsP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -256,12 +245,6 @@ static bool setNgsiPayload RenderFormat renderFormat ) { - // Prepare a map for macro replacements. We firstly tried to pass Entity object to - // orion::CompoundValueNode()::toJson(), but the include Entity.h in CompoundValueNode.h - // makes compiler to cry (maybe some kind of circular dependency problem?) - std::map replacements; - buildReplacementsMap(en, service, token, &replacements); - NotifyContextRequest ncr; ContextElementResponse cer; @@ -273,7 +256,7 @@ static bool setNgsiPayload else { // If id is not found in the replacements macro, we use en.id. - effectiveId = removeQuotes(smartStringValue(ngsi.id, &replacements, '"' + en.id + '"')); + effectiveId = removeQuotes(smartStringValue(ngsi.id, replacementsP, '"' + en.id + '"')); } std::string effectiveType; @@ -284,7 +267,7 @@ static bool setNgsiPayload else { // If type is not found in the replacements macro, we use en.type. - effectiveType = removeQuotes(smartStringValue(ngsi.type, &replacements, '"' + en.type + '"')); + effectiveType = removeQuotes(smartStringValue(ngsi.type, replacementsP, '"' + en.type + '"')); } cer.entity.fill(effectiveId, effectiveType, en.isPattern, en.servicePath); @@ -310,11 +293,11 @@ static bool setNgsiPayload if ((renderFormat == NGSI_V2_SIMPLIFIEDNORMALIZED) || (renderFormat == NGSI_V2_SIMPLIFIEDKEYVALUES)) { - *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, &replacements); + *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, replacementsP); } else { - *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, &replacements); + *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, replacementsP); } return true; @@ -331,7 +314,7 @@ static SenderThreadParams* buildSenderParamsCustom const SubscriptionId& subscriptionId, ContextElementResponse* notifyCerP, const ngsiv2::Notification& notification, - const std::string& tenant, + const std::string& tenant, // FIXME PR: duplicated in service thread variable? long long maxFailsLimit, long long failsCounter, const std::string& xauthToken, @@ -396,7 +379,7 @@ static SenderThreadParams* buildSenderParamsCustom { bool includePayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.includePayload : notification.mqttInfo.includePayload); std::string notifPayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.payload : notification.mqttInfo.payload); - if (!setPayload(includePayload, notifPayload, subscriptionId, en, tenant, xauthToken, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) + if (!setPayload(includePayload, notifPayload, subscriptionId, en, &replacements, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) { // Warning already logged in macroSubstitute() return NULL; @@ -405,14 +388,14 @@ static SenderThreadParams* buildSenderParamsCustom else if (customPayloadType == ngsiv2::CustomPayloadType::Json) { orion::CompoundValueNode* json = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.json : notification.mqttInfo.json); - setJsonPayload(json, en, tenant, xauthToken, &payload, &mimeType); + setJsonPayload(json, &replacements, &payload, &mimeType); renderFormat = NGSI_V2_CUSTOM; } else // customPayloadType == ngsiv2::CustomPayloadType::Ngsi { // Important to use const& for Entity here. Otherwise problems may occur in the object release logic const Entity& ngsi = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.ngsi : notification.mqttInfo.ngsi); - if (!setNgsiPayload(ngsi, subscriptionId, en, tenant, xauthToken, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) + if (!setNgsiPayload(ngsi, subscriptionId, en, &replacements, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) { // Warning already logged in macroSubstitute() return NULL; From 92af4d4d9c3b0a9a7b2e9c7c5984b69d099fff1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 8 Feb 2024 08:58:05 +0100 Subject: [PATCH 138/390] FIX proper usage of tenant variable in buildReplacementsMap --- src/lib/ngsiNotify/Notifier.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 353c157d7c..aae1ecb34f 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -314,7 +314,7 @@ static SenderThreadParams* buildSenderParamsCustom const SubscriptionId& subscriptionId, ContextElementResponse* notifyCerP, const ngsiv2::Notification& notification, - const std::string& tenant, // FIXME PR: duplicated in service thread variable? + const std::string& tenant, long long maxFailsLimit, long long failsCounter, const std::string& xauthToken, @@ -337,7 +337,7 @@ static SenderThreadParams* buildSenderParamsCustom // Used by several macroSubstitute() calls along this function std::map replacements; - buildReplacementsMap(en, service, xauthToken, &replacements); + buildReplacementsMap(en, tenant, xauthToken, &replacements); // // 1. Verb/Method From 10386226b7fadea1ffa26d57b9fc84546455a597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 18:25:44 +0100 Subject: [PATCH 139/390] FIX replace map by JEXL context (compiles but still wip) --- src/lib/apiTypesV2/Entity.cpp | 9 +- src/lib/apiTypesV2/Entity.h | 5 +- src/lib/common/macroSubstitute.cpp | 32 +++---- src/lib/common/macroSubstitute.h | 8 +- src/lib/jexl/CMakeLists.txt | 4 +- src/lib/jexl/JexlContext.cpp | 83 +++++++++++++++++++ src/lib/jexl/JexlContext.h | 50 +++++++++++ src/lib/ngsi/ContextAttribute.cpp | 6 +- src/lib/ngsi/ContextAttribute.h | 3 +- src/lib/ngsi/ContextElementResponse.cpp | 4 +- src/lib/ngsi/ContextElementResponse.h | 3 +- src/lib/ngsi/ContextElementResponseVector.cpp | 4 +- src/lib/ngsi/ContextElementResponseVector.h | 3 +- src/lib/ngsi10/NotifyContextRequest.cpp | 8 +- src/lib/ngsi10/NotifyContextRequest.h | 2 +- src/lib/ngsiNotify/Notifier.cpp | 37 +++++---- src/lib/parse/CompoundValueNode.cpp | 8 +- src/lib/parse/CompoundValueNode.h | 4 +- 18 files changed, 210 insertions(+), 63 deletions(-) create mode 100644 src/lib/jexl/JexlContext.cpp create mode 100644 src/lib/jexl/JexlContext.h diff --git a/src/lib/apiTypesV2/Entity.cpp b/src/lib/apiTypesV2/Entity.cpp index 7c3fd7abba..5501c18e77 100644 --- a/src/lib/apiTypesV2/Entity.cpp +++ b/src/lib/apiTypesV2/Entity.cpp @@ -41,6 +41,7 @@ #include "rest/OrionError.h" #include "apiTypesV2/Entity.h" +#include "jexl/JexlContext.h" @@ -307,7 +308,7 @@ std::string Entity::toJson bool blacklist, const std::vector& metadataFilter, bool renderNgsiField, - std::map* replacementsP + JexlContext* jexlContextP ) { std::vector orderedAttrs; @@ -326,7 +327,7 @@ std::string Entity::toJson out = toJsonKeyvalues(orderedAttrs); break; default: // NGSI_V2_NORMALIZED - out = toJsonNormalized(orderedAttrs, metadataFilter, renderNgsiField, replacementsP); + out = toJsonNormalized(orderedAttrs, metadataFilter, renderNgsiField, jexlContextP); break; } @@ -443,7 +444,7 @@ std::string Entity::toJsonNormalized const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField, - std::map* replacementsP + JexlContext* jexlContextP ) { JsonObjectHelper jh; @@ -475,7 +476,7 @@ std::string Entity::toJsonNormalized for (unsigned int ix = 0; ix < orderedAttrs.size(); ix++) { ContextAttribute* caP = orderedAttrs[ix]; - jh.addRaw(caP->name, caP->toJson(metadataFilter, renderNgsiField, replacementsP)); + jh.addRaw(caP->name, caP->toJson(metadataFilter, renderNgsiField, jexlContextP)); } return jh.str(); diff --git a/src/lib/apiTypesV2/Entity.h b/src/lib/apiTypesV2/Entity.h index e4906bb998..1813faa0c1 100644 --- a/src/lib/apiTypesV2/Entity.h +++ b/src/lib/apiTypesV2/Entity.h @@ -32,6 +32,7 @@ #include "ngsi/ContextAttributeVector.h" #include "ngsi/EntityId.h" #include "rest/OrionError.h" +#include "jexl/JexlContext.h" @@ -91,7 +92,7 @@ class Entity bool blacklist, const std::vector& metadataFilter, bool renderNgsiField = false, - std::map* replacementsP = NULL); + JexlContext* jexlContext = NULL); std::string toJson(RenderFormat renderFormat, bool renderNgsiField = false); @@ -145,7 +146,7 @@ class Entity std::string toJsonNormalized(const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField = false, - std::map* replacementsP = NULL); + JexlContext* jexlContext = NULL); }; #endif // SRC_LIB_APITYPESV2_ENTITY_H_ diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index b17ac6487c..9ab9ac2131 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -40,13 +40,14 @@ * * Returns the effective string value, taking into account replacements */ -std::string smartStringValue(const std::string stringValue, std::map* replacementsP, const std::string notFoundDefault) +std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault) { // This code is pretty similar to the one in CompoundValueNode::toJson() // The program logic branching is the same, but the result at the end of each if-else // is different, which makes difficult to unify both them - if ((replacementsP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) + if ((jexlContextP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) { + /* FIXME PR: needs adaptation replacementsP -> jexlContextP // "Full replacement" case. In this case, the result is not always a string // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); @@ -59,13 +60,14 @@ std::string smartStringValue(const std::string stringValue, std::mapsecond; - } + }*/ + return ""; } - else if (replacementsP != NULL) + else if (jexlContextP != NULL) { // "Partial replacement" case. In this case, the result is always a string std::string effectiveValue; - if (!macroSubstitute(&effectiveValue, stringValue, replacementsP, "null")) + if (!macroSubstitute(&effectiveValue, stringValue, jexlContextP, "null")) { // error already logged in macroSubstitute, using stringValue itself as failsafe effectiveValue = stringValue; @@ -87,7 +89,7 @@ std::string smartStringValue(const std::string stringValue, std::mapname] = en.attributeVector[ix]->toJsonValue(); } -} +}*/ @@ -114,16 +116,17 @@ void buildReplacementsMap * * stringValueOrNothing - */ -static std::string stringValueOrNothing(std::map* replacementsP, const std::string key, const std::string& notFoundDefault) +static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault) { - std::map::iterator iter = replacementsP->find(key); + // FIXME PR: to from replacementsP to jexlContextP + /*std::map::iterator iter = replacementsP->find(key); if (iter == replacementsP->end()) { return notFoundDefault; } else { - // replacementP contents are prepared for "full replacement" case, so string values use + // replacementsP contents are prepared for "full replacement" case, so string values use // double quotes. But in this case we are in a "partial replacement" case, so we have // to remove them if we find them std::string value = iter->second; @@ -135,7 +138,8 @@ static std::string stringValueOrNothing(std::map* repl { return value; } - } + }*/ + return ""; } @@ -165,7 +169,7 @@ static std::string stringValueOrNothing(std::map* repl * Date: Mon Jun 19 16:33:29 2017 +0200 * */ -bool macroSubstitute(std::string* to, const std::string& from, std::map* replacementsP, const std::string& notFoundDefault) +bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexlContextP, const std::string& notFoundDefault) { // Initial size check: is the string to convert too big? // @@ -228,7 +232,7 @@ bool macroSubstitute(std::string* to, const std::string& from, std::map outReqMsgMaxSize) @@ -246,7 +250,7 @@ bool macroSubstitute(std::string* to, const std::string& from, std::mapsecond; std::string macro = "${" + macroName + "}"; - std::string value = stringValueOrNothing(replacementsP, macroName, notFoundDefault); + std::string value = stringValueOrNothing(jexlContextP, macroName, notFoundDefault); // We have to do the replace operation as many times as macro occurrences for (unsigned int ix = 0; ix < times; ix++) diff --git a/src/lib/common/macroSubstitute.h b/src/lib/common/macroSubstitute.h index 288c25540b..0483fc23ae 100644 --- a/src/lib/common/macroSubstitute.h +++ b/src/lib/common/macroSubstitute.h @@ -34,7 +34,7 @@ * smartStringValue - * */ -extern std::string smartStringValue(const std::string stringValue, std::map* replacementsP, const std::string notFoundDefault); +extern std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault); @@ -43,13 +43,13 @@ extern std::string smartStringValue(const std::string stringValue, std::map* replacementsP -); +);*/ @@ -58,6 +58,6 @@ extern void buildReplacementsMap * macroSubstitute - * */ -extern bool macroSubstitute(std::string* sP, const std::string& in, std::map* replacementsP, const std::string& notFoundDefault); +extern bool macroSubstitute(std::string* sP, const std::string& in, JexlContext* jexlContextP, const std::string& notFoundDefault); #endif // SRC_LIB_COMMON_MACROSUBSTITUTE_H_ diff --git a/src/lib/jexl/CMakeLists.txt b/src/lib/jexl/CMakeLists.txt index 67a7a10e24..abacbeb3ca 100644 --- a/src/lib/jexl/CMakeLists.txt +++ b/src/lib/jexl/CMakeLists.txt @@ -22,11 +22,13 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES JexlManager.cpp - jexlMgr.cpp + JexlContext.cpp + jexlMgr.cpp ) SET (HEADERS JexlManager.h + JexlContext.h jexlMgr.h ) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp new file mode 100644 index 0000000000..33dd5e877d --- /dev/null +++ b/src/lib/jexl/JexlContext.cpp @@ -0,0 +1,83 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "jexl/JexlContext.h" + +#include + +/* **************************************************************************** +* +* JexlContext::JexlContext - +*/ +JexlContext::JexlContext(const std::string& id, const std::string& type, const std::string& service, const std::string& servicePath, const std::string& token) +{ + jexl_context = PyDict_New(); + + if (jexl_context == NULL) + { + // FIXME PR: error control + } + + PyObject* value; + + value = Py_BuildValue("s", id.c_str()); + PyDict_SetItemString(jexl_context, "id", value); + Py_DECREF(value); + + value = Py_BuildValue("s", type.c_str()); + PyDict_SetItemString(jexl_context, "type", value); + Py_DECREF(value); + + value = Py_BuildValue("s", service.c_str()); + PyDict_SetItemString(jexl_context, "service", value); + Py_DECREF(value); + + value = Py_BuildValue("s", servicePath.c_str()); + PyDict_SetItemString(jexl_context, "servicePath", value); + Py_DECREF(value); + + value = Py_BuildValue("s", token.c_str()); + PyDict_SetItemString(jexl_context, "authToken", value); + Py_DECREF(value); + + /*for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) + { + // FIXME PR: the first argument in Py_BuildValue would depend on the attribute type, "s" is only for string type + value = Py_BuildValue("s", en.attributeVector[ix]->toJsonValue().c_str()); + PyDict_SetItemString(jexl_context, en.attributeVector[ix]->name.c_str(), value); + Py_DECREF(value); + }*/ +} + + + +/* **************************************************************************** +* +* JexlContext::~JexlContext - +*/ +JexlContext::~JexlContext() +{ + Py_XDECREF(jexl_context); +} diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h new file mode 100644 index 0000000000..bc5bbd27dd --- /dev/null +++ b/src/lib/jexl/JexlContext.h @@ -0,0 +1,50 @@ +#ifndef SRC_LIB_JEXL_JEXLCONTEXT_H_ +#define SRC_LIB_JEXL_JEXLCONTEXT_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include + +#include + +/* **************************************************************************** +* +* JexlContext - +*/ +class JexlContext +{ +private: + PyObject* jexl_context; + +public: + ~JexlContext(); + + JexlContext(const std::string& id, const std::string& type, const std::string& service, const std::string& servicePath, const std::string& token); +}; + + + +#endif // #define SRC_LIB_JEXL_JEXLCONTEXT_H_ diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 88681d879a..f4407a89fc 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1052,7 +1052,7 @@ void ContextAttribute::filterAndOrderMetadata * renderNgsiField true is used in custom notification payloads, which have some small differences * with regards to conventional rendering */ -std::string ContextAttribute::toJson(const std::vector& metadataFilter, bool renderNgsiField, std::map* replacementsP) +std::string ContextAttribute::toJson(const std::vector& metadataFilter, bool renderNgsiField, JexlContext* jexlContextP) { JsonObjectHelper jh; @@ -1085,7 +1085,7 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi // of DB entities) may lead to NULL, so the check is needed if (childToRenderP != NULL) { - jh.addRaw("value", childToRenderP->toJson(replacementsP)); + jh.addRaw("value", childToRenderP->toJson(jexlContextP)); } } else if (valueType == orion::ValueTypeNumber) @@ -1101,7 +1101,7 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi } else if (valueType == orion::ValueTypeString) { - jh.addRaw("value", smartStringValue(stringValue, replacementsP, "null")); + jh.addRaw("value", smartStringValue(stringValue, jexlContextP, "null")); } else if (valueType == orion::ValueTypeBoolean) { diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index 58d7ed39f1..1ca58a0a98 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -37,6 +37,7 @@ #include "parse/CompoundValueNode.h" #include "rest/HttpStatusCode.h" #include "mongoDriver/BSONObjBuilder.h" +#include "jexl/JexlContext.h" @@ -107,7 +108,7 @@ typedef struct ContextAttribute std::string toJsonV1AsNameString(bool comma); - std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, std::map* replacementsP = NULL); + std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, JexlContext* jexlContextP = NULL); std::string toJsonValue(void); diff --git a/src/lib/ngsi/ContextElementResponse.cpp b/src/lib/ngsi/ContextElementResponse.cpp index ba9d3ae0b2..262d0f2a4c 100644 --- a/src/lib/ngsi/ContextElementResponse.cpp +++ b/src/lib/ngsi/ContextElementResponse.cpp @@ -215,12 +215,12 @@ std::string ContextElementResponse::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP + JexlContext* jexlContextP ) { std::string out; - out = entity.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, false, replacementsP); + out = entity.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, false, jexlContextP); return out; } diff --git a/src/lib/ngsi/ContextElementResponse.h b/src/lib/ngsi/ContextElementResponse.h index 87b3bc17b1..471a8e66e6 100644 --- a/src/lib/ngsi/ContextElementResponse.h +++ b/src/lib/ngsi/ContextElementResponse.h @@ -33,6 +33,7 @@ #include "ngsi/StringList.h" #include "ngsi/ContextAttribute.h" #include "apiTypesV2/Entity.h" +#include "jexl/JexlContext.h" #include "mongoDriver/BSONObj.h" @@ -82,7 +83,7 @@ typedef struct ContextElementResponse const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP); + JexlContext* jexlContext); void applyUpdateOperators(void); diff --git a/src/lib/ngsi/ContextElementResponseVector.cpp b/src/lib/ngsi/ContextElementResponseVector.cpp index e044e9779d..03fd35741a 100644 --- a/src/lib/ngsi/ContextElementResponseVector.cpp +++ b/src/lib/ngsi/ContextElementResponseVector.cpp @@ -119,14 +119,14 @@ std::string ContextElementResponseVector::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP + JexlContext* jexlContextP ) { JsonVectorHelper jvh; for (unsigned int ix = 0; ix < vec.size(); ++ix) { - jvh.addRaw(vec[ix]->toJson(renderFormat, attrsFilter, blacklist, metadataFilter, replacementsP)); + jvh.addRaw(vec[ix]->toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP)); } return jvh.str(); diff --git a/src/lib/ngsi/ContextElementResponseVector.h b/src/lib/ngsi/ContextElementResponseVector.h index 92f74e5ff6..16cb7afb7c 100644 --- a/src/lib/ngsi/ContextElementResponseVector.h +++ b/src/lib/ngsi/ContextElementResponseVector.h @@ -31,6 +31,7 @@ #include "ngsi/ContextElementResponse.h" #include "apiTypesV2/EntityVector.h" #include "common/RenderFormat.h" +#include "jexl/JexlContext.h" @@ -60,7 +61,7 @@ typedef struct ContextElementResponseVector const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP); + JexlContext* jexlContextP); void push_back(ContextElementResponse* item); unsigned int size(void) const; ContextElementResponse* lookup(Entity* eP, HttpStatusCode code = SccNone); diff --git a/src/lib/ngsi10/NotifyContextRequest.cpp b/src/lib/ngsi10/NotifyContextRequest.cpp index b9b4ac5af7..30122bfcb0 100644 --- a/src/lib/ngsi10/NotifyContextRequest.cpp +++ b/src/lib/ngsi10/NotifyContextRequest.cpp @@ -80,7 +80,7 @@ std::string NotifyContextRequest::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP + JexlContext* jexlContextP ) { if ((renderFormat != NGSI_V2_NORMALIZED) && (renderFormat != NGSI_V2_KEYVALUES) && (renderFormat != NGSI_V2_VALUES) && (renderFormat != NGSI_V2_SIMPLIFIEDKEYVALUES) && (renderFormat != NGSI_V2_SIMPLIFIEDNORMALIZED)) @@ -100,7 +100,7 @@ std::string NotifyContextRequest::toJson else { std::string out; - out += contextElementResponseVector[0]->toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, replacementsP); + out += contextElementResponseVector[0]->toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, jexlContextP); return out; } } @@ -114,7 +114,7 @@ std::string NotifyContextRequest::toJson else { std::string out; - out += contextElementResponseVector[0]->toJson(NGSI_V2_KEYVALUES, attrsFilter, blacklist, metadataFilter, replacementsP); + out += contextElementResponseVector[0]->toJson(NGSI_V2_KEYVALUES, attrsFilter, blacklist, metadataFilter, jexlContextP); return out; } } @@ -123,7 +123,7 @@ std::string NotifyContextRequest::toJson JsonObjectHelper jh; jh.addString("subscriptionId", subscriptionId.get()); - jh.addRaw("data", contextElementResponseVector.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, replacementsP)); + jh.addRaw("data", contextElementResponseVector.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP)); return jh.str(); } } diff --git a/src/lib/ngsi10/NotifyContextRequest.h b/src/lib/ngsi10/NotifyContextRequest.h index c4043704d5..80e4a19c8d 100644 --- a/src/lib/ngsi10/NotifyContextRequest.h +++ b/src/lib/ngsi10/NotifyContextRequest.h @@ -53,7 +53,7 @@ typedef struct NotifyContextRequest const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - std::map* replacementsP = NULL); + JexlContext* jexlContextP = NULL); std::string check(ApiVersion apiVersion, const std::string& predetectedError); void release(void); NotifyContextRequest* clone(void); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index aae1ecb34f..32db3e2adc 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -121,7 +121,7 @@ static bool setPayload const std::string& notifPayload, const SubscriptionId& subscriptionId, Entity& en, - std::map* replacementsP, + JexlContext* jexlContextP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -170,7 +170,7 @@ static bool setPayload } else { - if (!macroSubstitute(payloadP, notifPayload, replacementsP, "")) + if (!macroSubstitute(payloadP, notifPayload, jexlContextP, "")) { return false; } @@ -194,12 +194,12 @@ static bool setPayload static bool setJsonPayload ( orion::CompoundValueNode* json, - std::map* replacementsP, + JexlContext* jexlContextP, std::string* payloadP, std::string* mimeTypeP ) { - *payloadP = json->toJson(replacementsP); + *payloadP = json->toJson(jexlContextP); *mimeTypeP = "application/json"; // this can be overriden by headers field return true; } @@ -237,7 +237,7 @@ static bool setNgsiPayload const Entity& ngsi, const SubscriptionId& subscriptionId, Entity& en, - std::map* replacementsP, + JexlContext* jexlContextP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -256,7 +256,7 @@ static bool setNgsiPayload else { // If id is not found in the replacements macro, we use en.id. - effectiveId = removeQuotes(smartStringValue(ngsi.id, replacementsP, '"' + en.id + '"')); + effectiveId = removeQuotes(smartStringValue(ngsi.id, jexlContextP, '"' + en.id + '"')); } std::string effectiveType; @@ -267,7 +267,7 @@ static bool setNgsiPayload else { // If type is not found in the replacements macro, we use en.type. - effectiveType = removeQuotes(smartStringValue(ngsi.type, replacementsP, '"' + en.type + '"')); + effectiveType = removeQuotes(smartStringValue(ngsi.type, jexlContextP, '"' + en.type + '"')); } cer.entity.fill(effectiveId, effectiveType, en.isPattern, en.servicePath); @@ -293,11 +293,11 @@ static bool setNgsiPayload if ((renderFormat == NGSI_V2_SIMPLIFIEDNORMALIZED) || (renderFormat == NGSI_V2_SIMPLIFIEDKEYVALUES)) { - *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, replacementsP); + *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP); } else { - *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, replacementsP); + *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, jexlContextP); } return true; @@ -336,8 +336,9 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - std::map replacements; - buildReplacementsMap(en, tenant, xauthToken, &replacements); + //std::map replacements; + //buildReplacementsMap(en, tenant, xauthToken, &replacements); + JexlContext jexlContext(en.id, en.type, tenant, en.servicePath, xauthToken); // // 1. Verb/Method @@ -363,7 +364,7 @@ static SenderThreadParams* buildSenderParamsCustom // 2. URL // std::string notifUrl = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.url : notification.mqttInfo.url); - if (macroSubstitute(&url, notifUrl, &replacements, "") == false) + if (macroSubstitute(&url, notifUrl, &jexlContext, "") == false) { // Warning already logged in macroSubstitute() return NULL; @@ -379,7 +380,7 @@ static SenderThreadParams* buildSenderParamsCustom { bool includePayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.includePayload : notification.mqttInfo.includePayload); std::string notifPayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.payload : notification.mqttInfo.payload); - if (!setPayload(includePayload, notifPayload, subscriptionId, en, &replacements, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) + if (!setPayload(includePayload, notifPayload, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) { // Warning already logged in macroSubstitute() return NULL; @@ -388,14 +389,14 @@ static SenderThreadParams* buildSenderParamsCustom else if (customPayloadType == ngsiv2::CustomPayloadType::Json) { orion::CompoundValueNode* json = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.json : notification.mqttInfo.json); - setJsonPayload(json, &replacements, &payload, &mimeType); + setJsonPayload(json, &jexlContext, &payload, &mimeType); renderFormat = NGSI_V2_CUSTOM; } else // customPayloadType == ngsiv2::CustomPayloadType::Ngsi { // Important to use const& for Entity here. Otherwise problems may occur in the object release logic const Entity& ngsi = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.ngsi : notification.mqttInfo.ngsi); - if (!setNgsiPayload(ngsi, subscriptionId, en, &replacements, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) + if (!setNgsiPayload(ngsi, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) { // Warning already logged in macroSubstitute() return NULL; @@ -414,7 +415,7 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &replacements, "") == false) || (macroSubstitute(&value, it->second, &replacements, "") == false)) + if ((macroSubstitute(&key, it->first, &jexlContext, "") == false) || (macroSubstitute(&value, it->second, &jexlContext, "") == false)) { // Warning already logged in macroSubstitute() return NULL; @@ -440,7 +441,7 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &replacements, "") == false) || (macroSubstitute(&value, it->second, &replacements, "") == false)) + if ((macroSubstitute(&key, it->first, &jexlContext, "") == false) || (macroSubstitute(&value, it->second, &jexlContext, "") == false)) { // Warning already logged in macroSubstitute() return NULL; @@ -505,7 +506,7 @@ static SenderThreadParams* buildSenderParamsCustom // 8. Topic (only in the case of MQTT notifications) if (notification.type == ngsiv2::MqttNotification) { - if (macroSubstitute(&topic, notification.mqttInfo.topic, &replacements, "") == false) + if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "") == false) { // Warning already logged in macroSubstitute() return NULL; diff --git a/src/lib/parse/CompoundValueNode.cpp b/src/lib/parse/CompoundValueNode.cpp index 454c239383..6dfb6f59d4 100644 --- a/src/lib/parse/CompoundValueNode.cpp +++ b/src/lib/parse/CompoundValueNode.cpp @@ -612,7 +612,7 @@ bool CompoundValueNode::equal(const orion::BSONElement& be) * CompoundValueNode:toJson * */ -std::string CompoundValueNode::toJson(std::map* replacementsP) +std::string CompoundValueNode::toJson(JexlContext* jexlContextP) { std::string out; JsonVectorHelper jvh; @@ -621,7 +621,7 @@ std::string CompoundValueNode::toJson(std::map* replac switch (valueType) { case orion::ValueTypeString: - return smartStringValue(stringValue, replacementsP, "null"); + return smartStringValue(stringValue, jexlContextP, "null"); case orion::ValueTypeNumber: return double2string(numberValue); @@ -635,14 +635,14 @@ std::string CompoundValueNode::toJson(std::map* replac case orion::ValueTypeVector: for (unsigned int ix = 0; ix < childV.size(); ix++) { - jvh.addRaw(childV[ix]->toJson(replacementsP)); + jvh.addRaw(childV[ix]->toJson(jexlContextP)); } return jvh.str(); case orion::ValueTypeObject: for (unsigned int ix = 0; ix < childV.size(); ix++) { - joh.addRaw(childV[ix]->name, childV[ix]->toJson(replacementsP)); + joh.addRaw(childV[ix]->name, childV[ix]->toJson(jexlContextP)); } return joh.str(); diff --git a/src/lib/parse/CompoundValueNode.h b/src/lib/parse/CompoundValueNode.h index b46aa2c461..12fd070a31 100644 --- a/src/lib/parse/CompoundValueNode.h +++ b/src/lib/parse/CompoundValueNode.h @@ -35,6 +35,8 @@ #include "mongoDriver/BSONElement.h" +#include "jexl/JexlContext.h" + namespace orion { @@ -117,7 +119,7 @@ class CompoundValueNode bool equal(const orion::BSONElement& be); std::string finish(void); - std::string toJson(std::map* replacementsP = NULL); + std::string toJson(JexlContext* jexlContextP = NULL); void shortShow(const std::string& indent); void show(const std::string& indent); From cda7164eae804cb35dc5354f4ed49600cfb0ca91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 18:46:25 +0100 Subject: [PATCH 140/390] FIX improve JexlContext class --- src/lib/jexl/JexlContext.cpp | 28 ++++++++++++++++++++-------- src/lib/jexl/JexlContext.h | 12 +++++++++++- src/lib/ngsiNotify/Notifier.cpp | 4 ++++ 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index 33dd5e877d..5e8d76441c 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -31,7 +31,14 @@ * * JexlContext::JexlContext - */ -JexlContext::JexlContext(const std::string& id, const std::string& type, const std::string& service, const std::string& servicePath, const std::string& token) +JexlContext::JexlContext +( + const std::string& id, + const std::string& type, + const std::string& service, + const std::string& servicePath, + const std::string& token +) { jexl_context = PyDict_New(); @@ -61,14 +68,19 @@ JexlContext::JexlContext(const std::string& id, const std::string& type, const s value = Py_BuildValue("s", token.c_str()); PyDict_SetItemString(jexl_context, "authToken", value); Py_DECREF(value); +} - /*for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) - { - // FIXME PR: the first argument in Py_BuildValue would depend on the attribute type, "s" is only for string type - value = Py_BuildValue("s", en.attributeVector[ix]->toJsonValue().c_str()); - PyDict_SetItemString(jexl_context, en.attributeVector[ix]->name.c_str(), value); - Py_DECREF(value); - }*/ + + +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, const std::string& _value) +{ + PyObject* value = Py_BuildValue("s", _value.c_str()); + PyDict_SetItemString(jexl_context, key.c_str(), value); + Py_DECREF(value); } diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index bc5bbd27dd..88ef424896 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -30,6 +30,7 @@ #include + /* **************************************************************************** * * JexlContext - @@ -42,7 +43,16 @@ class JexlContext public: ~JexlContext(); - JexlContext(const std::string& id, const std::string& type, const std::string& service, const std::string& servicePath, const std::string& token); + JexlContext + ( + const std::string& id, + const std::string& type, + const std::string& service, + const std::string& servicePath, + const std::string& token + ); + + void add(const std::string& key, const std::string& value); }; diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 32db3e2adc..f44d4addf8 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -339,6 +339,10 @@ static SenderThreadParams* buildSenderParamsCustom //std::map replacements; //buildReplacementsMap(en, tenant, xauthToken, &replacements); JexlContext jexlContext(en.id, en.type, tenant, en.servicePath, xauthToken); + for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) + { + jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); + } // // 1. Verb/Method From e1389f6e0373f7aa7546be4ad874beca6dba7698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 19:26:43 +0100 Subject: [PATCH 141/390] FIX improve JexlManager and JexlContext classes --- src/lib/jexl/JexlContext.cpp | 28 +++++++++++++++++++++++++++- src/lib/jexl/JexlContext.h | 4 +++- src/lib/jexl/JexlManager.cpp | 35 +++++++++++++++++++++++++++++++++++ src/lib/jexl/JexlManager.h | 7 +++++-- 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index 5e8d76441c..b9e5968cbe 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -71,6 +71,16 @@ JexlContext::JexlContext } +/* **************************************************************************** +* +* JexlContext::get - +*/ +PyObject* JexlContext::get(void) +{ + return jexl_context; +} + + /* **************************************************************************** * @@ -85,6 +95,22 @@ void JexlContext::add(const std::string& key, const std::string& _value) +/* **************************************************************************** +* +* JexlContext::hasKey - +*/ +bool JexlContext::hasKey(const std::string& key) +{ + // Check if the key exists in the jexl_context dictionary + PyObject* keyObject = PyUnicode_FromString(key.c_str()); + int result = PyDict_Contains(jexl_context, keyObject); + Py_DECREF(keyObject); + + return result == 1; // Return true if key exists, false otherwise +} + + + /* **************************************************************************** * * JexlContext::~JexlContext - @@ -92,4 +118,4 @@ void JexlContext::add(const std::string& key, const std::string& _value) JexlContext::~JexlContext() { Py_XDECREF(jexl_context); -} +} \ No newline at end of file diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index 88ef424896..77d6083c07 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -52,7 +52,9 @@ class JexlContext const std::string& token ); - void add(const std::string& key, const std::string& value); + PyObject* get(void); + void add(const std::string& key, const std::string& value); + bool hasKey(const std::string& key); }; diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 00ba1331b6..4ad612d391 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -100,6 +100,41 @@ void JexlManager::init(void) LM_T(LmtJexl, ("jexl engine has been created")); } + + +/* **************************************************************************** +* +* JexlManager::evaluate - +*/ +std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) +{ + PyObject* expression = Py_BuildValue("s", _expression); + if (expression == NULL) + { + // FIXME PR: manage error + return "null"; + } + + PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, jexlContextP->get()); + if (result == NULL) + { + // FIXME PR: manage error + return "null"; + } + + PyObject* repr = PyObject_Repr(result); + if (repr == NULL) + { + // FIXME PR: manage error + return "null"; + } + + const char* result_str = PyUnicode_AsUTF8(repr); + return std::string(result_str); +} + + + /* **************************************************************************** * * JexlManager::release - diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h index 6d430154ec..89cadcf534 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/jexl/JexlManager.h @@ -29,6 +29,8 @@ #include #include +#include "jexl/JexlContext.h" + /* **************************************************************************** * * JexlManager - @@ -41,8 +43,9 @@ class JexlManager sem_t sem; public: - void init(void); - void release(void); + void init(void); + std::string evaluate(JexlContext* jexlContextP, const std::string& expression); + void release(void); }; #endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file From cbd523b3cfffd8b459f8a2caa4688fe658134eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 19:33:00 +0100 Subject: [PATCH 142/390] FIX use JEXL evaluate to evaluate macros --- src/lib/common/macroSubstitute.cpp | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 9ab9ac2131..5c1a98c74d 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -32,6 +32,7 @@ #include "common/JsonHelper.h" #include "common/macroSubstitute.h" +#include "jexl/jexlMgr.h" /* **************************************************************************** @@ -39,6 +40,9 @@ * smartStringValue - * * Returns the effective string value, taking into account replacements +* +* FIXME PR: notFoundDefault is not longer used? +* */ std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault) { @@ -47,21 +51,11 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // is different, which makes difficult to unify both them if ((jexlContextP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) { - /* FIXME PR: needs adaptation replacementsP -> jexlContextP // "Full replacement" case. In this case, the result is not always a string // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - std::map::iterator iter = replacementsP->find(macroName); - if (iter == replacementsP->end()) - { - // macro doesn't exist in the replacement map, so we use null as failsafe - return notFoundDefault; - } - else - { - return iter->second; - }*/ - return ""; + + return jexlMgr.evaluate(jexlContextP, macroName); } else if (jexlContextP != NULL) { @@ -115,10 +109,12 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon /* **************************************************************************** * * stringValueOrNothing - +* +* FIXME PR: notFoundDefault no longer used? */ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault) { - // FIXME PR: to from replacementsP to jexlContextP + return jexlMgr.evaluate(jexlContextP, key); /*std::map::iterator iter = replacementsP->find(key); if (iter == replacementsP->end()) { @@ -139,7 +135,6 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st return value; } }*/ - return ""; } From 590af6e7d4126e7eff4d29f6dd8ab184e6d35d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Feb 2024 19:54:24 +0100 Subject: [PATCH 143/390] FIX increase debug log tracing in jexl library --- src/lib/jexl/JexlContext.cpp | 11 ++++++++++- src/lib/jexl/JexlManager.cpp | 22 ++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index b9e5968cbe..a34f6962c4 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -23,9 +23,12 @@ * Author: Fermin Galan */ +#include + +#include "logMsg/logMsg.h" #include "jexl/JexlContext.h" -#include + /* **************************************************************************** * @@ -49,22 +52,27 @@ JexlContext::JexlContext PyObject* value; + LM_T(LmtJexl, ("adding to JEXL context: id=%s", id.c_str())); value = Py_BuildValue("s", id.c_str()); PyDict_SetItemString(jexl_context, "id", value); Py_DECREF(value); + LM_T(LmtJexl, ("adding to JEXL context: type=%s", type.c_str())); value = Py_BuildValue("s", type.c_str()); PyDict_SetItemString(jexl_context, "type", value); Py_DECREF(value); + LM_T(LmtJexl, ("adding to JEXL context: service=%s", service.c_str())); value = Py_BuildValue("s", service.c_str()); PyDict_SetItemString(jexl_context, "service", value); Py_DECREF(value); + LM_T(LmtJexl, ("adding to JEXL context: servicePath=%s", servicePath.c_str())); value = Py_BuildValue("s", servicePath.c_str()); PyDict_SetItemString(jexl_context, "servicePath", value); Py_DECREF(value); + LM_T(LmtJexl, ("adding to JEXL context: token=%s", token.c_str())); value = Py_BuildValue("s", token.c_str()); PyDict_SetItemString(jexl_context, "authToken", value); Py_DECREF(value); @@ -88,6 +96,7 @@ PyObject* JexlContext::get(void) */ void JexlContext::add(const std::string& key, const std::string& _value) { + LM_T(LmtJexl, ("adding to JEXL context: %s=%s", key.c_str(), _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); PyDict_SetItemString(jexl_context, key.c_str(), value); Py_DECREF(value); diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 4ad612d391..20dd12864f 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -108,29 +108,39 @@ void JexlManager::init(void) */ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { - PyObject* expression = Py_BuildValue("s", _expression); + LM_T(LmtJexl, ("evaluating JEXL expresion: %s", _expression.c_str())); + + PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { - // FIXME PR: manage error + // FIXME PR: grab error message from Python stack + LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, jexlContextP->get()); + Py_XDECREF(expression); if (result == NULL) { - // FIXME PR: manage error + // FIXME PR: grab error message from Python stack + LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } PyObject* repr = PyObject_Repr(result); + Py_XDECREF(result); if (repr == NULL) { - // FIXME PR: manage error + // FIXME PR: grab error message from Python stack + LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } - const char* result_str = PyUnicode_AsUTF8(repr); - return std::string(result_str); + std::string result_str = std::string(PyUnicode_AsUTF8(repr)); + Py_XDECREF(repr); + + LM_T(LmtJexl, ("JEXL expression evaluation result: %s", result_str.c_str())); + return result_str; } From 87364185f3c8d0b55169675224ff8075664c096e Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Mon, 12 Feb 2024 13:01:33 +0530 Subject: [PATCH 144/390] updated entity deleting script to python3 --- CHANGES_NEXT_RELEASE | 1 + scripts/utils/delete_entities_by_list.py | 25 +++++++++---------- scripts/utils/delete_entities_by_query.py | 29 ++++++++++++----------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index fdb575ad63..78a446e7ef 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ - Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) - Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) +- Fix: Delete multiple entities using `type` is not working (#4507) diff --git a/scripts/utils/delete_entities_by_list.py b/scripts/utils/delete_entities_by_list.py index bc26070106..24f6bee9b9 100755 --- a/scripts/utils/delete_entities_by_list.py +++ b/scripts/utils/delete_entities_by_list.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U # # This file is part of Orion Context Broker. @@ -51,16 +51,16 @@ def response_is_ok(data): :return: """ - if not 'contextResponses' in data: + if 'contextResponses' not in data: return False if len(data['contextResponses']) == 0: return False - if not 'statusCode' in data['contextResponses'][0]: + if 'statusCode' not in data['contextResponses'][0]: return False - if not 'code' in data['contextResponses'][0]['statusCode']: + if 'code' not in data['contextResponses'][0]['statusCode']: return False if data['contextResponses'][0]['statusCode']['code'] != '200': @@ -85,7 +85,7 @@ def remove_batch(entities): res = requests.post(cb_endpoint + '/v1/updateContext', json=body, headers=headers, verify=False) if res.status_code != 200 or not response_is_ok(res.json()): - print '*** Error deleting entities (%d): %s' % (res.status_code, res.json()) + print('*** Error deleting entities (%d): %s' % (res.status_code, res.json())) return False return True @@ -108,7 +108,7 @@ def remove_all(filename): n = n + 1 i = 0 - print '- Deleting batch %d (%d entities)' % (n, batch_size) + print('- Deleting batch %d (%d entities)' % (n, batch_size)) if not remove_batch(entities): return False @@ -116,7 +116,7 @@ def remove_all(filename): # Last batch with remaining entities if i > 0: - print '- Deleting batch %d (%d entities)' % (n + 1, i) + print('- Deleting batch %d (%d entities)' % (n + 1, i)) remove_batch(entities) @@ -129,13 +129,14 @@ def remove_all(filename): filename = sys.argv[1] # Warn user -print "WARNING!!!! This script will delete all the entities in the file '%s'" % filename -print "This action cannot be undone. If you are sure you want to continue type 'yes' and press Enter" +print("WARNING!!!! This script will delete all the entities in the file '%s'" % filename) +print("This action cannot be undone. If you are sure you want to continue type 'yes' and press Enter") -confirm = raw_input() +confirm = input() -if (confirm != 'yes'): +if confirm != 'yes': sys.exit() if remove_all(filename): - print 'We are done!' + print('We are done!') + diff --git a/scripts/utils/delete_entities_by_query.py b/scripts/utils/delete_entities_by_query.py index 9bb5b15106..3a8363cfaf 100755 --- a/scripts/utils/delete_entities_by_query.py +++ b/scripts/utils/delete_entities_by_query.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U # # This file is part of Orion Context Broker. @@ -27,7 +27,7 @@ ############################################################## # BEGIN of the configuration part (don't touch above this line ;) -cb_endpoint = 'https://localhost:1026' +cb_endpoint = 'http://localhost:1026' headers = { 'fiware-service': 'service', @@ -51,7 +51,7 @@ def testing_populate(n, entity_type): :param n: number of entitis to populate (with ids from e1 to e) :param entity_type: entity type used for the populated entities (all with the same type) """ - print 'Populating %d entities of type %s' % (n, entity_type) + print('Populating %d entities of type %s' % (n, entity_type)) entities = [] for i in range(1, 110): @@ -76,7 +76,7 @@ def testing_populate(n, entity_type): res = requests.post(cb_endpoint + '/v2/op/update', json=body, headers=headers, verify=False) if res.status_code != 204: - print 'Error populating entities (%d): %s' % (res.status_code, res.json()) + print('Error populating entities (%d): %s' % (res.status_code, res.json())) def get_entities_count(): @@ -90,7 +90,7 @@ def get_entities_count(): res = requests.get(cb_endpoint + '/v2/entities?limit=1&options=count&attrs=noexist' + filter, headers=headers, verify=False) if res.status_code != 200: - print 'Error getting entities count (%d): %s' % (res.status_code, res.json()) + print('Error getting entities count (%d): %s' % (res.status_code, res.json())) return -1 else: return int(res.headers['fiware-total-count']) @@ -101,10 +101,10 @@ def initial_statistics(): Print some initial data statistics """ total = get_entities_count() - print 'There are %d entities' % total - pages = total / page_size + print('There are %d entities' % total) + pages = total // page_size rest = total - (pages * page_size) - print 'There are %d pages of %d entities (and a final page of %d entities)' % (pages, page_size, rest) + print('There are %d pages of %d entities (and a final page of %d entities)' % (pages, page_size, rest)) def remove_page(): @@ -130,7 +130,7 @@ def remove_page(): res = requests.post(cb_endpoint + '/v2/op/update', json=body, headers=headers, verify=False) if res.status_code != 204: - print 'Error deleting entities (%d): %s' % (res.status_code, res.json()) + print('Error deleting entities (%d): %s' % (res.status_code, res.json())) return False return True @@ -142,7 +142,7 @@ def remove_all(): """ i = 1 while get_entities_count() > 0: - print '- Remove page %d' % i + print('- Remove page %d' % i) remove_page() i += 1 @@ -150,10 +150,10 @@ def remove_all(): ### Main program starts here ### # Warn user -print "WARNING!!!! This script will delete all the entities matching the filter '%s'" % filter -print "This action cannot be undone. If you are sure you want to continue type 'yes' and press Enter" +print("WARNING!!!! This script will delete all the entities matching the filter '%s'" % filter) +print("This action cannot be undone. If you are sure you want to continue type 'yes' and press Enter") -confirm = raw_input() +confirm = input() if (confirm != 'yes'): sys.exit() @@ -162,4 +162,5 @@ def remove_all(): #testing_populate(109, 'testing') initial_statistics() remove_all() -print 'We are done!' +print('We are done!') + From eea99bd4771275fe0877f04be02722388db0af54 Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Mon, 12 Feb 2024 13:13:50 +0530 Subject: [PATCH 145/390] updated attribute deleting script with python3 --- CHANGES_NEXT_RELEASE | 1 + scripts/utils/delete_attribute_by_query.py | 37 +++++++++++----------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index fdb575ad63..0fee87c741 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ - Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) - Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) +- Fix: Attribute deleting script is not working (#4508) diff --git a/scripts/utils/delete_attribute_by_query.py b/scripts/utils/delete_attribute_by_query.py index 9351606563..8c6054bc07 100755 --- a/scripts/utils/delete_attribute_by_query.py +++ b/scripts/utils/delete_attribute_by_query.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U # # This file is part of Orion Context Broker. @@ -53,7 +53,7 @@ def testing_populate(n, entity_type): :param n: number of entities to populate (with ids from e1 to e) :param entity_type: entity type used for the populated entities (all with the same type) """ - print 'Populating %d entities of type %s' % (n, entity_type) + print('Populating %d entities of type %s' % (n, entity_type)) entities = [] for i in range(0, n): @@ -78,7 +78,7 @@ def testing_populate(n, entity_type): res = requests.post(cb_endpoint + '/v2/op/update', json=body, headers=headers, verify=False) if res.status_code != 204: - print 'Error populating entities (%d): %s' % (res.status_code, res.json()) + print('Error populating entities (%d): %s' % (res.status_code, res.json())) def get_entities_count(): @@ -92,7 +92,7 @@ def get_entities_count(): res = requests.get(cb_endpoint + '/v2/entities?limit=1&options=count&attrs=noexist' + filter, headers=headers, verify=False) if res.status_code != 200: - print 'Error getting entities count (%d): %s' % (res.status_code, res.json()) + print('Error getting entities count (%d): %s' % (res.status_code, res.json())) return -1 else: return int(res.headers['fiware-total-count']) @@ -105,10 +105,10 @@ def initial_statistics(): :return: the total number of entities """ total = get_entities_count() - print 'There are %d entities' % total - pages = total / page_size + print('There are %d entities' % total) + pages = total // page_size rest = total - (pages * page_size) - print 'There are %d pages of %d entities (and a final page of %d entities)' % (pages, page_size, rest) + print('There are %d pages of %d entities (and a final page of %d entities)' % (pages, page_size, rest)) return total @@ -128,7 +128,7 @@ def send_batch(entities): res = requests.post(cb_endpoint + '/v2/op/update?options=keyValues', json=body, headers=headers, verify=False) if res.status_code != 204: - print 'Error in batch operation (%d): %s' % (res.status_code, res.json()) + print('Error in batch operation (%d): %s' % (res.status_code, res.json())) return False return True @@ -137,10 +137,10 @@ def send_batch(entities): ### Main program starts here ### # Warn user -print "WARNING!!!! This script will delete the atrribute '%s' in all the entities matching the filter '%s'" % (attr_to_delete, filter) -print "This action cannot be undone. If you are sure you want to continue type 'yes' and press Enter" +print("WARNING!!!! This script will delete the atrribute '%s' in all the entities matching the filter '%s'" % (attr_to_delete, filter)) +print("This action cannot be undone. If you are sure you want to continue type 'yes' and press Enter") -confirm = raw_input() +confirm = input() if (confirm != 'yes'): sys.exit() @@ -150,21 +150,21 @@ def send_batch(entities): count = initial_statistics() if count == 0: - print 'Nothing to do' + print('Nothing to do') sys.exit(0) entities = [] # Number of batches -pages = count / page_size +pages = count // page_size for i in range(0, pages + 1): - print '- Getting entities page %d' % (i + 1) + print('- Getting entities page %d' % (i + 1)) offset = i * page_size res = requests.get('%s/v2/entities?offset=%s&limit=%s%s' % (cb_endpoint, str(offset), str(page_size), filter), headers=headers, verify=False) if res.status_code != 200: - print 'Error getting entities (%d): %s' % (res.status_code, res.json()) + print('Error getting entities (%d): %s' % (res.status_code, res.json())) sys.exit(1) for entity in res.json(): @@ -178,14 +178,15 @@ def send_batch(entities): accum += 1 if (accum == page_size): - print '- Update batch of %d entities' % len(batch) + print('- Update batch of %d entities' % len(batch)) send_batch(batch) accum = 0 batch = [] # Last batch if len(batch) > 0: - print '- Update batch of %d entities' % len(batch) + print('- Update batch of %d entities' % len(batch)) send_batch(batch) -print 'We are done!' +print('We are done!') + From 9dfdccc5085b38c98bee39de4687035acb7adcb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 12 Feb 2024 08:54:04 +0100 Subject: [PATCH 146/390] FIX jexl implementation --- src/lib/common/macroSubstitute.cpp | 39 +++++++++++++++++++++++------- src/lib/common/macroSubstitute.h | 2 +- src/lib/jexl/JexlContext.cpp | 18 +++++++------- src/lib/jexl/JexlManager.cpp | 22 +++++++++++++---- src/lib/ngsiNotify/Notifier.cpp | 13 ++++++---- 5 files changed, 65 insertions(+), 29 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 5c1a98c74d..83526a319e 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -41,8 +41,6 @@ * * Returns the effective string value, taking into account replacements * -* FIXME PR: notFoundDefault is not longer used? -* */ std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault) { @@ -55,13 +53,18 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); + if (!jexlContextP->hasKey(macroName)) + { + return notFoundDefault; + } + return jexlMgr.evaluate(jexlContextP, macroName); } else if (jexlContextP != NULL) { // "Partial replacement" case. In this case, the result is always a string std::string effectiveValue; - if (!macroSubstitute(&effectiveValue, stringValue, jexlContextP, "null")) + if (!macroSubstitute(&effectiveValue, stringValue, jexlContextP, "null", true)) { // error already logged in macroSubstitute, using stringValue itself as failsafe effectiveValue = stringValue; @@ -110,11 +113,28 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon * * stringValueOrNothing - * -* FIXME PR: notFoundDefault no longer used? */ -static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault) +// FIXME PR: inTheMiddle -> raw ? +static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault, bool inTheMiddle) { - return jexlMgr.evaluate(jexlContextP, key); + if (!jexlContextP->hasKey(key)) + { + return notFoundDefault; + } + + std::string s = jexlMgr.evaluate(jexlContextP, key); + + if (inTheMiddle) + { + // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), + // so double quotes have to be be removed + if (s[0] == '"') + { + s = s.substr(1, s.size()-2); + } + } + + return s; /*std::map::iterator iter = replacementsP->find(key); if (iter == replacementsP->end()) { @@ -164,7 +184,8 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st * Date: Mon Jun 19 16:33:29 2017 +0200 * */ -bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexlContextP, const std::string& notFoundDefault) +// FIXME PR: inTheMiddle -> raw ? +bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexlContextP, const std::string& notFoundDefault, bool inTheMiddle) { // Initial size check: is the string to convert too big? // @@ -227,7 +248,7 @@ bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexl // The +3 is due to "${" and "}" toReduce += (macroName.length() + 3) * times; - toAdd += stringValueOrNothing(jexlContextP, macroName, notFoundDefault).length() * times; + toAdd += stringValueOrNothing(jexlContextP, macroName, notFoundDefault, inTheMiddle).length() * times; } if (from.length() + toAdd - toReduce > outReqMsgMaxSize) @@ -245,7 +266,7 @@ bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexl unsigned int times = it->second; std::string macro = "${" + macroName + "}"; - std::string value = stringValueOrNothing(jexlContextP, macroName, notFoundDefault); + std::string value = stringValueOrNothing(jexlContextP, macroName, notFoundDefault, inTheMiddle); // We have to do the replace operation as many times as macro occurrences for (unsigned int ix = 0; ix < times; ix++) diff --git a/src/lib/common/macroSubstitute.h b/src/lib/common/macroSubstitute.h index 0483fc23ae..cdb66ceaf7 100644 --- a/src/lib/common/macroSubstitute.h +++ b/src/lib/common/macroSubstitute.h @@ -58,6 +58,6 @@ extern std::string smartStringValue(const std::string stringValue, JexlContext* * macroSubstitute - * */ -extern bool macroSubstitute(std::string* sP, const std::string& in, JexlContext* jexlContextP, const std::string& notFoundDefault); +extern bool macroSubstitute(std::string* sP, const std::string& in, JexlContext* jexlContextP, const std::string& notFoundDefault, bool inTheMiddle = false); #endif // SRC_LIB_COMMON_MACROSUBSTITUTE_H_ diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index a34f6962c4..5e01dc951f 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -53,27 +53,27 @@ JexlContext::JexlContext PyObject* value; LM_T(LmtJexl, ("adding to JEXL context: id=%s", id.c_str())); - value = Py_BuildValue("s", id.c_str()); + value = Py_BuildValue("s", ("\"" + id + "\"").c_str()); PyDict_SetItemString(jexl_context, "id", value); Py_DECREF(value); - LM_T(LmtJexl, ("adding to JEXL context: type=%s", type.c_str())); - value = Py_BuildValue("s", type.c_str()); + LM_T(LmtJexl, ("adding to JEXL context: type=\"%s\"", type.c_str())); + value = Py_BuildValue("s", ("\"" + type + "\"").c_str()); PyDict_SetItemString(jexl_context, "type", value); Py_DECREF(value); - LM_T(LmtJexl, ("adding to JEXL context: service=%s", service.c_str())); - value = Py_BuildValue("s", service.c_str()); + LM_T(LmtJexl, ("adding to JEXL context: service=\"%s\"", service.c_str())); + value = Py_BuildValue("s", ("\"" + service + "\"").c_str()); PyDict_SetItemString(jexl_context, "service", value); Py_DECREF(value); - LM_T(LmtJexl, ("adding to JEXL context: servicePath=%s", servicePath.c_str())); - value = Py_BuildValue("s", servicePath.c_str()); + LM_T(LmtJexl, ("adding to JEXL context: servicePath=\"%s\"", servicePath.c_str())); + value = Py_BuildValue("s", ("\"" + servicePath + "\"").c_str()); PyDict_SetItemString(jexl_context, "servicePath", value); Py_DECREF(value); - LM_T(LmtJexl, ("adding to JEXL context: token=%s", token.c_str())); - value = Py_BuildValue("s", token.c_str()); + LM_T(LmtJexl, ("adding to JEXL context: token=\"%s\"", token.c_str())); + value = Py_BuildValue("s", ("\"" + token + "\"").c_str()); PyDict_SetItemString(jexl_context, "authToken", value); Py_DECREF(value); } diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 20dd12864f..462e192c30 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -108,12 +108,12 @@ void JexlManager::init(void) */ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { - LM_T(LmtJexl, ("evaluating JEXL expresion: %s", _expression.c_str())); + LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { - // FIXME PR: grab error message from Python stack + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } @@ -122,7 +122,7 @@ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& Py_XDECREF(expression); if (result == NULL) { - // FIXME PR: grab error message from Python stack + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } @@ -131,7 +131,7 @@ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& Py_XDECREF(result); if (repr == NULL) { - // FIXME PR: grab error message from Python stack + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); return "null"; } @@ -139,7 +139,19 @@ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& std::string result_str = std::string(PyUnicode_AsUTF8(repr)); Py_XDECREF(repr); - LM_T(LmtJexl, ("JEXL expression evaluation result: %s", result_str.c_str())); + // Check if the string has quotes + /*if ((result_str.front() == '\'') && (result_str.back() == '\'')) + { + // Erase the first and last characters (quotes) + result_str.erase(0, 1); // Erase the first character + result_str.erase(result_str.size() - 1); // Erase the last character + }*/ + if (result_str[0] == '\'') + { + result_str = result_str.substr(1, result_str.size()-2); + } + + LM_T(LmtJexl, ("JEXL expression evaluation result: <%s>", result_str.c_str())); return result_str; } diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index f44d4addf8..1a8f19a58f 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -170,7 +170,7 @@ static bool setPayload } else { - if (!macroSubstitute(payloadP, notifPayload, jexlContextP, "")) + if (!macroSubstitute(payloadP, notifPayload, jexlContextP, "", true)) { return false; } @@ -211,6 +211,8 @@ static bool setJsonPayload * * Entity id and type are special. Different from a attribute, they are always * strings and cannot take a number, boolean, etc. as value. +* +* FIXME PR: duplicated logic! (both " and ' in different points of the code) */ inline std::string removeQuotes(std::string s) { @@ -341,6 +343,7 @@ static SenderThreadParams* buildSenderParamsCustom JexlContext jexlContext(en.id, en.type, tenant, en.servicePath, xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { + // FIXME PR: this works with every attribute type? (number, bool, etc.) jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); } @@ -368,7 +371,7 @@ static SenderThreadParams* buildSenderParamsCustom // 2. URL // std::string notifUrl = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.url : notification.mqttInfo.url); - if (macroSubstitute(&url, notifUrl, &jexlContext, "") == false) + if (macroSubstitute(&url, notifUrl, &jexlContext, "", true) == false) { // Warning already logged in macroSubstitute() return NULL; @@ -419,7 +422,7 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &jexlContext, "") == false) || (macroSubstitute(&value, it->second, &jexlContext, "") == false)) + if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) { // Warning already logged in macroSubstitute() return NULL; @@ -445,7 +448,7 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &jexlContext, "") == false) || (macroSubstitute(&value, it->second, &jexlContext, "") == false)) + if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) { // Warning already logged in macroSubstitute() return NULL; @@ -510,7 +513,7 @@ static SenderThreadParams* buildSenderParamsCustom // 8. Topic (only in the case of MQTT notifications) if (notification.type == ngsiv2::MqttNotification) { - if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "") == false) + if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "", true) == false) { // Warning already logged in macroSubstitute() return NULL; From 1ab0cc9292b305710d7d21fe75998e207c138bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 12 Feb 2024 09:52:57 +0100 Subject: [PATCH 147/390] FIX roadmap.md to add new issues and change terms on others --- doc/roadmap.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/roadmap.md b/doc/roadmap.md index 0926f30fc3..f2b51bfa69 100644 --- a/doc/roadmap.md +++ b/doc/roadmap.md @@ -32,6 +32,9 @@ Disclaimer: The following list of features are planned to be addressed in the short term, and incorporated into the coming release(s) of the product: +- Expression language support (JEXL) ([#4004](https://github.com/telefonicaid/fiware-orion/issues/4004)) +- Dynamic / high order attribute values (e.g. an attribute being a sum of two other attributes) based on subscriptions to do internal modifications to CB entities +([#3815](https://github.com/telefonicaid/fiware-orion/issues/3815), [#4513](https://github.com/telefonicaid/fiware-orion/issues/4513)) - MQTT notification retrial ([#4439](https://github.com/telefonicaid/fiware-orion/issues/4439)) - Allow multiple types in entity to support UNE 178503 requirements ([#3638](https://github.com/telefonicaid/fiware-orion/issues/3638)) - Pattern/filter batch updates ([#2389](https://github.com/telefonicaid/fiware-orion/issues/2389)) @@ -46,12 +49,9 @@ The following list of features are planned to be addressed in the medium term, typically within the subsequent release(s) generated in the next **9 months** after next planned release: +- Permissions attribute on entities ([#4514](https://github.com/telefonicaid/fiware-orion/issues/4514)) - Advanced query language ([#4395](https://github.com/telefonicaid/fiware-orion/issues/4395)) - Signed entities ([#4398](https://github.com/telefonicaid/fiware-orion/issues/4398)) -- Dynamic / high order attribute values (e.g. an attribute being a sum of two other attributes) - supported by a Expressions Language - help wanted -([#4004](https://github.com/telefonicaid/fiware-orion/issues/4004)), -([#3815](https://github.com/telefonicaid/fiware-orion/issues/3815)) - Service provisioning API (pools, etc.) ([#4442](https://github.com/telefonicaid/fiware-orion/issues/4442)) - Advanced subscription management - Subscription debug mode (precise statistics consolidation, keep recent history of notifications sent, etc.) ([#4399](https://github.com/telefonicaid/fiware-orion/issues/4399)) From 726dc9c17aa8906a682f4af3dfb4a7f8a7e9991a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 12 Feb 2024 10:16:58 +0100 Subject: [PATCH 148/390] FIX add pyjexl to CI image --- ci/deb/build-dep.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/deb/build-dep.sh b/ci/deb/build-dep.sh index ea8531172a..4d35fe450e 100755 --- a/ci/deb/build-dep.sh +++ b/ci/deb/build-dep.sh @@ -55,6 +55,7 @@ echo "INSTALL: MongoDB shell" \ && apt-get -y update \ && apt-get -y install mongodb-mongosh +# pyjexl is needed by Context Broker functionality, the others are for ftest suite echo "INSTALL: python special dependencies" \ && cd /opt \ && python3 -m venv /opt/ft_env \ @@ -63,6 +64,7 @@ echo "INSTALL: python special dependencies" \ && pip install Werkzeug==2.0.2 \ && pip install paho-mqtt==1.6.1 \ && pip install amqtt==0.11.0b1 \ +&& pip install pyjexl==0.3.0 \ && deactivate # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0 From bdd5ebeeb1f6e8530c48c37ea100546652c1f197 Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Mon, 12 Feb 2024 16:56:38 +0530 Subject: [PATCH 149/390] updated python utility scripts --- scripts/notification_errors_tester.py | 44 +++++++++---------- .../utils/check_entity_existence_by_list.py | 16 +++---- scripts/utils/dump_entities_by_list.py | 16 +++---- scripts/utils/get_coordinates.py | 20 ++++----- .../populate_500bytes_entities_and_subs.py | 12 ++--- scripts/utils/sth_periodic_update.py | 8 ++-- 6 files changed, 58 insertions(+), 58 deletions(-) diff --git a/scripts/notification_errors_tester.py b/scripts/notification_errors_tester.py index 9113402c52..d2f8f734b2 100755 --- a/scripts/notification_errors_tester.py +++ b/scripts/notification_errors_tester.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: latin-1 -*- # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U # @@ -301,8 +301,8 @@ def usage_and_exit(msg): """ if msg != '': - print msg - print + print(msg) + print() usage() sys.exit(1) @@ -313,17 +313,17 @@ def usage(): Print usage message """ - print 'Usage: %s --host --port --url -v -u' % os.path.basename(__file__) - print '' - print 'Parameters:' - print " --host : host to use database to use (default is '0.0.0.0')" - print " --port : port to use (default is 1028)" - print " --url : server URL to use (default is /)" - print " --https: start in https" - print " --key: key file (only used if https is enabled)" - print " --cert: cert file (only used if https is enabled)" - print " -v: verbose mode" - print " -u: print this usage message" + print('Usage: %s --host --port --url -v -u' % os.path.basename(__file__)) + print('') + print('Parameters:') + print(" --host : host to use database to use (default is '0.0.0.0')") + print(" --port : port to use (default is 1028)") + print(" --url : server URL to use (default is /)") + print(" --https: start in https") + print(" --key: key file (only used if https is enabled)") + print(" --cert: cert file (only used if https is enabled)") + print(" -v: verbose mode") + print(" -u: print this usage message") # This function is registered to be called upon termination @@ -370,18 +370,18 @@ def all_done(): if https: if key_file is None or cert_file is None: - print "if --https is used then you have to provide --key and --cert" + print("if --https is used then you have to provide --key and --cert") sys.exit(1) if verbose: - print "verbose mode is on" - print "port: " + str(port) - print "host: " + str(host) - print "url_root: " + str(url_root) - print "https: " + str(https) + print("verbose mode is on") + print("port: " + str(port)) + print("host: " + str(host)) + print("url_root: " + str(url_root)) + print("https: " + str(https)) if https: - print "key file: " + key_file - print "cert file: " + cert_file + print("key file: " + key_file) + print("cert file: " + cert_file) app = Flask(__name__) diff --git a/scripts/utils/check_entity_existence_by_list.py b/scripts/utils/check_entity_existence_by_list.py index c086032c77..bc63bd89ab 100755 --- a/scripts/utils/check_entity_existence_by_list.py +++ b/scripts/utils/check_entity_existence_by_list.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U # # This file is part of Orion Context Broker. @@ -64,12 +64,12 @@ def check_entity(id): if res.status_code != 200: # res.json() is not used but it can be if we want - print '%s|-|FAIL|%d' % (id, res.status_code) + print('%s|-|FAIL|%d' % (id, res.status_code)) n_fail += 1 return type = res.json()['type'] - print '%s|%s|OK|200' % (id, type) + print('%s|%s|OK|200' % (id, type)) n_ok += 1 return @@ -89,7 +89,7 @@ def check_entities(filename): ### Main program starts here ### if len(sys.argv) != 2: - print "invalid number of arguments, please provide the file name of entities" + print("invalid number of arguments, please provide the file name of entities") sys.exit() filename = sys.argv[1] @@ -98,7 +98,7 @@ def check_entities(filename): n_fail = 0 check_entities(filename) -print '----------------' -print 'OK: %d' % n_ok -print 'FAIL: %d' % n_fail -print 'TOTAL: %d' % (n_ok + n_fail) +print('----------------') +print('OK: %d' % n_ok) +print('FAIL: %d' % n_fail) +print('TOTAL: %d' % (n_ok + n_fail)) diff --git a/scripts/utils/dump_entities_by_list.py b/scripts/utils/dump_entities_by_list.py index 83a4dc01af..b940c105e1 100644 --- a/scripts/utils/dump_entities_by_list.py +++ b/scripts/utils/dump_entities_by_list.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U # # This file is part of Orion Context Broker. @@ -62,16 +62,16 @@ def response_is_ok(data): :return: """ - if not 'contextResponses' in data: + if 'contextResponses' not in data: return False if len(data['contextResponses']) != 1: return False - if not 'statusCode' in data['contextResponses'][0]: + if 'statusCode' not in data['contextResponses'][0]: return False - if not 'code' in data['contextResponses'][0]['statusCode']: + if 'code' not in data['contextResponses'][0]['statusCode']: return False if data['contextResponses'][0]['statusCode']['code'] != '200': @@ -97,12 +97,12 @@ def dump_entity(id): res = requests.post(cb_endpoint + '/v1/queryContext', json=body, headers=headers, verify=False) if res.status_code != 200 or not response_is_ok(res.json()): - print - print '*** Error getting entity %s (%d): %s' % (id, res.status_code, res.json()) + print() + print('*** Error getting entity %s (%d): %s' % (id, res.status_code, res.json())) return False entity_body = res.json()['contextResponses'][0]['contextElement'] - print json.dumps(entity_body).decode('unicode-escape') + print(json.dumps(entity_body)) return True @@ -123,7 +123,7 @@ def dump_entities(filename): ### Main program starts here ### if len(sys.argv) != 2: - print "invalid number of arguments, please provide the file name of entities" + print("invalid number of arguments, please provide the file name of entities") sys.exit() filename = sys.argv[1] diff --git a/scripts/utils/get_coordinates.py b/scripts/utils/get_coordinates.py index d801cf8605..1c4954f2db 100755 --- a/scripts/utils/get_coordinates.py +++ b/scripts/utils/get_coordinates.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U # # This file is part of Orion Context Broker. @@ -88,7 +88,7 @@ def process_page(entities): if loc_attr is None: loc_attr = get_ngsiv1_location(entity) if loc_attr is not None: - print '%s: %s' % (entity['id'], entity[loc_attr]['value']) + print('%s: %s' % (entity['id'], entity[loc_attr]['value'])) ### Main program starts here ### @@ -97,8 +97,8 @@ def process_page(entities): res = requests.get('%s/v2/entities?offset=0&limit=%s&options=count' % (cb_endpoint, str(page_size)), headers=headers, verify=False) if res.status_code != 200: - print 'Error getting entities (%d): %s' % (res.status_code, res.json()) - sys.exit(1) + print('Error getting entities (%d): %s' % (res.status_code, res.json())) + sys.exit(1) # Get count count = int(res.headers['fiware-total-count']) @@ -107,16 +107,16 @@ def process_page(entities): process_page(res.json()) # Number of batches -pages = count / page_size -for i in range(0, pages): +pages = count // page_size +for i in range(pages): offset = (i + 1) * page_size res = requests.get('%s/v2/entities?offset=%s&limit=%s' % (cb_endpoint, str(offset), str(page_size)), headers=headers, verify=False) if res.status_code != 200: - print 'Error getting entities (%d): %s' % (res.status_code, res.json()) + print('Error getting entities (%d): %s' % (res.status_code, res.json())) sys.exit(1) process_page(res.json()) -print 'Hints:' -print '* use "awk -F \': \' \'{print $2}\'" filter if you want to grab just the coordinates' -print '* copy-paste output to https://www.darrinward.com/lat-long so see where you entities are on a map' +print('Hints:') +print('* use "awk -F \': \' \'{print $2}\'" filter if you want to grab just the coordinates') +print('* copy-paste output to https://www.darrinward.com/lat-long so see where you entities are on a map') diff --git a/scripts/utils/populate_500bytes_entities_and_subs.py b/scripts/utils/populate_500bytes_entities_and_subs.py index d9b4774e35..6f82af6288 100644 --- a/scripts/utils/populate_500bytes_entities_and_subs.py +++ b/scripts/utils/populate_500bytes_entities_and_subs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U # # This file is part of Orion Context Broker. @@ -90,12 +90,12 @@ } for i in range(0, 1500): - print '* Creating entity ' + str(i) + print('* Creating entity ' + str(i)) payload_entities['id'] = str(i).zfill(10) r = post(url_entities, data=json.dumps(payload_entities), headers=headers) - #print r.text - + #print(r.text) + for i in range(0, 1500): - print '* Creating subscription ' + str(i) + print('* Creating subscription ' + str(i)) r = post(url_subscription, data=json.dumps(payload_sub), headers=headers) - #print r.text + #print(r.text) diff --git a/scripts/utils/sth_periodic_update.py b/scripts/utils/sth_periodic_update.py index 957dc1c1f3..0f7cbe00ff 100755 --- a/scripts/utils/sth_periodic_update.py +++ b/scripts/utils/sth_periodic_update.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U # # This file is part of Orion Context Broker. @@ -69,11 +69,11 @@ def update_entity(): 'value': value } - print '%s: Updating attribute %s in entity %s with value %d' % (datetime.now().isoformat(), entity, attribute, value) + print('%s: Updating attribute %s in entity %s with value %d' % (datetime.now().isoformat(), entity, attribute, value)) res = requests.put(cb_endpoint + '/v2/entities/' + entity + '/attrs/' + attribute, json=body, headers=headers, verify=False) if res.status_code != 204: - print '*** Error updating entity (%d): %s' % (res.status_code, res.json()) + print('*** Error updating entity (%d): %s' % (res.status_code, res.json())) return False return True @@ -83,5 +83,5 @@ def update_entity(): while True: if not update_entity(): - break; + break sleep(interval) From 498d13368d615f8ba1a471feb4d981fa0c1ffd0c Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Mon, 12 Feb 2024 17:05:44 +0530 Subject: [PATCH 150/390] updated CNR --- CHANGES_NEXT_RELEASE | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0fee87c741..fdb575ad63 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,3 @@ - Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) - Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) -- Fix: Attribute deleting script is not working (#4508) From 297929b08c3be666f7d47b615b18159ad890fbdc Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Mon, 12 Feb 2024 17:10:28 +0530 Subject: [PATCH 151/390] updated CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 78a446e7ef..fdb575ad63 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,3 @@ - Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) - Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) -- Fix: Delete multiple entities using `type` is not working (#4507) From 904c9652c1424bd63a2b5ba7b80a38938ffd4aad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 09:32:23 +0100 Subject: [PATCH 152/390] FIX comments in Dockerfiles --- docker/Dockerfile | 1 + docker/Dockerfile.alpine | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5b80157d4..a54f4844f4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -88,6 +88,7 @@ RUN \ cd cmake-build && \ # Different from ci/deb/build-dep.sh and build from source documentation, we add here also # the MONGOC_TEST_USE_CRYPT_SHARED=FALSE. It needs Python and in this tiny image we don't have it + # FIXME PR: now that Python is going to be include for JEXL functionalty, maybe this should be changed... cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DMONGOC_TEST_USE_CRYPT_SHARED=FALSE .. && \ make && \ make install && \ diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 524335f677..b871d42a5d 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -93,6 +93,7 @@ RUN \ cd cmake-build && \ # Different from ci/deb/build-dep.sh and build from source documentation, we add here also # the MONGOC_TEST_USE_CRYPT_SHARED=FALSE. It needs Python and in this tiny image we don't have it + # FIXME PR: now that Python is going to be include for JEXL functionalty, maybe this should be changed... cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DMONGOC_TEST_USE_CRYPT_SHARED=FALSE .. && \ make && \ make install && \ From ae88c6e93694e2a2ab0e76fcc114252aa5479d80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 09:34:52 +0100 Subject: [PATCH 153/390] ADD JexlResult class (still wip) --- src/lib/common/macroSubstitute.cpp | 20 +++--- src/lib/jexl/CMakeLists.txt | 2 + src/lib/jexl/JexlContext.cpp | 81 +++++++++++++++-------- src/lib/jexl/JexlContext.h | 3 + src/lib/jexl/JexlManager.cpp | 101 ++++++++++++++++++++++------- src/lib/jexl/JexlManager.h | 7 +- src/lib/jexl/JexlResult.cpp | 63 ++++++++++++++++++ src/lib/jexl/JexlResult.h | 52 +++++++++++++++ src/lib/ngsi/ContextAttribute.cpp | 48 ++++++++++++++ src/lib/ngsi/ContextAttribute.h | 2 + src/lib/ngsiNotify/Notifier.cpp | 3 +- 11 files changed, 319 insertions(+), 63 deletions(-) create mode 100644 src/lib/jexl/JexlResult.cpp create mode 100644 src/lib/jexl/JexlResult.h diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 83526a319e..530b7f4b3a 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -53,12 +53,12 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - if (!jexlContextP->hasKey(macroName)) - { - return notFoundDefault; - } + //if (!jexlContextP->hasKey(macroName)) + //{ + // return notFoundDefault; + //} - return jexlMgr.evaluate(jexlContextP, macroName); + return jexlMgr.evaluate(jexlContextP, macroName).toString(); } else if (jexlContextP != NULL) { @@ -117,12 +117,12 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // FIXME PR: inTheMiddle -> raw ? static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault, bool inTheMiddle) { - if (!jexlContextP->hasKey(key)) - { - return notFoundDefault; - } + //if (!jexlContextP->hasKey(key)) + //{ + // return notFoundDefault; + //} - std::string s = jexlMgr.evaluate(jexlContextP, key); + std::string s = jexlMgr.evaluate(jexlContextP, key).toString(); if (inTheMiddle) { diff --git a/src/lib/jexl/CMakeLists.txt b/src/lib/jexl/CMakeLists.txt index abacbeb3ca..a2ea7eb6e2 100644 --- a/src/lib/jexl/CMakeLists.txt +++ b/src/lib/jexl/CMakeLists.txt @@ -23,12 +23,14 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES JexlManager.cpp JexlContext.cpp + JexlResult.cpp jexlMgr.cpp ) SET (HEADERS JexlManager.h JexlContext.h + JexlResult.h jexlMgr.h ) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index 5e01dc951f..d0c1b6f51f 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -50,35 +50,15 @@ JexlContext::JexlContext // FIXME PR: error control } - PyObject* value; - - LM_T(LmtJexl, ("adding to JEXL context: id=%s", id.c_str())); - value = Py_BuildValue("s", ("\"" + id + "\"").c_str()); - PyDict_SetItemString(jexl_context, "id", value); - Py_DECREF(value); - - LM_T(LmtJexl, ("adding to JEXL context: type=\"%s\"", type.c_str())); - value = Py_BuildValue("s", ("\"" + type + "\"").c_str()); - PyDict_SetItemString(jexl_context, "type", value); - Py_DECREF(value); - - LM_T(LmtJexl, ("adding to JEXL context: service=\"%s\"", service.c_str())); - value = Py_BuildValue("s", ("\"" + service + "\"").c_str()); - PyDict_SetItemString(jexl_context, "service", value); - Py_DECREF(value); - - LM_T(LmtJexl, ("adding to JEXL context: servicePath=\"%s\"", servicePath.c_str())); - value = Py_BuildValue("s", ("\"" + servicePath + "\"").c_str()); - PyDict_SetItemString(jexl_context, "servicePath", value); - Py_DECREF(value); - - LM_T(LmtJexl, ("adding to JEXL context: token=\"%s\"", token.c_str())); - value = Py_BuildValue("s", ("\"" + token + "\"").c_str()); - PyDict_SetItemString(jexl_context, "authToken", value); - Py_DECREF(value); + add("id", id); + add("type", type); + add("service", service); + add("servicePath", servicePath); + add("authToken", token); } + /* **************************************************************************** * * JexlContext::get - @@ -96,7 +76,7 @@ PyObject* JexlContext::get(void) */ void JexlContext::add(const std::string& key, const std::string& _value) { - LM_T(LmtJexl, ("adding to JEXL context: %s=%s", key.c_str(), _value.c_str())); + LM_T(LmtJexl, ("adding to JEXL context (string): %s=%s", key.c_str(), _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); PyDict_SetItemString(jexl_context, key.c_str(), value); Py_DECREF(value); @@ -104,6 +84,52 @@ void JexlContext::add(const std::string& key, const std::string& _value) +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, double _value) +{ + LM_T(LmtJexl, ("adding to JEXL context (double): %s=%f", key.c_str(), _value)); + PyObject* value = Py_BuildValue("d", _value); + PyDict_SetItemString(jexl_context, key.c_str(), value); + Py_DECREF(value); +} + + + + +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, bool _value) +{ + LM_T(LmtJexl, ("adding to JEXL context (bool): %s=%s", key.c_str(), _value? "true" : "false")); + if (_value) + { + PyDict_SetItemString(jexl_context, key.c_str(), Py_True); + } + else + { + PyDict_SetItemString(jexl_context, key.c_str(), Py_False); + } +} + + + +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key) +{ + LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyDict_SetItemString(jexl_context, key.c_str(), Py_None); +} + + + /* **************************************************************************** * * JexlContext::hasKey - @@ -126,5 +152,6 @@ bool JexlContext::hasKey(const std::string& key) */ JexlContext::~JexlContext() { + // FIXME PR: this is not correct. Recursively release of the dict object Py_XDECREF(jexl_context); } \ No newline at end of file diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index 77d6083c07..0eb1f987da 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -54,6 +54,9 @@ class JexlContext PyObject* get(void); void add(const std::string& key, const std::string& value); + void add(const std::string& key, double value); + void add(const std::string& key, bool value); + void add(const std::string& key); bool hasKey(const std::string& key); }; diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 462e192c30..9de0ee88c5 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -26,8 +26,11 @@ #include #include "jexl/JexlManager.h" +#include "jexl/JexlResult.h" #include "logMsg/logMsg.h" +#include "orionTypes/OrionValueType.h" + /* **************************************************************************** @@ -65,6 +68,36 @@ static const char* capturePythonError() } +/* **************************************************************************** +* +* getPyObjectType - +* +* FIXME PR: ValueTypeVector and ValueTypeObject should be taken into account +*/ +static orion::ValueType getPyObjectType(PyObject* obj) +{ + // PyBool_Check() has to be donde before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be + // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 + if (PyBool_Check(obj)) + { + return orion::ValueTypeBoolean; + } + else if (PyLong_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyFloat_Check(obj)) + { + return orion::ValueTypeNumber; + } + else + { + // For other types (including dict and list) we use string as failsafe + return orion::ValueTypeString; + } +} + + /* **************************************************************************** * @@ -106,16 +139,21 @@ void JexlManager::init(void) * * JexlManager::evaluate - */ -std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) +JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); + JexlResult r; + + // If nothing changes, the returned value is null + r.valueType = orion::ValueTypeNull; + PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); - return "null"; + return r; } PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, jexlContextP->get()); @@ -124,35 +162,54 @@ std::string JexlManager::evaluate(JexlContext* jexlContextP, const std::string& { // FIXME PR: grab error message from Python stack, use LM_E/LM_W? LM_T(LmtJexl, ("error evaluating expression, result is null")); - return "null"; + return r; } - PyObject* repr = PyObject_Repr(result); - Py_XDECREF(result); - if (repr == NULL) + // Special case: expresion evalutes to None + if (result == Py_None) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); - return "null"; + Py_XDECREF(result); + r.valueType = orion::ValueTypeNull; + LM_T(LmtJexl, ("JEXL evaluation result is null")); + return r; } - std::string result_str = std::string(PyUnicode_AsUTF8(repr)); - Py_XDECREF(repr); - - // Check if the string has quotes - /*if ((result_str.front() == '\'') && (result_str.back() == '\'')) + // Other not null types + r.valueType = getPyObjectType(result); + if (r.valueType == orion::ValueTypeNumber) { - // Erase the first and last characters (quotes) - result_str.erase(0, 1); // Erase the first character - result_str.erase(result_str.size() - 1); // Erase the last character - }*/ - if (result_str[0] == '\'') + r.numberValue = PyFloat_AsDouble(result); + LM_T(LmtJexl, ("JEXL evaluation result (double): %f", r.numberValue)); + Py_XDECREF(result); + } + else if (r.valueType == orion::ValueTypeBoolean) { - result_str = result_str.substr(1, result_str.size()-2); + r.boolValue = PyObject_IsTrue(result); + LM_T(LmtJexl, ("JEXL evaluation result (bool): %s", r.boolValue ? "true": "false")); + Py_XDECREF(result); } + else if (r.valueType == orion::ValueTypeString) + { + const char* repr = PyUnicode_AsUTF8(result); + Py_XDECREF(result); + if (repr == NULL) + { + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? + LM_T(LmtJexl, ("error evaluating expression, result is null")); + } + else + { + r.stringValue = std::string(repr); + LM_T(LmtJexl, ("JEXL evaluation result (string): %s", r.stringValue.c_str())); + } + } + + //if (result_str[0] == '\'') + //{ + // result_str = result_str.substr(1, result_str.size()-2); + //} - LM_T(LmtJexl, ("JEXL expression evaluation result: <%s>", result_str.c_str())); - return result_str; + return r; } diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h index 89cadcf534..ad83fd56ea 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/jexl/JexlManager.h @@ -30,6 +30,7 @@ #include #include "jexl/JexlContext.h" +#include "jexl/JexlResult.h" /* **************************************************************************** * @@ -43,9 +44,9 @@ class JexlManager sem_t sem; public: - void init(void); - std::string evaluate(JexlContext* jexlContextP, const std::string& expression); - void release(void); + void init(void); + JexlResult evaluate(JexlContext* jexlContextP, const std::string& expression); + void release(void); }; #endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp new file mode 100644 index 0000000000..654c16582b --- /dev/null +++ b/src/lib/jexl/JexlResult.cpp @@ -0,0 +1,63 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "jexl/JexlResult.h" + +#include "common/string.h" +#include "common/JsonHelper.h" +#include "logMsg/logMsg.h" + +/* **************************************************************************** +* +* toString - +* +* Pretty similar to ContextAttribute::toJsonValue() +* +* FIXME PR: ValueTypeVector and ValueTypeObject should be taken into account +*/ +std::string JexlResult::toString(void) +{ + if (valueType == orion::ValueTypeNumber) + { + return double2string(numberValue); + } + else if (valueType == orion::ValueTypeBoolean) + { + return boolValue ? "true" : "false"; + } + else if (valueType == orion::ValueTypeString) + { + return "\"" + toJsonString(stringValue) + "\""; + } + else if (valueType == orion::ValueTypeNull) + { + return "null"; + } + else + { + LM_E(("Runtime Error (not allowed type in JexlResult: %s)", valueTypeName(valueType))); + return ""; + } +} \ No newline at end of file diff --git a/src/lib/jexl/JexlResult.h b/src/lib/jexl/JexlResult.h new file mode 100644 index 0000000000..a3e0709820 --- /dev/null +++ b/src/lib/jexl/JexlResult.h @@ -0,0 +1,52 @@ +#ifndef SRC_LIB_JEXL_JEXLRESULT_H_ +#define SRC_LIB_JEXL_JEXLRESULT_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "orionTypes/OrionValueType.h" + +#include + +/* **************************************************************************** +* +* JexlResult - +*/ +class JexlResult +{ +public: + // Similar to the fields used in ContextAttribute.h + + orion::ValueType valueType; // Type of value + std::string stringValue; // "value" as a String + double numberValue; // "value" as a Number + bool boolValue; // "value" as a Boolean + + std::string toString(void); +}; + + + +#endif // #define SRC_LIB_JEXL_JEXLRESULT_H_ diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index f4407a89fc..534a1f93de 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1290,6 +1290,54 @@ std::string ContextAttribute::toJsonAsValue +/* **************************************************************************** +* +* addToContext - +* +* Pretty similar in structure to toJsonValue +*/ +void ContextAttribute::addToContext(JexlContext* jexlContextP) +{ + if (compoundValueP != NULL) + { + // FIXME PR: not sure if this has the proper effect... + jexlContextP->add(name, compoundValueP->toJson()); + } + else if (valueType == orion::ValueTypeNumber) + { + if ((type == DATE_TYPE) || (type == DATE_TYPE_ALT)) + { + jexlContextP->add(name, toJsonString(isodate2str(numberValue))); + } + else // regular number + { + jexlContextP->add(name, numberValue); + } + } + else if (valueType == orion::ValueTypeString) + { + jexlContextP->add(name, toJsonString(stringValue)); + } + else if (valueType == orion::ValueTypeBoolean) + { + jexlContextP->add(name, boolValue); + } + else if (valueType == orion::ValueTypeNull) + { + jexlContextP->add(name); + } + else if (valueType == orion::ValueTypeNotGiven) + { + LM_E(("Runtime Error (value not given for attribute %s)", name.c_str())); + } + else + { + LM_E(("Runtime Error (invalid value type %s for attribute %s)", valueTypeName(valueType), name.c_str())); + } +} + + + /* **************************************************************************** * * ContextAttribute::check - diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index 1ca58a0a98..6b65827d29 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -119,6 +119,8 @@ typedef struct ContextAttribute MimeType* outMimeTypeP, HttpStatusCode* scP); + void addToContext(JexlContext* jexlContextP); + void release(void); std::string getName(void); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 1a8f19a58f..a7fc2ce48b 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -344,7 +344,8 @@ static SenderThreadParams* buildSenderParamsCustom for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { // FIXME PR: this works with every attribute type? (number, bool, etc.) - jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); + //jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); + en.attributeVector[ix]->addToContext(&jexlContext); } // From fb5939d268710023e493982a87a4ac46f04d16f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 11:07:01 +0100 Subject: [PATCH 154/390] ADD new JexlContextList --- src/lib/jexl/JexlContext.cpp | 169 +++++++++++++++++++++++++++++--- src/lib/jexl/JexlContext.h | 34 +++++-- src/lib/jexl/JexlManager.cpp | 42 +++++++- src/lib/jexl/JexlResult.cpp | 4 + src/lib/ngsiNotify/Notifier.cpp | 7 +- 5 files changed, 233 insertions(+), 23 deletions(-) diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index d0c1b6f51f..ba5975e6a1 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -23,6 +23,9 @@ * Author: Fermin Galan */ +// FIXME PR: use better names than JexlContext and JexlContextList. Also in LM_T +// FIXME PR: add methods should check error in Py_BuildValue() for NULL + #include #include "logMsg/logMsg.h" @@ -35,26 +38,15 @@ * JexlContext::JexlContext - */ JexlContext::JexlContext -( - const std::string& id, - const std::string& type, - const std::string& service, - const std::string& servicePath, - const std::string& token -) +() { jexl_context = PyDict_New(); if (jexl_context == NULL) { // FIXME PR: error control + // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) } - - add("id", id); - add("type", type); - add("service", service); - add("servicePath", servicePath); - add("authToken", token); } @@ -130,6 +122,32 @@ void JexlContext::add(const std::string& key) +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, JexlContext _jexlContext) +{ + // FIXME PR: implement a toString() method in JexlContext to be used here + //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyDict_SetItemString(jexl_context, key.c_str(), _jexlContext.get()); +} + + + +/* **************************************************************************** +* +* JexlContext::add - +*/ +void JexlContext::add(const std::string& key, JexlContextList jexlContextList) +{ + // FIXME PR: implement a toString() method in JexlContextList to be used here + //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyDict_SetItemString(jexl_context, key.c_str(), jexlContextList.get()); +} + + + /* **************************************************************************** * * JexlContext::hasKey - @@ -154,4 +172,129 @@ JexlContext::~JexlContext() { // FIXME PR: this is not correct. Recursively release of the dict object Py_XDECREF(jexl_context); +} + + +/* **************************************************************************** +* +* JexlContextList::JexlContextList - +*/ +JexlContextList::JexlContextList() +{ + jexl_context = PyList_New(0); + + if (jexl_context == NULL) + { + // FIXME PR: error control + // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + } +} + + + +/* **************************************************************************** +* +* JexlContextList::get - +*/ +PyObject* JexlContextList::get(void) +{ + return jexl_context; +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(const std::string& _value) +{ + LM_T(LmtJexl, ("adding to JEXL context list (string): %s=", _value.c_str())); + PyObject* value = Py_BuildValue("s", _value.c_str()); + PyList_Append(jexl_context, value); + Py_DECREF(value); +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(double _value) +{ + LM_T(LmtJexl, ("adding to JEXL context list (double): %f", _value)); + PyObject* value = Py_BuildValue("d", _value); + PyList_Append(jexl_context, value); + Py_DECREF(value); +} + + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(bool _value) +{ + LM_T(LmtJexl, ("adding to JEXL context list (bool): %s", _value? "true" : "false")); + if (_value) + { + PyList_Append(jexl_context, Py_True); + } + else + { + PyList_Append(jexl_context, Py_False); + } +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(void) +{ + LM_T(LmtJexl, ("adding to JEXL context list (none)")); + PyList_Append(jexl_context, Py_None); +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(JexlContext _jexlContextP) +{ + // FIXME PR: implement a toString() method in JexlContext to be used here + //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyList_Append(jexl_context, _jexlContextP.get()); +} + + + +/* **************************************************************************** +* +* JexlContextList::add - +*/ +void JexlContextList::add(JexlContextList _jexlContextList) +{ + // FIXME PR: implement a toString() method in JexlContextList to be used here + //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); + PyList_Append(jexl_context, _jexlContextList.get()); +} + + + +/* **************************************************************************** +* +* JexlContextList::~JexlContextList - +*/ +JexlContextList::~JexlContextList() +{ + // FIXME PR: this is not correct. Recursively release of the list object + Py_XDECREF(jexl_context); } \ No newline at end of file diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index 0eb1f987da..bf343b4e7e 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -26,10 +26,15 @@ * Author: Fermin Galan */ +// FIXME PR: rename this to a better name (ExprContextDict?) + #include #include +//#include "jexl/JexlContextList.h" + +class JexlContextList; // forward declaration /* **************************************************************************** * @@ -43,23 +48,36 @@ class JexlContext public: ~JexlContext(); - JexlContext - ( - const std::string& id, - const std::string& type, - const std::string& service, - const std::string& servicePath, - const std::string& token - ); + JexlContext(); PyObject* get(void); void add(const std::string& key, const std::string& value); void add(const std::string& key, double value); void add(const std::string& key, bool value); void add(const std::string& key); + void add(const std::string& key, JexlContext jexlContext); + void add(const std::string& key, JexlContextList jexlContextList); bool hasKey(const std::string& key); }; +class JexlContextList +{ +private: + PyObject* jexl_context; + +public: + ~JexlContextList(); + + JexlContextList(); + + PyObject* get(void); + void add(const std::string& value); + void add(double value); + void add(bool value); + void add(void); + void add(JexlContext jexlContext); + void add(JexlContextList jexlContextList); +}; #endif // #define SRC_LIB_JEXL_JEXLCONTEXT_H_ diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 9de0ee88c5..e8bd7bf5fb 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -90,9 +90,17 @@ static orion::ValueType getPyObjectType(PyObject* obj) { return orion::ValueTypeNumber; } + else if (PyDict_Check(obj)) + { + return orion::ValueTypeObject; + } + else if (PyList_Check(obj)) + { + return orion::ValueTypeVector; + } else { - // For other types (including dict and list) we use string as failsafe + // For other types we use string (this is also a failsafe for types not being strings) return orion::ValueTypeString; } } @@ -188,6 +196,38 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ LM_T(LmtJexl, ("JEXL evaluation result (bool): %s", r.boolValue ? "true": "false")); Py_XDECREF(result); } + else if (r.valueType == orion::ValueTypeObject) + { + // FIXME PR: same processing than string? Unify? + const char* repr = PyUnicode_AsUTF8(result); + Py_XDECREF(result); + if (repr == NULL) + { + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? + LM_T(LmtJexl, ("error evaluating expression, result is null")); + } + else + { + r.stringValue = std::string(repr); + LM_T(LmtJexl, ("JEXL evaluation result (object): %s", r.stringValue.c_str())); + } + } + else if (r.valueType == orion::ValueTypeVector) + { + // FIXME PR: same processing than string? Unify? + const char* repr = PyUnicode_AsUTF8(result); + Py_XDECREF(result); + if (repr == NULL) + { + // FIXME PR: grab error message from Python stack, use LM_E/LM_W? + LM_T(LmtJexl, ("error evaluating expression, result is null")); + } + else + { + r.stringValue = std::string(repr); + LM_T(LmtJexl, ("JEXL evaluation result (list): %s", r.stringValue.c_str())); + } + } else if (r.valueType == orion::ValueTypeString) { const char* repr = PyUnicode_AsUTF8(result); diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 654c16582b..1197b1491a 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -51,6 +51,10 @@ std::string JexlResult::toString(void) { return "\"" + toJsonString(stringValue) + "\""; } + else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) + { + return toJsonString(stringValue); + } else if (valueType == orion::ValueTypeNull) { return "null"; diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index a7fc2ce48b..24c6953ef0 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -340,7 +340,12 @@ static SenderThreadParams* buildSenderParamsCustom // Used by several macroSubstitute() calls along this function //std::map replacements; //buildReplacementsMap(en, tenant, xauthToken, &replacements); - JexlContext jexlContext(en.id, en.type, tenant, en.servicePath, xauthToken); + JexlContext jexlContext; + jexlContext.add("id", en.id); + jexlContext.add("type", en.type); + jexlContext.add("service", tenant); + jexlContext.add("servicePath", en.servicePath); + jexlContext.add("authToken", xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { // FIXME PR: this works with every attribute type? (number, bool, etc.) From a0d75b391b0382d67f3e68ae1337f39db3a423aa Mon Sep 17 00:00:00 2001 From: MD ARQUM FAROOQUI Date: Wed, 14 Feb 2024 19:16:05 +0530 Subject: [PATCH 155/390] Fix 0 length servicepath levels (#4503) * Fix 0 length servicepath levels * updated FT and error message * updated code and added FT * updated FT and CNR --- CHANGES_NEXT_RELEASE | 1 + src/lib/rest/rest.cpp | 9 +- ...5_servicepath_levels_with_zero_length.test | 319 ++++++++++++++++++ 3 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 test/functionalTest/cases/4495_servicepath_levels_with_zero_length/4495_servicepath_levels_with_zero_length.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index fdb575ad63..68d1efb13a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ +- Fix: service path levels with 0 length should not be allowed (#4495) - Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) - Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) diff --git a/src/lib/rest/rest.cpp b/src/lib/rest/rest.cpp index c3a4a2ef71..cc35ff57fd 100644 --- a/src/lib/rest/rest.cpp +++ b/src/lib/rest/rest.cpp @@ -704,6 +704,13 @@ int servicePathCheck(ConnectionInfo* ciP, const char* servicePath) return 1; } + if (servicePath[1] == '/') + { + OrionError oe(SccBadRequest, "empty component in ServicePath"); + ciP->answer = oe.setStatusCodeAndSmartRender(ciP->apiVersion, &(ciP->httpStatusCode)); + return 1; + } + if (ciP->servicePathV.size() > 1) { if (ciP->verb == PATCH) @@ -795,7 +802,7 @@ static char* removeTrailingSlash(std::string path) char* cpath = (char*) path.c_str(); /* strlen(cpath) > 1 ensures that root service path "/" is not touched */ - while ((strlen(cpath) > 1) && (cpath[strlen(cpath) - 1] == '/')) + if ((strlen(cpath) > 1) && (cpath[strlen(cpath) - 1] == '/')) { cpath[strlen(cpath) - 1] = 0; } diff --git a/test/functionalTest/cases/4495_servicepath_levels_with_zero_length/4495_servicepath_levels_with_zero_length.test b/test/functionalTest/cases/4495_servicepath_levels_with_zero_length/4495_servicepath_levels_with_zero_length.test new file mode 100644 index 0000000000..b29637b9a1 --- /dev/null +++ b/test/functionalTest/cases/4495_servicepath_levels_with_zero_length/4495_servicepath_levels_with_zero_length.test @@ -0,0 +1,319 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Servicapath with zero length levels are not allowed + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1/T1/A1=1, with servicepath '/Level_1' (success case) +# 02. GET /v2/entities/E1, with servicepath '/Level_1' to see E1/T1/A1=1 +# 03. Create entity E2/T2/A2=2, with servicepath '/Level_1/Level_2' (success case) +# 04. GET /v2/entities/E2, with servicepath '/Level_1/Level_2' to see E2/T2/A2=2 +# 05. Create entity E3/T3/A3=3, with servicepath '//Level_1' (SP level with 0 length), see error +# 06. Create entity E3/T3/A3=3, with servicepath '///Level_1///Level_2//' (SP level with 0 length), see error +# 07. Create entity E3/T3/A3=3, with servicepath '/Level_1///Level_2' (SP level with 0 length), see error +# 08. Create entity E3/T3/A3=3, with servicepath '/Level_1/Level_2//' (SP level with 0 length), see error +# 09. Create entity E4/T4/A4=4, with servicepath '/Level_1/Level_2/' (success case) +# 10. GET /v2/entities/E4, with servicepath '/Level_1/Level_2' (without slash) to see E4/T4/A4=4 +# + + +echo "01. Create entity E1/T1/A1=1, with servicepath '/Level_1' (success case)" +echo "========================================================================" +payload='{ + "id": "E1", + "type": "T1", + "A1": { + "value": 1, + "type": "Integer" + } +}' +orionCurl --url /v2/entities --payload "$payload" --servicePath /Level_1 +echo +echo + + +echo "02. GET /v2/entities/E1 with servicepath '/Level_1' to see E1/T1/A1=1" +echo "=====================================================================" +orionCurl --url /v2/entities/E1 --servicePath /Level_1 +echo +echo + + +echo "03. Create entity E2/T2/A2=2, with servicepath '/Level_1/Level_2' (success case)" +echo "================================================================================" +payload='{ + "id": "E2", + "type": "T2", + "A2": { + "value": 2, + "type": "Integer" + } +}' +orionCurl --url /v2/entities --payload "$payload" --servicePath /Level_1/Level_2 +echo +echo + + +echo "04. GET /v2/entities/E2, with servicepath '/Level_1/Level_2' to see E2/T2/A2=2" +echo "==============================================================================" +orionCurl --url /v2/entities/E2 --servicePath /Level_1/Level_2 +echo +echo + + +echo "05. Create entity E3/T3/A3=3, with servicepath '//Level_1' (SP level with 0 length), see 400 error" +echo "==================================================================================================" +payload='{ + "id": "E3", + "type": "T3", + "A3": { + "value": 3, + "type": "Integer" + } +}' +orionCurl --url /v2/entities --payload "$payload" --servicePath //Level_1 +echo +echo + + +echo "06. Create entity E3/T3/A3=3, with servicepath '///Level_1///Level_2//' (SP level with 0 length), see 400 error" +echo "===============================================================================================================" +payload='{ + "id": "E3", + "type": "T3", + "A3": { + "value": 3, + "type": "Integer" + } +}' +orionCurl --url /v2/entities --payload "$payload" --servicePath ///Level_1///Level_2// +echo +echo + + +echo "07. Create entity E3/T3/A3=3, with servicepath '/Level_1///Level_2' (SP level with 0 length), see error" +echo "=======================================================================================================" +payload='{ + "id": "E3", + "type": "T3", + "A3": { + "value": 3, + "type": "Integer" + } +}' +orionCurl --url /v2/entities --payload "$payload" --servicePath /Level_1///Level_2 +echo +echo + + +echo "08. Create entity E3/T3/A3=3, with servicepath '/Level_1/Level_2//' (SP level with 0 length), see error" +echo "=======================================================================================================" +payload='{ + "id": "E3", + "type": "T3", + "A3": { + "value": 3, + "type": "Integer" + } +}' +orionCurl --url /v2/entities --payload "$payload" --servicePath /Level_1/Level_2// +echo +echo + + +echo "09. Create entity E4/T4/A4=4, with servicepath '/Level_1/Level_2/' (success case)" +echo "=================================================================================" +payload='{ + "id": "E4", + "type": "T4", + "A4": { + "value": 4, + "type": "Integer" + } +}' +orionCurl --url /v2/entities --payload "$payload" --servicePath /Level_1/Level_2/ +echo +echo + + +echo "10. GET /v2/entities/E4, with servicepath '/Level_1/Level_2' (without slash) to see E4/T4/A4=4" +echo "==============================================================================================" +orionCurl --url /v2/entities/E4 --servicePath /Level_1/Level_2 +echo +echo + + +--REGEXPECT-- +01. Create entity E1/T1/A1=1, with servicepath '/Level_1' (success case) +======================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T1 +Content-Length: 0 + + + +02. GET /v2/entities/E1 with servicepath '/Level_1' to see E1/T1/A1=1 +===================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 71 + +{ + "A1": { + "metadata": {}, + "type": "Integer", + "value": 1 + }, + "id": "E1", + "type": "T1" +} + + +03. Create entity E2/T2/A2=2, with servicepath '/Level_1/Level_2' (success case) +================================================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T2 +Content-Length: 0 + + + +04. GET /v2/entities/E2, with servicepath '/Level_1/Level_2' to see E2/T2/A2=2 +============================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 71 + +{ + "A2": { + "metadata": {}, + "type": "Integer", + "value": 2 + }, + "id": "E2", + "type": "T2" +} + + +05. Create entity E3/T3/A3=3, with servicepath '//Level_1' (SP level with 0 length), see 400 error +================================================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 69 + +{ + "description": "empty component in ServicePath", + "error": "BadRequest" +} + + +06. Create entity E3/T3/A3=3, with servicepath '///Level_1///Level_2//' (SP level with 0 length), see 400 error +=============================================================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 69 + +{ + "description": "empty component in ServicePath", + "error": "BadRequest" +} + + +07. Create entity E3/T3/A3=3, with servicepath '/Level_1///Level_2' (SP level with 0 length), see error +======================================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 69 + +{ + "description": "empty component in ServicePath", + "error": "BadRequest" +} + + +08. Create entity E3/T3/A3=3, with servicepath '/Level_1/Level_2//' (SP level with 0 length), see error +======================================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 69 + +{ + "description": "empty component in ServicePath", + "error": "BadRequest" +} + + +09. Create entity E4/T4/A4=4, with servicepath '/Level_1/Level_2/' (success case) +================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E4?type=T4 +Content-Length: 0 + + + +10. GET /v2/entities/E4, with servicepath '/Level_1/Level_2' (without slash) to see E4/T4/A4=4 +============================================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 71 + +{ + "A4": { + "metadata": {}, + "type": "Integer", + "value": 4 + }, + "id": "E4", + "type": "T4" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB + From 8a6c762cdb408a9c8800db382932243fcf59254c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 16:48:39 +0100 Subject: [PATCH 156/390] FIX expression evaluation logic --- src/lib/common/macroSubstitute.cpp | 58 ++-- src/lib/jexl/JexlContext.cpp | 10 +- src/lib/jexl/JexlContext.h | 6 +- src/lib/jexl/JexlManager.cpp | 172 +++++++---- src/lib/jexl/JexlManager.h | 6 +- src/lib/jexl/JexlResult.cpp | 2 +- src/lib/ngsi/ContextAttribute.cpp | 10 +- src/lib/ngsiNotify/Notifier.cpp | 12 +- src/lib/parse/CompoundValueNode.cpp | 100 +++++++ src/lib/parse/CompoundValueNode.h | 3 + ...fication_templates_many_notifications.test | 7 +- .../jexl_basic.test | 279 ++++++++++++++++++ 12 files changed, 549 insertions(+), 116 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 530b7f4b3a..10c5ec0251 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -53,12 +53,15 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - //if (!jexlContextP->hasKey(macroName)) - //{ - // return notFoundDefault; - //} - - return jexlMgr.evaluate(jexlContextP, macroName).toString(); + JexlResult r = jexlMgr.evaluate(jexlContextP, macroName); + if (r.valueType == orion::ValueTypeNull) + { + return notFoundDefault; + } + else + { + return r.toString(); + } } else if (jexlContextP != NULL) { @@ -117,44 +120,27 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon // FIXME PR: inTheMiddle -> raw ? static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault, bool inTheMiddle) { - //if (!jexlContextP->hasKey(key)) - //{ - // return notFoundDefault; - //} - - std::string s = jexlMgr.evaluate(jexlContextP, key).toString(); + JexlResult r = jexlMgr.evaluate(jexlContextP, key); - if (inTheMiddle) - { - // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), - // so double quotes have to be be removed - if (s[0] == '"') - { - s = s.substr(1, s.size()-2); - } - } - - return s; - /*std::map::iterator iter = replacementsP->find(key); - if (iter == replacementsP->end()) + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } else { - // replacementsP contents are prepared for "full replacement" case, so string values use - // double quotes. But in this case we are in a "partial replacement" case, so we have - // to remove them if we find them - std::string value = iter->second; - if (value[0] == '"') - { - return value.substr(1, value.size()-2); - } - else + std::string s = r.toString(); + if (inTheMiddle) { - return value; + // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), + // so double quotes have to be be removed + if (s[0] == '"') + { + s = s.substr(1, s.size()-2); + } } - }*/ + + return s; + } } diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp index ba5975e6a1..82ab5e3dd4 100644 --- a/src/lib/jexl/JexlContext.cpp +++ b/src/lib/jexl/JexlContext.cpp @@ -166,12 +166,12 @@ bool JexlContext::hasKey(const std::string& key) /* **************************************************************************** * -* JexlContext::~JexlContext - +* JexlContext::release - */ -JexlContext::~JexlContext() +void JexlContext::release(void) { // FIXME PR: this is not correct. Recursively release of the dict object - Py_XDECREF(jexl_context); + //Py_XDECREF(jexl_context); } @@ -291,9 +291,9 @@ void JexlContextList::add(JexlContextList _jexlContextList) /* **************************************************************************** * -* JexlContextList::~JexlContextList - +* JexlContextList::relesase - */ -JexlContextList::~JexlContextList() +void JexlContextList::release(void) { // FIXME PR: this is not correct. Recursively release of the list object Py_XDECREF(jexl_context); diff --git a/src/lib/jexl/JexlContext.h b/src/lib/jexl/JexlContext.h index bf343b4e7e..4c91fc95a6 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/jexl/JexlContext.h @@ -46,8 +46,6 @@ class JexlContext PyObject* jexl_context; public: - ~JexlContext(); - JexlContext(); PyObject* get(void); @@ -58,6 +56,7 @@ class JexlContext void add(const std::string& key, JexlContext jexlContext); void add(const std::string& key, JexlContextList jexlContextList); bool hasKey(const std::string& key); + void release(void); }; class JexlContextList @@ -66,8 +65,6 @@ class JexlContextList PyObject* jexl_context; public: - ~JexlContextList(); - JexlContextList(); PyObject* get(void); @@ -77,6 +74,7 @@ class JexlContextList void add(void); void add(JexlContext jexlContext); void add(JexlContextList jexlContextList); + void release(void); }; diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index e8bd7bf5fb..f82f1f9e2b 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -71,8 +71,6 @@ static const char* capturePythonError() /* **************************************************************************** * * getPyObjectType - -* -* FIXME PR: ValueTypeVector and ValueTypeObject should be taken into account */ static orion::ValueType getPyObjectType(PyObject* obj) { @@ -107,14 +105,46 @@ static orion::ValueType getPyObjectType(PyObject* obj) +#if 0 +/* **************************************************************************** +* +* customJsonSerializerFunction - +* +* To be used in the json.dumps() step in JexlManager::evaluate() in order to get +* integer numbers when possible (i.e. "2" instead of "2.0"), this way behaving as +* the pre-JEXL macro replacement logic +*/ +static PyObject* customJsonSerializerFunction(PyObject* obj) +{ + // Check if the object is a float + if (PyFloat_Check(obj)) + { + // Convert float to integer if it represents a whole number + double value = PyFloat_AsDouble(obj); + if (value == (int)value) + { + return PyLong_FromDouble(value); + } + } + + // Return the original object + Py_INCREF(obj); + return obj; +} +#endif + + + /* **************************************************************************** * * JexlManager::init - */ void JexlManager::init(void) { - jexl_module = NULL; - jexl_engine = NULL; + pyjexlModule = NULL; + jsonModule = NULL; + jexlEngine = NULL; + //customJsonSerializer = NULL; if (sem_init(&sem, 0, 1) == -1) { @@ -124,19 +154,37 @@ void JexlManager::init(void) Py_Initialize(); LM_T(LmtJexl, ("Python interpreter has been initialized")); - jexl_module = PyImport_ImportModule("pyjexl"); - if (jexl_module == NULL) + pyjexlModule = PyImport_ImportModule("pyjexl"); + if (pyjexlModule == NULL) + { + const char* error = capturePythonError(); + LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", error)); + } + LM_T(LmtJexl, ("pyjexl module has been loaded")); + + jsonModule = PyImport_ImportModule("json"); + if (jsonModule == NULL) + { + const char* error = capturePythonError(); + LM_X(1, ("Fatal Error (error importing json module: %s)", error)); + } + LM_T(LmtJexl, ("json module has been loaded")); + +#if 0 + customJsonSerializer = PyCapsule_New((void*)customJsonSerializerFunction, NULL, NULL); + if (customJsonSerializer == NULL) { const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error importing jexl_module: %s)", error)); + LM_X(1, ("Fatal Error (error creating json custom serializer function: %s)", error)); } - LM_T(LmtJexl, ("jexl module has been loaded")); + LM_T(LmtJexl, ("json custom serializer has been createdmodule has been loaded")); +#endif - jexl_engine = PyObject_CallMethod(jexl_module, "JEXL", NULL); - if (jexl_engine == NULL) + jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); + if (jexlEngine == NULL) { const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error creating jexl_engine: %s)", error)); + LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", error)); } LM_T(LmtJexl, ("jexl engine has been created")); } @@ -153,32 +201,32 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ JexlResult r; - // If nothing changes, the returned value is null + // If nothing changes, the returned value would be null (failsafe) r.valueType = orion::ValueTypeNull; PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error building expression: %s", capturePythonError())); return r; } - PyObject* result = PyObject_CallMethod(jexl_engine, "evaluate", "OO", expression, jexlContextP->get()); + PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, jexlContextP->get()); Py_XDECREF(expression); if (result == NULL) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error evaluating expression: %s", capturePythonError())); return r; } // Special case: expresion evalutes to None if (result == Py_None) { - Py_XDECREF(result); - r.valueType = orion::ValueTypeNull; LM_T(LmtJexl, ("JEXL evaluation result is null")); + r.valueType = orion::ValueTypeNull; + Py_XDECREF(result); return r; } @@ -196,59 +244,53 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ LM_T(LmtJexl, ("JEXL evaluation result (bool): %s", r.boolValue ? "true": "false")); Py_XDECREF(result); } - else if (r.valueType == orion::ValueTypeObject) - { - // FIXME PR: same processing than string? Unify? - const char* repr = PyUnicode_AsUTF8(result); - Py_XDECREF(result); - if (repr == NULL) - { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); - } - else - { - r.stringValue = std::string(repr); - LM_T(LmtJexl, ("JEXL evaluation result (object): %s", r.stringValue.c_str())); - } - } - else if (r.valueType == orion::ValueTypeVector) + else if ((r.valueType == orion::ValueTypeObject) || (r.valueType == orion::ValueTypeVector)) { - // FIXME PR: same processing than string? Unify? - const char* repr = PyUnicode_AsUTF8(result); + // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot + // be used, as it produce string with ' instead of " in the resulting JSON string. + // FIXME PR: is the customJsonSerializer going to be used at the end? + PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); Py_XDECREF(result); if (repr == NULL) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); + r.valueType = orion::ValueTypeNull; } else { - r.stringValue = std::string(repr); - LM_T(LmtJexl, ("JEXL evaluation result (list): %s", r.stringValue.c_str())); + const char* str = PyUnicode_AsUTF8(repr); + Py_XDECREF(repr); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); + r.valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtJexl, ("JEXL evaluation result (object or vector): %s", str)); + r.stringValue = std::string(str); + } } } else if (r.valueType == orion::ValueTypeString) { - const char* repr = PyUnicode_AsUTF8(result); + const char* str = PyUnicode_AsUTF8(result); Py_XDECREF(result); - if (repr == NULL) + if (str == NULL) { - // FIXME PR: grab error message from Python stack, use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression, result is null")); + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + r.valueType = orion::ValueTypeNull; } else { - r.stringValue = std::string(repr); - LM_T(LmtJexl, ("JEXL evaluation result (string): %s", r.stringValue.c_str())); + LM_T(LmtJexl, ("JEXL evaluation result (string): %s", str)); + r.stringValue = std::string(str); } } - //if (result_str[0] == '\'') - //{ - // result_str = result_str.substr(1, result_str.size()-2); - //} - return r; } @@ -260,16 +302,30 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ */ void JexlManager::release(void) { - if (jexl_engine != NULL) + if (jexlEngine != NULL) { - Py_XDECREF(jexl_engine); + Py_XDECREF(jexlEngine); LM_T(LmtJexl, ("jexl engine has been freed")); } - if (jexl_module != NULL) + if (pyjexlModule != NULL) + { + Py_XDECREF(pyjexlModule); + LM_T(LmtJexl, ("pyjexl module has been freed")); + } + +#if 0 + if (customJsonSerializer != NULL) + { + Py_XDECREF(customJsonSerializer); + LM_T(LmtJexl, ("json custom serializer module has been freed")); + } +#endif + + if (jsonModule != NULL) { - Py_XDECREF(jexl_module); - LM_T(LmtJexl, ("jexl module has been freed")); + Py_XDECREF(jsonModule); + LM_T(LmtJexl, ("json module has been freed")); } Py_Finalize(); diff --git a/src/lib/jexl/JexlManager.h b/src/lib/jexl/JexlManager.h index ad83fd56ea..adce908c7c 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/jexl/JexlManager.h @@ -39,8 +39,10 @@ class JexlManager { private: - PyObject* jexl_module; - PyObject* jexl_engine; + PyObject* pyjexlModule; + PyObject* jexlEngine; + PyObject* jsonModule; + //PyObject* customJsonSerializer; sem_t sem; public: diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 1197b1491a..2769844403 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -53,7 +53,7 @@ std::string JexlResult::toString(void) } else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) { - return toJsonString(stringValue); + return stringValue; } else if (valueType == orion::ValueTypeNull) { diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 534a1f93de..725e159ed9 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1300,8 +1300,14 @@ void ContextAttribute::addToContext(JexlContext* jexlContextP) { if (compoundValueP != NULL) { - // FIXME PR: not sure if this has the proper effect... - jexlContextP->add(name, compoundValueP->toJson()); + if (valueType == orion::ValueTypeObject) + { + jexlContextP->add(name, compoundValueP->toJexlContext()); + } + else // valueType == orion::ValueTypeVector + { + jexlContextP->add(name, compoundValueP->toJexlContextList()); + } } else if (valueType == orion::ValueTypeNumber) { diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 24c6953ef0..4ab2b5262f 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -338,8 +338,6 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - //std::map replacements; - //buildReplacementsMap(en, tenant, xauthToken, &replacements); JexlContext jexlContext; jexlContext.add("id", en.id); jexlContext.add("type", en.type); @@ -348,8 +346,6 @@ static SenderThreadParams* buildSenderParamsCustom jexlContext.add("authToken", xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - // FIXME PR: this works with every attribute type? (number, bool, etc.) - //jexlContext.add(en.attributeVector[ix]->name, en.attributeVector[ix]->toJsonValue()); en.attributeVector[ix]->addToContext(&jexlContext); } @@ -380,6 +376,7 @@ static SenderThreadParams* buildSenderParamsCustom if (macroSubstitute(&url, notifUrl, &jexlContext, "", true) == false) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } @@ -396,6 +393,7 @@ static SenderThreadParams* buildSenderParamsCustom if (!setPayload(includePayload, notifPayload, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } } @@ -412,6 +410,7 @@ static SenderThreadParams* buildSenderParamsCustom if (!setNgsiPayload(ngsi, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } mimeType = "application/json"; @@ -431,6 +430,7 @@ static SenderThreadParams* buildSenderParamsCustom if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } @@ -457,6 +457,7 @@ static SenderThreadParams* buildSenderParamsCustom if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } @@ -488,6 +489,7 @@ static SenderThreadParams* buildSenderParamsCustom if (!parseUrl(url, host, port, uriPath, protocol)) { LM_E(("Runtime Error (not sending notification: malformed URL: '%s')", url.c_str())); + jexlContext.release(); return NULL; } @@ -522,6 +524,7 @@ static SenderThreadParams* buildSenderParamsCustom if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "", true) == false) { // Warning already logged in macroSubstitute() + jexlContext.release(); return NULL; } } @@ -557,6 +560,7 @@ static SenderThreadParams* buildSenderParamsCustom snprintf(suffix, sizeof(suffix), "%u", correlatorCounter); paramsP->fiwareCorrelator = fiwareCorrelator + "; cbnotif=" + suffix; + jexlContext.release(); return paramsP; } diff --git a/src/lib/parse/CompoundValueNode.cpp b/src/lib/parse/CompoundValueNode.cpp index 6dfb6f59d4..f9e45c1555 100644 --- a/src/lib/parse/CompoundValueNode.cpp +++ b/src/lib/parse/CompoundValueNode.cpp @@ -658,6 +658,106 @@ std::string CompoundValueNode::toJson(JexlContext* jexlContextP) +/* **************************************************************************** +* +* CompoundValueNode:toJexlContext +* +*/ +JexlContext CompoundValueNode::toJexlContext(void) +{ + JexlContext jc; + for (uint64_t ix = 0; ix < childV.size(); ++ix) + { + CompoundValueNode* child = childV[ix]; + switch (child->valueType) + { + case orion::ValueTypeString: + jc.add(child->name, child->stringValue); + break; + + case orion::ValueTypeNumber: + jc.add(child->name, child->numberValue); + break; + + case orion::ValueTypeBoolean: + jc.add(child->name, child->boolValue); + break; + + case orion::ValueTypeNull: + jc.add(child->name); + break; + + case orion::ValueTypeVector: + jc.add(child->name, child->toJexlContextList()); + break; + + case orion::ValueTypeObject: + jc.add(child->name, child->toJexlContext()); + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given (%s))", name.c_str())); + break; + + default: + LM_E(("Runtime Error (value type unknown (%s))", name.c_str())); + } + } + return jc; +} + + + +/* **************************************************************************** +* +* CompoundValueNode:toJexlContextList +* +*/ +JexlContextList CompoundValueNode::toJexlContextList(void) +{ + JexlContextList jcl; + for (uint64_t ix = 0; ix < childV.size(); ++ix) + { + CompoundValueNode* child = childV[ix]; + switch (child->valueType) + { + case orion::ValueTypeString: + jcl.add(child->stringValue); + break; + + case orion::ValueTypeNumber: + jcl.add(child->numberValue); + break; + + case orion::ValueTypeBoolean: + jcl.add(child->boolValue); + break; + + case orion::ValueTypeNull: + jcl.add(); + break; + + case orion::ValueTypeVector: + jcl.add(child->toJexlContextList()); + break; + + case orion::ValueTypeObject: + jcl.add(child->toJexlContext()); + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given)")); + break; + + default: + LM_E(("Runtime Error (value type unknown)")); + } + } + return jcl; +} + + + /* **************************************************************************** * * clone - diff --git a/src/lib/parse/CompoundValueNode.h b/src/lib/parse/CompoundValueNode.h index 12fd070a31..a77eb1a393 100644 --- a/src/lib/parse/CompoundValueNode.h +++ b/src/lib/parse/CompoundValueNode.h @@ -121,6 +121,9 @@ class CompoundValueNode std::string toJson(JexlContext* jexlContextP = NULL); + JexlContext toJexlContext(void); + JexlContextList toJexlContextList(void); + void shortShow(const std::string& indent); void show(const std::string& indent); diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test index 67f39b8667..341c55b7b9 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test @@ -348,22 +348,21 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a2=null&id=E1&type=T2 +PUT http://127.0.0.1:REGEX(\d+)/notify?id=E1&type=T2 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 32 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json -A2: null Entity-Type: T2 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { "A1": "", - "A2": "null", + "A2": "", "A3": "" } #SORT_END diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test new file mode 100644 index 0000000000..209b61474c --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test @@ -0,0 +1,279 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression sum: A+B +# 02. Create entity E1 with A=1 and B=2 +# 03. Update entity E1 with A=foo and B=bar +# 04. Update entity E1 with A=2.1 and B=-3.8 +# 05. Dump accumulator and see two notifications (sum: 3 and sum: foobar) +# + + +echo "01. Create custom sub with custom expression sum: A+B" +echo "=====================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "sum": { + "value": "${A+B}", + "type": "Calculated" + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1 and B=2" +echo "=====================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A=foo and B=bar" +echo "=========================================" +payload='{ + "A": { + "value": "foo", + "type": "Text" + }, + "B": { + "value": "bar", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Update entity E1 with A=2.1 and B=-3.8" +echo "==========================================" +payload='{ + "A": { + "value": 2.1, + "type": "Number" + }, + "B": { + "value": -3.8, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1,7)" +echo "=================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: A+B +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 and B=2 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A=foo and B=bar +========================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Update entity E1 with A=2.1 and B=-3.8 +========================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1,7) +================================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 221 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 232 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Text", + "value": "foo" + }, + "B": { + "metadata": {}, + "type": "Text", + "value": "bar" + }, + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": "foobar" + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 229 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2.1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": -3.8 + }, + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": -1.7 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 7231ad45a0f942aee4f2c1aab9648c36bef786bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 18:58:27 +0100 Subject: [PATCH 157/390] FIX value not given processing logic --- src/lib/common/macroSubstitute.cpp | 4 +-- src/lib/jexl/JexlManager.cpp | 4 +-- ...fication_templates_many_notifications.test | 33 +++++++++++-------- .../covered_custom_notification.test | 4 +-- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 10c5ec0251..fd6efea913 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -54,7 +54,7 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon std::string macroName = stringValue.substr(2, stringValue.size() - 3); JexlResult r = jexlMgr.evaluate(jexlContextP, macroName); - if (r.valueType == orion::ValueTypeNull) + if (r.valueType == orion::ValueTypeNotGiven) { return notFoundDefault; } @@ -122,7 +122,7 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st { JexlResult r = jexlMgr.evaluate(jexlContextP, key); - if (r.valueType == orion::ValueTypeNull) + if (r.valueType == orion::ValueTypeNotGiven) { return notFoundDefault; } diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index f82f1f9e2b..286b3e6260 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -201,8 +201,8 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ JexlResult r; - // If nothing changes, the returned value would be null (failsafe) - r.valueType = orion::ValueTypeNull; + // If nothing changes, the returned value would be ValueTypeNotGiven + r.valueType = orion::ValueTypeNotGiven; PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test index 341c55b7b9..774a462d13 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test @@ -277,23 +277,25 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&a2=null&a3=null&id=E1&type=T1 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 44 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json A1: true +A2: null +A3: null Entity-Type: T1 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { "A1": "true", - "A2": "", - "A3": "" + "A2": "null", + "A3": "null" } #SORT_END #SORT_START @@ -348,22 +350,25 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?id=E1&type=T2 +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=null&a2=null&a3=null&id=E1&type=T2 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 32 +Content-Length: 44 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json +A1: null +A2: null +A3: null Entity-Type: T2 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "", - "A2": "", - "A3": "" + "A1": "null", + "A2": "null", + "A3": "null" } #SORT_END #SORT_START @@ -418,11 +423,13 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=null&a2=null&a3=3&id=E1&type=T3 Fiware-Servicepath: / +A1: null +A2: null A3: 3 Entity-Id: E1 -Content-Length: 33 +Content-Length: 41 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -432,8 +439,8 @@ Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "", - "A2": "", + "A1": "null", + "A2": "null", "A3": "3" } ======================================= diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test index 460ffe731a..33c6efe8b2 100644 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test @@ -199,7 +199,7 @@ Content-Length: 0 ======================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 31 +Content-Length: 35 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -207,7 +207,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": }======================================= +{ "A1-value": 1, "A2-value": null }======================================= --TEARDOWN-- From 06e0d98c95c33cb76e7c907ef3522f9c14d80d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 14 Feb 2024 20:02:30 +0100 Subject: [PATCH 158/390] FIX ftest --- .../notification_templates.test | 7 ++++--- .../http_info_for_sub_update.test | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates.test b/test/functionalTest/cases/2015_notification_templates/notification_templates.test index 23df76a2fe..7ff8e92606 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates.test @@ -536,10 +536,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) "A3": "13" } ======================================= -POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&id=E1&step=07&type=Thing +POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&a4=null&id=E1&step=07&type=Thing Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 40 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Step: 07 Ngsiv2-Attrsformat: custom @@ -547,6 +547,7 @@ Host: 127.0.0.1:REGEX(\d+) Accept: application/json A2: a2 A1: a1 +A4: null Entity-Type: Thing Content-Type: user-specified Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) @@ -554,7 +555,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) { "A1": "a1", "A2": "a2", - "A4": "" + "A4": "null" } ======================================= #SORT_END diff --git a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test index 7a4956a172..00a402dfd1 100644 --- a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test +++ b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test @@ -297,7 +297,7 @@ Content-Length: 0 07. Dump accumulator ==================== -PATCH http://localhost:REGEX(\d+)/notify +PATCH http://localhost:REGEX(\d+)/notify?Q2=null Fiware-Servicepath: / Content-Length: 36 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) @@ -306,6 +306,7 @@ Host: localhost:REGEX(\d+) Accept: application/json Content-Type: application/json Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 +H2: null { "What": "PAYLOAD2", From 444b6e241b3a8fda1f0446e6bf32629eb6e8d88a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 15 Feb 2024 11:48:18 +0100 Subject: [PATCH 159/390] Revert "FIX value not given processing logic" This reverts commit 7231ad45a0f942aee4f2c1aab9648c36bef786bb. --- src/lib/common/macroSubstitute.cpp | 4 +-- src/lib/jexl/JexlManager.cpp | 4 +-- ...fication_templates_many_notifications.test | 33 ++++++++----------- .../covered_custom_notification.test | 4 +-- 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index fd6efea913..10c5ec0251 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -54,7 +54,7 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon std::string macroName = stringValue.substr(2, stringValue.size() - 3); JexlResult r = jexlMgr.evaluate(jexlContextP, macroName); - if (r.valueType == orion::ValueTypeNotGiven) + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } @@ -122,7 +122,7 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st { JexlResult r = jexlMgr.evaluate(jexlContextP, key); - if (r.valueType == orion::ValueTypeNotGiven) + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index 286b3e6260..f82f1f9e2b 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -201,8 +201,8 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ JexlResult r; - // If nothing changes, the returned value would be ValueTypeNotGiven - r.valueType = orion::ValueTypeNotGiven; + // If nothing changes, the returned value would be null (failsafe) + r.valueType = orion::ValueTypeNull; PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test index 774a462d13..341c55b7b9 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test @@ -277,25 +277,23 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&a2=null&a3=null&id=E1&type=T1 +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 44 +Content-Length: 36 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json A1: true -A2: null -A3: null Entity-Type: T1 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { "A1": "true", - "A2": "null", - "A3": "null" + "A2": "", + "A3": "" } #SORT_END #SORT_START @@ -350,25 +348,22 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=null&a2=null&a3=null&id=E1&type=T2 +PUT http://127.0.0.1:REGEX(\d+)/notify?id=E1&type=T2 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 44 +Content-Length: 32 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) Accept: application/json -A1: null -A2: null -A3: null Entity-Type: T2 Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "null", - "A2": "null", - "A3": "null" + "A1": "", + "A2": "", + "A3": "" } #SORT_END #SORT_START @@ -423,13 +418,11 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) "subscriptionId": "REGEX([0-9a-f]{24})" } ======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=null&a2=null&a3=3&id=E1&type=T3 +PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 Fiware-Servicepath: / -A1: null -A2: null A3: 3 Entity-Id: E1 -Content-Length: 41 +Content-Length: 33 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -439,8 +432,8 @@ Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "null", - "A2": "null", + "A1": "", + "A2": "", "A3": "3" } ======================================= diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test index 33c6efe8b2..460ffe731a 100644 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test @@ -199,7 +199,7 @@ Content-Length: 0 ======================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 35 +Content-Length: 31 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -207,7 +207,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": null }======================================= +{ "A1-value": 1, "A2-value": }======================================= --TEARDOWN-- From e20b1e3adf0236c6941a1b2ec0fb7e771b41c488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 15 Feb 2024 11:51:47 +0100 Subject: [PATCH 160/390] Revert "FIX ftest" This reverts commit 06e0d98c95c33cb76e7c907ef3522f9c14d80d96. --- .../notification_templates.test | 7 +++---- .../http_info_for_sub_update.test | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates.test b/test/functionalTest/cases/2015_notification_templates/notification_templates.test index 7ff8e92606..23df76a2fe 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates.test @@ -536,10 +536,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) "A3": "13" } ======================================= -POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&a4=null&id=E1&step=07&type=Thing +POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&id=E1&step=07&type=Thing Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 40 +Content-Length: 36 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Step: 07 Ngsiv2-Attrsformat: custom @@ -547,7 +547,6 @@ Host: 127.0.0.1:REGEX(\d+) Accept: application/json A2: a2 A1: a1 -A4: null Entity-Type: Thing Content-Type: user-specified Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) @@ -555,7 +554,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) { "A1": "a1", "A2": "a2", - "A4": "null" + "A4": "" } ======================================= #SORT_END diff --git a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test index 00a402dfd1..7a4956a172 100644 --- a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test +++ b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test @@ -297,7 +297,7 @@ Content-Length: 0 07. Dump accumulator ==================== -PATCH http://localhost:REGEX(\d+)/notify?Q2=null +PATCH http://localhost:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 36 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) @@ -306,7 +306,6 @@ Host: localhost:REGEX(\d+) Accept: application/json Content-Type: application/json Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -H2: null { "What": "PAYLOAD2", From ae3d83b7df3f71d0572b802e4e63ebcbec4054e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 15 Feb 2024 12:00:15 +0100 Subject: [PATCH 161/390] FIX ftest --- .../covered_custom_notification.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test index 460ffe731a..6d40e8e28b 100644 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test @@ -174,7 +174,7 @@ Content-Length: 0 ====================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 35 +Content-Length: 31 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -182,7 +182,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": null }======================================= +{ "A1-value": 1, "A2-value": }======================================= 05. Create E2 with attribute A1=1 From a57599d45a6d32c1f8e84b17c117389c5850fc9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 16 Feb 2024 12:36:00 +0100 Subject: [PATCH 162/390] FIX add CompoundValue to JexlResult --- src/lib/jexl/JexlResult.cpp | 26 ++++++++++++++++++++++++-- src/lib/jexl/JexlResult.h | 8 +++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 2769844403..f4cf5b044d 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -35,7 +35,6 @@ * * Pretty similar to ContextAttribute::toJsonValue() * -* FIXME PR: ValueTypeVector and ValueTypeObject should be taken into account */ std::string JexlResult::toString(void) { @@ -53,7 +52,15 @@ std::string JexlResult::toString(void) } else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) { - return stringValue; + if (compoundValueP != NULL) + { + return compoundValueP->toJson(); + } + else + { + LM_E(("Runtime Error (result is vector/object but compountValue is NULL)")); + return ""; + } } else if (valueType == orion::ValueTypeNull) { @@ -64,4 +71,19 @@ std::string JexlResult::toString(void) LM_E(("Runtime Error (not allowed type in JexlResult: %s)", valueTypeName(valueType))); return ""; } +} + + + +/* **************************************************************************** +* +* JexlResult::release - +*/ +void JexlResult::release(void) +{ + if (compoundValueP != NULL) + { + delete compoundValueP; + compoundValueP = NULL; + } } \ No newline at end of file diff --git a/src/lib/jexl/JexlResult.h b/src/lib/jexl/JexlResult.h index a3e0709820..34c8effdc7 100644 --- a/src/lib/jexl/JexlResult.h +++ b/src/lib/jexl/JexlResult.h @@ -28,6 +28,8 @@ #include "orionTypes/OrionValueType.h" +#include "parse/CompoundValueNode.h" + #include /* **************************************************************************** @@ -44,7 +46,11 @@ class JexlResult double numberValue; // "value" as a Number bool boolValue; // "value" as a Boolean - std::string toString(void); + // Use only when valueType is object or vector + orion::CompoundValueNode* compoundValueP; + + std::string toString(void); + void release(void); }; From 9ea1c8291247dc44467c4c92913fa55c1e2f32a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 16 Feb 2024 16:39:33 +0100 Subject: [PATCH 163/390] FIX JexlResult improvements (wip) --- src/lib/jexl/JexlManager.cpp | 48 ++------- src/lib/jexl/JexlResult.cpp | 202 +++++++++++++++++++++++++++++++++++ src/lib/jexl/JexlResult.h | 6 ++ 3 files changed, 214 insertions(+), 42 deletions(-) diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index f82f1f9e2b..bfb261f944 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -68,42 +68,6 @@ static const char* capturePythonError() } -/* **************************************************************************** -* -* getPyObjectType - -*/ -static orion::ValueType getPyObjectType(PyObject* obj) -{ - // PyBool_Check() has to be donde before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be - // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 - if (PyBool_Check(obj)) - { - return orion::ValueTypeBoolean; - } - else if (PyLong_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyFloat_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyDict_Check(obj)) - { - return orion::ValueTypeObject; - } - else if (PyList_Check(obj)) - { - return orion::ValueTypeVector; - } - else - { - // For other types we use string (this is also a failsafe for types not being strings) - return orion::ValueTypeString; - } -} - - #if 0 /* **************************************************************************** @@ -197,12 +161,9 @@ void JexlManager::init(void) */ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { - LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - JexlResult r; - // If nothing changes, the returned value would be null (failsafe) - r.valueType = orion::ValueTypeNull; + LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) @@ -221,8 +182,11 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ return r; } + r.fill(result); + Py_XDECREF(result); + // Special case: expresion evalutes to None - if (result == Py_None) + /*if (result == Py_None) { LM_T(LmtJexl, ("JEXL evaluation result is null")); r.valueType = orion::ValueTypeNull; @@ -289,7 +253,7 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ LM_T(LmtJexl, ("JEXL evaluation result (string): %s", str)); r.stringValue = std::string(str); } - } + }*/ return r; } diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index f4cf5b044d..d103b4edff 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -29,6 +29,208 @@ #include "common/JsonHelper.h" #include "logMsg/logMsg.h" + + +/* **************************************************************************** +* +* capturePythonError - +* +* FIXME PR: duplicate code. Unify +*/ +static const char* capturePythonError() +{ + if (PyErr_Occurred()) + { + PyObject* ptype; + PyObject* pvalue; + PyObject* ptraceback; + + // Fetch the exception type, value, and traceback + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + + if (pvalue != NULL) + { + PyObject* str_obj = PyObject_Str(pvalue); + const char* error_message = PyUnicode_AsUTF8(str_obj); + + // Release the Python objects + Py_XDECREF(str_obj); + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + + return error_message; + } + } + + PyErr_Clear(); + return ""; +} + + + +/* **************************************************************************** +* +* getPyObjectType - +*/ +static orion::ValueType getPyObjectType(PyObject* obj) +{ + // PyBool_Check() has to be done before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be + // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 + if (PyBool_Check(obj)) + { + return orion::ValueTypeBoolean; + } + else if (PyLong_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyFloat_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyDict_Check(obj)) + { + return orion::ValueTypeObject; + } + else if (PyList_Check(obj)) + { + return orion::ValueTypeVector; + } + else + { + // For other types we use string (this is also a failsafe for types not being strings) + return orion::ValueTypeString; + } +} + + + +/* **************************************************************************** +* +* fill - +* +*/ +void JexlResult::fill(PyObject* result) +{ + // If nothing changes, the returned value would be null (failsafe) + valueType = orion::ValueTypeNull; + + // Special case: expresion evalutes to None + if (result == Py_None) + { + LM_T(LmtJexl, ("JexlResult is null")); + valueType = orion::ValueTypeNull; + return; + } + + // Other not null types + // FIXME PR: integrate getPyObjetType function into this one + valueType = getPyObjectType(result); + if (valueType == orion::ValueTypeNumber) + { + numberValue = PyFloat_AsDouble(result); + LM_T(LmtJexl, ("JexlResult (double): %f", numberValue)); + } + else if (valueType == orion::ValueTypeBoolean) + { + boolValue = PyObject_IsTrue(result); + LM_T(LmtJexl, ("JexlResult (bool): %s", boolValue ? "true": "false")); + } + else if (valueType == orion::ValueTypeObject) + { + processDict(compoundValueP, result); + // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot + // be used, as it produce string with ' instead of " in the resulting JSON string. + // FIXME PR: is the customJsonSerializer going to be used at the end? + /*PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); + if (repr == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + const char* str = PyUnicode_AsUTF8(repr); + Py_XDECREF(repr); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtJexl, ("JexlResult (object or vector): %s", str)); + stringValue = std::string(str); + } + }*/ + } + else if (valueType == orion::ValueTypeVector) + { + processList(compoundValueP, result); + // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot + // be used, as it produce string with ' instead of " in the resulting JSON string. + // FIXME PR: is the customJsonSerializer going to be used at the end? + /*PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); + if (repr == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + const char* str = PyUnicode_AsUTF8(repr); + Py_XDECREF(repr); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtJexl, ("JexlResult (object or vector): %s", str)); + stringValue = std::string(str); + } + }*/ + } + else if (valueType == orion::ValueTypeString) + { + const char* str = PyUnicode_AsUTF8(result); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtJexl, ("JexlResult (string): %s", str)); + stringValue = std::string(str); + } + } +} + + + + void JexlResult::processList(orion::CompoundValueNode* node, PyObject* result) + { + + } + + + + + void JexlResult::processDict(orion::CompoundValueNode* node, PyObject* result) + { + + } + + + /* **************************************************************************** * * toString - diff --git a/src/lib/jexl/JexlResult.h b/src/lib/jexl/JexlResult.h index 34c8effdc7..86144b8f07 100644 --- a/src/lib/jexl/JexlResult.h +++ b/src/lib/jexl/JexlResult.h @@ -30,6 +30,7 @@ #include "parse/CompoundValueNode.h" +#include #include /* **************************************************************************** @@ -49,6 +50,11 @@ class JexlResult // Use only when valueType is object or vector orion::CompoundValueNode* compoundValueP; + void fill(PyObject* result); + + void processList(orion::CompoundValueNode* node, PyObject* result); + void processDict(orion::CompoundValueNode* node, PyObject* result); + std::string toString(void); void release(void); }; From f9f9ed6bd954ae509751cbf61d2da488aecc0f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 09:36:49 +0100 Subject: [PATCH 164/390] FIX JexlResult improvements (wip) --- src/lib/jexl/JexlResult.cpp | 146 ++++++++++++++++++++++++++++++++++-- src/lib/jexl/JexlResult.h | 4 +- 2 files changed, 140 insertions(+), 10 deletions(-) diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index d103b4edff..4923fafb6b 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -125,7 +125,6 @@ void JexlResult::fill(PyObject* result) } // Other not null types - // FIXME PR: integrate getPyObjetType function into this one valueType = getPyObjectType(result); if (valueType == orion::ValueTypeNumber) { @@ -139,7 +138,13 @@ void JexlResult::fill(PyObject* result) } else if (valueType == orion::ValueTypeObject) { - processDict(compoundValueP, result); + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(result, &pos, &key, &value)) + { + processDictItem(compoundValueP, key, value); + } // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot // be used, as it produce string with ' instead of " in the resulting JSON string. // FIXME PR: is the customJsonSerializer going to be used at the end? @@ -169,7 +174,12 @@ void JexlResult::fill(PyObject* result) } else if (valueType == orion::ValueTypeVector) { - processList(compoundValueP, result); + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); + Py_ssize_t size = PyList_Size(result); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + processListItem(compoundValueP, PyList_GetItem(result, ix)); + } // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot // be used, as it produce string with ' instead of " in the resulting JSON string. // FIXME PR: is the customJsonSerializer going to be used at the end? @@ -216,18 +226,138 @@ void JexlResult::fill(PyObject* result) - void JexlResult::processList(orion::CompoundValueNode* node, PyObject* result) +/* **************************************************************************** +* +* processListItem - +* +* FIXME PR: maybe this should be static function out of the class? +*/ +void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* item) +{ + orion::CompoundValueNode* nodeP; + const char* str; + double d; + bool b; + switch (getPyObjectType(item)) { - + case orion::ValueTypeString: + str = PyUnicode_AsUTF8(item); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + } + else + { + LM_T(LmtJexl, ("processListITem (string): %s", str)); + nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); + parentP->add(nodeP); + } + break; + + case orion::ValueTypeNumber: + d = PyFloat_AsDouble(item); + LM_T(LmtJexl, ("processList (double): %f", d)); + nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); + parentP->add(nodeP); + break; + + case orion::ValueTypeBoolean: + b = PyObject_IsTrue(item); + LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); + nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); + parentP->add(nodeP); + break; + + case orion::ValueTypeNull: + nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeNull); + parentP->add(nodeP); + break; + + case orion::ValueTypeVector: + // TBD + break; + + case orion::ValueTypeObject: + // TBD + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given))")); + break; + + default: + LM_E(("Runtime Error (value type unknown))")); } +} - - void JexlResult::processDict(orion::CompoundValueNode* node, PyObject* result) +/* **************************************************************************** +* +* processDictItem - +* +* FIXME PR: maybe this should be static function out of the class? +*/ +void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +{ + std::string keyStr = ""; + orion::CompoundValueNode* nodeP; + const char* str; + double d; + bool b; + switch (getPyObjectType(value)) { - + case orion::ValueTypeString: + str = PyUnicode_AsUTF8(value); + if (str == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + } + else + { + LM_T(LmtJexl, ("processListITem (string): %s", str)); + nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); + parentP->add(nodeP); + } + break; + + case orion::ValueTypeNumber: + d = PyFloat_AsDouble(value); + LM_T(LmtJexl, ("processList (double): %f", d)); + nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); + parentP->add(nodeP); + break; + + case orion::ValueTypeBoolean: + b = PyObject_IsTrue(value); + LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); + nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); + parentP->add(nodeP); + break; + + case orion::ValueTypeNull: + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeNull); + parentP->add(nodeP); + break; + + case orion::ValueTypeVector: + // TBD + break; + + case orion::ValueTypeObject: + // TBD + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given))")); + break; + + default: + LM_E(("Runtime Error (value type unknown))")); } +} diff --git a/src/lib/jexl/JexlResult.h b/src/lib/jexl/JexlResult.h index 86144b8f07..cdf853c24a 100644 --- a/src/lib/jexl/JexlResult.h +++ b/src/lib/jexl/JexlResult.h @@ -52,8 +52,8 @@ class JexlResult void fill(PyObject* result); - void processList(orion::CompoundValueNode* node, PyObject* result); - void processDict(orion::CompoundValueNode* node, PyObject* result); + void processListItem(orion::CompoundValueNode* parentP, PyObject* item); + void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); std::string toString(void); void release(void); From d7b3a66dcbd4df20ccdc7c27336bbe32998fd70d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 10:31:57 +0100 Subject: [PATCH 165/390] FIX JexlResult improvements --- src/lib/jexl/JexlResult.cpp | 62 +++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 4923fafb6b..6a77126be7 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -97,6 +97,10 @@ static orion::ValueType getPyObjectType(PyObject* obj) { return orion::ValueTypeVector; } + else if (obj == Py_None) + { + return orion::ValueTypeNull; + } else { // For other types we use string (this is also a failsafe for types not being strings) @@ -232,16 +236,20 @@ void JexlResult::fill(PyObject* result) * * FIXME PR: maybe this should be static function out of the class? */ -void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* item) +void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* value) { orion::CompoundValueNode* nodeP; + const char* str; double d; bool b; - switch (getPyObjectType(item)) + PyObject *keyAux, *valueAux; + Py_ssize_t pos, size; + + switch (getPyObjectType(value)) { case orion::ValueTypeString: - str = PyUnicode_AsUTF8(item); + str = PyUnicode_AsUTF8(value); if (str == NULL) { // FIXME PR: use LM_E/LM_W? @@ -256,14 +264,14 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* it break; case orion::ValueTypeNumber: - d = PyFloat_AsDouble(item); + d = PyFloat_AsDouble(value); LM_T(LmtJexl, ("processList (double): %f", d)); nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); parentP->add(nodeP); break; case orion::ValueTypeBoolean: - b = PyObject_IsTrue(item); + b = PyObject_IsTrue(value); LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); parentP->add(nodeP); @@ -275,11 +283,23 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* it break; case orion::ValueTypeVector: - // TBD + nodeP = new orion::CompoundValueNode(orion::ValueTypeVector); + size = PyList_Size(value); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + processListItem(nodeP, PyList_GetItem(value, ix)); + } + parentP->add(nodeP); break; case orion::ValueTypeObject: - // TBD + nodeP = new orion::CompoundValueNode(orion::ValueTypeObject); + pos = 0; + while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + { + processDictItem(nodeP, keyAux, valueAux); + } + parentP->add(nodeP); break; case orion::ValueTypeNotGiven: @@ -301,11 +321,21 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* it */ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) { - std::string keyStr = ""; + const char * keyStr = PyUnicode_AsUTF8(key); + if (keyStr == NULL) + { + // FIXME PR: use LM_E/LM_W? + LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + return; + } + orion::CompoundValueNode* nodeP; const char* str; double d; bool b; + PyObject *keyAux, *valueAux; + Py_ssize_t pos, size; + switch (getPyObjectType(value)) { case orion::ValueTypeString: @@ -343,11 +373,23 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke break; case orion::ValueTypeVector: - // TBD + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeVector); + size = PyList_Size(value); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + processListItem(nodeP, PyList_GetItem(value, ix)); + } + parentP->add(nodeP); break; case orion::ValueTypeObject: - // TBD + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeObject); + pos = 0; + while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + { + processDictItem(nodeP, keyAux, valueAux); + } + parentP->add(nodeP); break; case orion::ValueTypeNotGiven: From 4a5d3e321e6b0de5d2c49c9c7921020805056459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 11:06:37 +0100 Subject: [PATCH 166/390] FIX JexlResult improvements --- src/lib/jexl/JexlManager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index bfb261f944..c0545b87a1 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -162,6 +162,7 @@ void JexlManager::init(void) JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) { JexlResult r; + r.valueType = orion::ValueTypeNull; LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); From 9827b46e685fb9aa468998cac42236dbb7e94ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 12:30:15 +0100 Subject: [PATCH 167/390] FIX cleanup --- src/lib/common/macroSubstitute.cpp | 28 ------- src/lib/common/macroSubstitute.h | 15 ---- src/lib/jexl/JexlManager.cpp | 119 ----------------------------- src/lib/jexl/JexlResult.cpp | 60 ++------------- 4 files changed, 7 insertions(+), 215 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 10c5ec0251..b9e6571ceb 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -84,34 +84,6 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon -/* **************************************************************************** -* -* buildReplacementMap - -* -*/ -/*void buildReplacementsMap -( - const Entity& en, - const std::string& service, - const std::string& token, - std::map* replacementsP -) -{ - replacementsP->insert(std::pair("id", "\"" + en.id + "\"")); - replacementsP->insert(std::pair("type", "\"" + en.type + "\"")); - replacementsP->insert(std::pair("service", "\"" + service + "\"")); - replacementsP->insert(std::pair("servicePath", "\"" + en.servicePath + "\"")); - replacementsP->insert(std::pair("authToken", "\"" + token + "\"")); - for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) - { - // Note that if some attribute is named service, servicePath or authToken (although it would be - // an anti-pattern), the attribute takes precedence - (*replacementsP)[en.attributeVector[ix]->name] = en.attributeVector[ix]->toJsonValue(); - } -}*/ - - - /* **************************************************************************** * * stringValueOrNothing - diff --git a/src/lib/common/macroSubstitute.h b/src/lib/common/macroSubstitute.h index cdb66ceaf7..a4ef0d89e3 100644 --- a/src/lib/common/macroSubstitute.h +++ b/src/lib/common/macroSubstitute.h @@ -38,21 +38,6 @@ extern std::string smartStringValue(const std::string stringValue, JexlContext* -/* **************************************************************************** -* -* buildReplacementMap - -* -*/ -/*extern void buildReplacementsMap -( - const Entity& en, - const std::string& service, - const std::string& token, - std::map* replacementsP -);*/ - - - /* **************************************************************************** * * macroSubstitute - diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/jexl/JexlManager.cpp index c0545b87a1..1d70241401 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/jexl/JexlManager.cpp @@ -69,36 +69,6 @@ static const char* capturePythonError() -#if 0 -/* **************************************************************************** -* -* customJsonSerializerFunction - -* -* To be used in the json.dumps() step in JexlManager::evaluate() in order to get -* integer numbers when possible (i.e. "2" instead of "2.0"), this way behaving as -* the pre-JEXL macro replacement logic -*/ -static PyObject* customJsonSerializerFunction(PyObject* obj) -{ - // Check if the object is a float - if (PyFloat_Check(obj)) - { - // Convert float to integer if it represents a whole number - double value = PyFloat_AsDouble(obj); - if (value == (int)value) - { - return PyLong_FromDouble(value); - } - } - - // Return the original object - Py_INCREF(obj); - return obj; -} -#endif - - - /* **************************************************************************** * * JexlManager::init - @@ -108,7 +78,6 @@ void JexlManager::init(void) pyjexlModule = NULL; jsonModule = NULL; jexlEngine = NULL; - //customJsonSerializer = NULL; if (sem_init(&sem, 0, 1) == -1) { @@ -134,16 +103,6 @@ void JexlManager::init(void) } LM_T(LmtJexl, ("json module has been loaded")); -#if 0 - customJsonSerializer = PyCapsule_New((void*)customJsonSerializerFunction, NULL, NULL); - if (customJsonSerializer == NULL) - { - const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error creating json custom serializer function: %s)", error)); - } - LM_T(LmtJexl, ("json custom serializer has been createdmodule has been loaded")); -#endif - jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); if (jexlEngine == NULL) { @@ -186,76 +145,6 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ r.fill(result); Py_XDECREF(result); - // Special case: expresion evalutes to None - /*if (result == Py_None) - { - LM_T(LmtJexl, ("JEXL evaluation result is null")); - r.valueType = orion::ValueTypeNull; - Py_XDECREF(result); - return r; - } - - // Other not null types - r.valueType = getPyObjectType(result); - if (r.valueType == orion::ValueTypeNumber) - { - r.numberValue = PyFloat_AsDouble(result); - LM_T(LmtJexl, ("JEXL evaluation result (double): %f", r.numberValue)); - Py_XDECREF(result); - } - else if (r.valueType == orion::ValueTypeBoolean) - { - r.boolValue = PyObject_IsTrue(result); - LM_T(LmtJexl, ("JEXL evaluation result (bool): %s", r.boolValue ? "true": "false")); - Py_XDECREF(result); - } - else if ((r.valueType == orion::ValueTypeObject) || (r.valueType == orion::ValueTypeVector)) - { - // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot - // be used, as it produce string with ' instead of " in the resulting JSON string. - // FIXME PR: is the customJsonSerializer going to be used at the end? - PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); - Py_XDECREF(result); - if (repr == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); - r.valueType = orion::ValueTypeNull; - } - else - { - const char* str = PyUnicode_AsUTF8(repr); - Py_XDECREF(repr); - if (str == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); - r.valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtJexl, ("JEXL evaluation result (object or vector): %s", str)); - r.stringValue = std::string(str); - } - } - } - else if (r.valueType == orion::ValueTypeString) - { - const char* str = PyUnicode_AsUTF8(result); - Py_XDECREF(result); - if (str == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); - r.valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtJexl, ("JEXL evaluation result (string): %s", str)); - r.stringValue = std::string(str); - } - }*/ - return r; } @@ -279,14 +168,6 @@ void JexlManager::release(void) LM_T(LmtJexl, ("pyjexl module has been freed")); } -#if 0 - if (customJsonSerializer != NULL) - { - Py_XDECREF(customJsonSerializer); - LM_T(LmtJexl, ("json custom serializer module has been freed")); - } -#endif - if (jsonModule != NULL) { Py_XDECREF(jsonModule); diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/jexl/JexlResult.cpp index 6a77126be7..19f4060f6f 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/jexl/JexlResult.cpp @@ -147,34 +147,9 @@ void JexlResult::fill(PyObject* result) Py_ssize_t pos = 0; while (PyDict_Next(result, &pos, &key, &value)) { + // FIXME PR: memory should be freed? processDictItem(compoundValueP, key, value); } - // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot - // be used, as it produce string with ' instead of " in the resulting JSON string. - // FIXME PR: is the customJsonSerializer going to be used at the end? - /*PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); - if (repr == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - const char* str = PyUnicode_AsUTF8(repr); - Py_XDECREF(repr); - if (str == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtJexl, ("JexlResult (object or vector): %s", str)); - stringValue = std::string(str); - } - }*/ } else if (valueType == orion::ValueTypeVector) { @@ -182,34 +157,9 @@ void JexlResult::fill(PyObject* result) Py_ssize_t size = PyList_Size(result); for (Py_ssize_t ix = 0; ix < size; ++ix) { - processListItem(compoundValueP, PyList_GetItem(result, ix)); + // FIXME PR: memory should be freed? + processListItem(compoundValueP, PyList_GetItem(result, ix)); } - // Using Python json.dumps() for this may seem overkill, but noe that PyObject_Repr(result) cannot - // be used, as it produce string with ' instead of " in the resulting JSON string. - // FIXME PR: is the customJsonSerializer going to be used at the end? - /*PyObject* repr = PyObject_CallMethod(jsonModule, "dumps", "O", result); - if (repr == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining dict/list representation: %s", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - const char* str = PyUnicode_AsUTF8(repr); - Py_XDECREF(repr); - if (str == NULL) - { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaining str representation (object or vector): %s", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtJexl, ("JexlResult (object or vector): %s", str)); - stringValue = std::string(str); - } - }*/ } else if (valueType == orion::ValueTypeString) { @@ -288,6 +238,7 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va for (Py_ssize_t ix = 0; ix < size; ++ix) { processListItem(nodeP, PyList_GetItem(value, ix)); + // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -298,6 +249,7 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va while (PyDict_Next(value, &pos, &keyAux, &valueAux)) { processDictItem(nodeP, keyAux, valueAux); + // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -378,6 +330,7 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke for (Py_ssize_t ix = 0; ix < size; ++ix) { processListItem(nodeP, PyList_GetItem(value, ix)); + // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -388,6 +341,7 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke while (PyDict_Next(value, &pos, &keyAux, &valueAux)) { processDictItem(nodeP, keyAux, valueAux); + // FIXME PR: memory should be freed? } parentP->add(nodeP); break; From 8e3bdd35acb67867417db8d9ad5ad2461abad162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 14:27:27 +0100 Subject: [PATCH 168/390] FIX rename jexl to expression in class names and so --- CMakeLists.txt | 4 +- src/app/contextBroker/contextBroker.cpp | 6 +- src/lib/apiTypesV2/Entity.cpp | 10 +- src/lib/apiTypesV2/Entity.h | 6 +- src/lib/common/macroSubstitute.cpp | 37 +-- src/lib/common/macroSubstitute.h | 4 +- src/lib/common/string.h | 21 ++ src/lib/{jexl => expressions}/CMakeLists.txt | 18 +- src/lib/expressions/ExprContext.cpp | 299 +++++++++++++++++ .../ExprContext.h} | 35 +- .../ExprManager.cpp} | 44 +-- .../ExprManager.h} | 16 +- .../ExprResult.cpp} | 66 ++-- .../JexlResult.h => expressions/ExprResult.h} | 10 +- .../jexlMgr.cpp => expressions/exprMgr.cpp} | 10 +- .../{jexl/jexlMgr.h => expressions/exprMgr.h} | 12 +- src/lib/jexl/JexlContext.cpp | 300 ------------------ src/lib/logMsg/traceLevels.h | 2 +- src/lib/ngsi/ContextAttribute.cpp | 22 +- src/lib/ngsi/ContextAttribute.h | 6 +- src/lib/ngsi/ContextElementResponse.cpp | 4 +- src/lib/ngsi/ContextElementResponse.h | 4 +- src/lib/ngsi/ContextElementResponseVector.cpp | 4 +- src/lib/ngsi/ContextElementResponseVector.h | 4 +- src/lib/ngsi10/NotifyContextRequest.cpp | 8 +- src/lib/ngsi10/NotifyContextRequest.h | 2 +- src/lib/ngsiNotify/Notifier.cpp | 84 ++--- src/lib/parse/CompoundValueNode.cpp | 48 +-- src/lib/parse/CompoundValueNode.h | 8 +- 29 files changed, 546 insertions(+), 548 deletions(-) rename src/lib/{jexl => expressions}/CMakeLists.txt (85%) create mode 100644 src/lib/expressions/ExprContext.cpp rename src/lib/{jexl/JexlContext.h => expressions/ExprContext.h} (71%) rename src/lib/{jexl/JexlManager.cpp => expressions/ExprManager.cpp} (76%) rename src/lib/{jexl/JexlManager.h => expressions/ExprManager.h} (80%) rename src/lib/{jexl/JexlResult.cpp => expressions/ExprResult.cpp} (81%) rename src/lib/{jexl/JexlResult.h => expressions/ExprResult.h} (92%) rename src/lib/{jexl/jexlMgr.cpp => expressions/exprMgr.cpp} (85%) rename src/lib/{jexl/jexlMgr.h => expressions/exprMgr.h} (80%) delete mode 100644 src/lib/jexl/JexlContext.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4cae22ff0b..03bfaab443 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,7 +204,7 @@ SET (ORION_LIBS common alarmMgr metricsMgr - jexl + expressions logSummary lm pa @@ -322,7 +322,7 @@ if (error EQUAL 0) ADD_SUBDIRECTORY(src/lib/cache) ADD_SUBDIRECTORY(src/lib/alarmMgr) ADD_SUBDIRECTORY(src/lib/metricsMgr) - ADD_SUBDIRECTORY(src/lib/jexl) + ADD_SUBDIRECTORY(src/lib/expressions) ADD_SUBDIRECTORY(src/lib/logSummary) ADD_SUBDIRECTORY(src/lib/mqtt) ADD_SUBDIRECTORY(src/app/contextBroker) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index b665a01555..ec994937fa 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -105,7 +105,7 @@ #include "alarmMgr/alarmMgr.h" #include "mqtt/mqttMgr.h" #include "metricsMgr/metricsMgr.h" -#include "jexl/jexlMgr.h" +#include "expressions/exprMgr.h" #include "logSummary/logSummary.h" #include "contextBroker/orionRestServices.h" @@ -598,7 +598,7 @@ void exitFunc(void) curl_context_cleanup(); curl_global_cleanup(); - jexlMgr.release(); + exprMgr.release(); #ifdef DEBUG // valgrind pass is done using DEBUG compilation, so we have to take care with @@ -1222,7 +1222,7 @@ int main(int argC, char* argV[]) SemOpType policy = policyGet(reqMutexPolicy); alarmMgr.init(relogAlarms); mqttMgr.init(mqttTimeout); - jexlMgr.init(); + exprMgr.init(); orionInit(orionExit, ORION_VERSION, policy, statCounters, statSemWait, statTiming, statNotifQueue, strictIdv1); mongoInit(dbURI, dbHost, rplSet, dbName, user, pwd, authMech, authDb, dbSSL, dbDisableRetryWrites, mtenant, dbTimeout, writeConcern, dbPoolSize, statSemWait); metricsMgr.init(!disableMetrics, statSemWait); diff --git a/src/lib/apiTypesV2/Entity.cpp b/src/lib/apiTypesV2/Entity.cpp index 5501c18e77..f864e0bcb2 100644 --- a/src/lib/apiTypesV2/Entity.cpp +++ b/src/lib/apiTypesV2/Entity.cpp @@ -41,7 +41,7 @@ #include "rest/OrionError.h" #include "apiTypesV2/Entity.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" @@ -308,7 +308,7 @@ std::string Entity::toJson bool blacklist, const std::vector& metadataFilter, bool renderNgsiField, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { std::vector orderedAttrs; @@ -327,7 +327,7 @@ std::string Entity::toJson out = toJsonKeyvalues(orderedAttrs); break; default: // NGSI_V2_NORMALIZED - out = toJsonNormalized(orderedAttrs, metadataFilter, renderNgsiField, jexlContextP); + out = toJsonNormalized(orderedAttrs, metadataFilter, renderNgsiField, exprContextObjectP); break; } @@ -444,7 +444,7 @@ std::string Entity::toJsonNormalized const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { JsonObjectHelper jh; @@ -476,7 +476,7 @@ std::string Entity::toJsonNormalized for (unsigned int ix = 0; ix < orderedAttrs.size(); ix++) { ContextAttribute* caP = orderedAttrs[ix]; - jh.addRaw(caP->name, caP->toJson(metadataFilter, renderNgsiField, jexlContextP)); + jh.addRaw(caP->name, caP->toJson(metadataFilter, renderNgsiField, exprContextObjectP)); } return jh.str(); diff --git a/src/lib/apiTypesV2/Entity.h b/src/lib/apiTypesV2/Entity.h index 1813faa0c1..f8acc3af97 100644 --- a/src/lib/apiTypesV2/Entity.h +++ b/src/lib/apiTypesV2/Entity.h @@ -32,7 +32,7 @@ #include "ngsi/ContextAttributeVector.h" #include "ngsi/EntityId.h" #include "rest/OrionError.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" @@ -92,7 +92,7 @@ class Entity bool blacklist, const std::vector& metadataFilter, bool renderNgsiField = false, - JexlContext* jexlContext = NULL); + ExprContextObject* exprContext = NULL); std::string toJson(RenderFormat renderFormat, bool renderNgsiField = false); @@ -146,7 +146,7 @@ class Entity std::string toJsonNormalized(const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField = false, - JexlContext* jexlContext = NULL); + ExprContextObject* exprContext = NULL); }; #endif // SRC_LIB_APITYPESV2_ENTITY_H_ diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index b9e6571ceb..3c0fc7e105 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -32,7 +32,7 @@ #include "common/JsonHelper.h" #include "common/macroSubstitute.h" -#include "jexl/jexlMgr.h" +#include "expressions/exprMgr.h" /* **************************************************************************** @@ -42,18 +42,18 @@ * Returns the effective string value, taking into account replacements * */ -std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault) +std::string smartStringValue(const std::string stringValue, ExprContextObject* exprContextObjectP, const std::string notFoundDefault) { // This code is pretty similar to the one in CompoundValueNode::toJson() // The program logic branching is the same, but the result at the end of each if-else // is different, which makes difficult to unify both them - if ((jexlContextP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) + if ((exprContextObjectP != NULL) && (stringValue.rfind("${") == 0) && (stringValue.rfind("}", stringValue.size()) == stringValue.size() - 1)) { // "Full replacement" case. In this case, the result is not always a string // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - JexlResult r = jexlMgr.evaluate(jexlContextP, macroName); + ExprResult r = exprMgr.evaluate(exprContextObjectP, macroName); if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; @@ -63,11 +63,11 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon return r.toString(); } } - else if (jexlContextP != NULL) + else if (exprContextObjectP != NULL) { // "Partial replacement" case. In this case, the result is always a string std::string effectiveValue; - if (!macroSubstitute(&effectiveValue, stringValue, jexlContextP, "null", true)) + if (!macroSubstitute(&effectiveValue, stringValue, exprContextObjectP, "null", true)) { // error already logged in macroSubstitute, using stringValue itself as failsafe effectiveValue = stringValue; @@ -89,10 +89,9 @@ std::string smartStringValue(const std::string stringValue, JexlContext* jexlCon * stringValueOrNothing - * */ -// FIXME PR: inTheMiddle -> raw ? -static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::string key, const std::string& notFoundDefault, bool inTheMiddle) +static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, const std::string key, const std::string& notFoundDefault, bool raw) { - JexlResult r = jexlMgr.evaluate(jexlContextP, key); + ExprResult r = exprMgr.evaluate(exprContextObjectP, key); if (r.valueType == orion::ValueTypeNull) { @@ -101,17 +100,16 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st else { std::string s = r.toString(); - if (inTheMiddle) + if (raw) { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), // so double quotes have to be be removed - if (s[0] == '"') - { - s = s.substr(1, s.size()-2); - } + return removeQuotes(s); + } + else + { + return s; } - - return s; } } @@ -142,8 +140,7 @@ static std::string stringValueOrNothing(JexlContext* jexlContextP, const std::st * Date: Mon Jun 19 16:33:29 2017 +0200 * */ -// FIXME PR: inTheMiddle -> raw ? -bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexlContextP, const std::string& notFoundDefault, bool inTheMiddle) +bool macroSubstitute(std::string* to, const std::string& from, ExprContextObject* exprContextObjectP, const std::string& notFoundDefault, bool raw) { // Initial size check: is the string to convert too big? // @@ -206,7 +203,7 @@ bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexl // The +3 is due to "${" and "}" toReduce += (macroName.length() + 3) * times; - toAdd += stringValueOrNothing(jexlContextP, macroName, notFoundDefault, inTheMiddle).length() * times; + toAdd += stringValueOrNothing(exprContextObjectP, macroName, notFoundDefault, raw).length() * times; } if (from.length() + toAdd - toReduce > outReqMsgMaxSize) @@ -224,7 +221,7 @@ bool macroSubstitute(std::string* to, const std::string& from, JexlContext* jexl unsigned int times = it->second; std::string macro = "${" + macroName + "}"; - std::string value = stringValueOrNothing(jexlContextP, macroName, notFoundDefault, inTheMiddle); + std::string value = stringValueOrNothing(exprContextObjectP, macroName, notFoundDefault, raw); // We have to do the replace operation as many times as macro occurrences for (unsigned int ix = 0; ix < times; ix++) diff --git a/src/lib/common/macroSubstitute.h b/src/lib/common/macroSubstitute.h index a4ef0d89e3..58f2aac2e5 100644 --- a/src/lib/common/macroSubstitute.h +++ b/src/lib/common/macroSubstitute.h @@ -34,7 +34,7 @@ * smartStringValue - * */ -extern std::string smartStringValue(const std::string stringValue, JexlContext* jexlContextP, const std::string notFoundDefault); +extern std::string smartStringValue(const std::string stringValue, ExprContextObject* exprContextObjectP, const std::string notFoundDefault); @@ -43,6 +43,6 @@ extern std::string smartStringValue(const std::string stringValue, JexlContext* * macroSubstitute - * */ -extern bool macroSubstitute(std::string* sP, const std::string& in, JexlContext* jexlContextP, const std::string& notFoundDefault, bool inTheMiddle = false); +extern bool macroSubstitute(std::string* sP, const std::string& in, ExprContextObject* exprContextObjectP, const std::string& notFoundDefault, bool raw = false); #endif // SRC_LIB_COMMON_MACROSUBSTITUTE_H_ diff --git a/src/lib/common/string.h b/src/lib/common/string.h index 886362d709..f3abcee416 100644 --- a/src/lib/common/string.h +++ b/src/lib/common/string.h @@ -216,4 +216,25 @@ extern std::string offuscatePassword(const std::string& uri, const std::string& */ extern bool regComp(regex_t* re, const char* pattern, int flags); + + +/* **************************************************************************** +* +* removeQuotes - +* +*/ +inline std::string removeQuotes(std::string s) +{ + if (s[0] == '"') + { + return s.substr(1, s.size()-2); + } + else + { + return s; + } +} + + + #endif // SRC_LIB_COMMON_STRING_H_ diff --git a/src/lib/jexl/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt similarity index 85% rename from src/lib/jexl/CMakeLists.txt rename to src/lib/expressions/CMakeLists.txt index a2ea7eb6e2..4f938e3634 100644 --- a/src/lib/jexl/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -21,17 +21,17 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES - JexlManager.cpp - JexlContext.cpp - JexlResult.cpp - jexlMgr.cpp + ExprManager.cpp + ExprContext.cpp + ExprResult.cpp + exprMgr.cpp ) SET (HEADERS - JexlManager.h - JexlContext.h - JexlResult.h - jexlMgr.h + ExprManager.h + ExprContext.h + ExprResult.h + exprMgr.h ) @@ -43,4 +43,4 @@ include_directories("${PROJECT_SOURCE_DIR}/src/lib") # Library declaration # ----------------------------------------------------------------- -ADD_LIBRARY(jexl STATIC ${SOURCES} ${HEADERS}) +ADD_LIBRARY(expressions STATIC ${SOURCES} ${HEADERS}) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp new file mode 100644 index 0000000000..8c05fd5764 --- /dev/null +++ b/src/lib/expressions/ExprContext.cpp @@ -0,0 +1,299 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +// FIXME PR: add methods should check error in Py_BuildValue() for NULL + +#include + +#include "logMsg/logMsg.h" +#include "expressions/ExprContext.h" + + + +/* **************************************************************************** +* +* ExprContextObject::ExprContextObject - +*/ +ExprContextObject::ExprContextObject +() +{ + jexlContext = PyDict_New(); + + if (jexlContext == NULL) + { + // FIXME PR: error control + // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + } +} + + + +/* **************************************************************************** +* +* ExprContextObject::get - +*/ +PyObject* ExprContextObject::get(void) +{ + return jexlContext; +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, const std::string& _value) +{ + LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); + PyObject* value = Py_BuildValue("s", _value.c_str()); + PyDict_SetItemString(jexlContext, key.c_str(), value); + Py_DECREF(value); +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, double _value) +{ + LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); + PyObject* value = Py_BuildValue("d", _value); + PyDict_SetItemString(jexlContext, key.c_str(), value); + Py_DECREF(value); +} + + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, bool _value) +{ + LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value? "true" : "false")); + if (_value) + { + PyDict_SetItemString(jexlContext, key.c_str(), Py_True); + } + else + { + PyDict_SetItemString(jexlContext, key.c_str(), Py_False); + } +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key) +{ + LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), Py_None); +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, ExprContextObject exprContextObject) +{ + // FIXME PR: implement a toString() method in ExprContextObject to be used here + //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.get()); +} + + + +/* **************************************************************************** +* +* ExprContextObject::add - +*/ +void ExprContextObject::add(const std::string& key, ExprContextList exprContextList) +{ + // FIXME PR: implement a toString() method in ExprContextList to be used here + //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); +} + + + +/* **************************************************************************** +* +* ExprContextObject::hasKey - +*/ +bool ExprContextObject::hasKey(const std::string& key) +{ + // Check if the key exists in the jexlContext dictionary + PyObject* keyObject = PyUnicode_FromString(key.c_str()); + int result = PyDict_Contains(jexlContext, keyObject); + Py_DECREF(keyObject); + + return result == 1; // Return true if key exists, false otherwise +} + + + +/* **************************************************************************** +* +* ExprContextObject::release - +*/ +void ExprContextObject::release(void) +{ + // FIXME PR: this is not correct. Recursively release of the dict object + //Py_XDECREF(jexlContext); +} + + +/* **************************************************************************** +* +* ExprContextList::ExprContextList - +*/ +ExprContextList::ExprContextList() +{ + jexlContext = PyList_New(0); + + if (jexlContext == NULL) + { + // FIXME PR: error control + // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + } +} + + + +/* **************************************************************************** +* +* ExprContextList::get - +*/ +PyObject* ExprContextList::get(void) +{ + return jexlContext; +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(const std::string& _value) +{ + LM_T(LmtExpr, ("adding to expression context list (string): %s=", _value.c_str())); + PyObject* value = Py_BuildValue("s", _value.c_str()); + PyList_Append(jexlContext, value); + Py_DECREF(value); +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(double _value) +{ + LM_T(LmtExpr, ("adding to expression context list (double): %f", _value)); + PyObject* value = Py_BuildValue("d", _value); + PyList_Append(jexlContext, value); + Py_DECREF(value); +} + + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(bool _value) +{ + LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value? "true" : "false")); + if (_value) + { + PyList_Append(jexlContext, Py_True); + } + else + { + PyList_Append(jexlContext, Py_False); + } +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(void) +{ + LM_T(LmtExpr, ("adding to expression context list (none)")); + PyList_Append(jexlContext, Py_None); +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(ExprContextObject exprContextObject) +{ + // FIXME PR: implement a toString() method in ExprContext to be used here + //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + PyList_Append(jexlContext, exprContextObject.get()); +} + + + +/* **************************************************************************** +* +* ExprContextList::add - +*/ +void ExprContextList::add(ExprContextList exprContextList) +{ + // FIXME PR: implement a toString() method in ExprContextList to be used here + //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + PyList_Append(jexlContext, exprContextList.get()); +} + + + +/* **************************************************************************** +* +* ExprContextList::relesase - +*/ +void ExprContextList::release(void) +{ + // FIXME PR: this is not correct. Recursively release of the list object + Py_XDECREF(jexlContext); +} \ No newline at end of file diff --git a/src/lib/jexl/JexlContext.h b/src/lib/expressions/ExprContext.h similarity index 71% rename from src/lib/jexl/JexlContext.h rename to src/lib/expressions/ExprContext.h index 4c91fc95a6..4db34f562c 100644 --- a/src/lib/jexl/JexlContext.h +++ b/src/lib/expressions/ExprContext.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_JEXLCONTEXT_H_ -#define SRC_LIB_JEXL_JEXLCONTEXT_H_ +#ifndef SRC_LIB_JEXL_EXPRCONTEXT_H_ +#define SRC_LIB_JEXL_EXPRCONTEXT_H_ /* * @@ -26,56 +26,51 @@ * Author: Fermin Galan */ -// FIXME PR: rename this to a better name (ExprContextDict?) - #include - #include -//#include "jexl/JexlContextList.h" - -class JexlContextList; // forward declaration +class ExprContextList; // forward declaration /* **************************************************************************** * -* JexlContext - +* ExprContext - */ -class JexlContext +class ExprContextObject { private: - PyObject* jexl_context; + PyObject* jexlContext; public: - JexlContext(); + ExprContextObject(); PyObject* get(void); void add(const std::string& key, const std::string& value); void add(const std::string& key, double value); void add(const std::string& key, bool value); void add(const std::string& key); - void add(const std::string& key, JexlContext jexlContext); - void add(const std::string& key, JexlContextList jexlContextList); + void add(const std::string& key, ExprContextObject exprContextObject); + void add(const std::string& key, ExprContextList exprContextList); bool hasKey(const std::string& key); void release(void); }; -class JexlContextList +class ExprContextList { private: - PyObject* jexl_context; + PyObject* jexlContext; public: - JexlContextList(); + ExprContextList(); PyObject* get(void); void add(const std::string& value); void add(double value); void add(bool value); void add(void); - void add(JexlContext jexlContext); - void add(JexlContextList jexlContextList); + void add(ExprContextObject exprContextObject); + void add(ExprContextList exprContextList); void release(void); }; -#endif // #define SRC_LIB_JEXL_JEXLCONTEXT_H_ +#endif // #define SRC_LIB_JEXL_EXPRCONTEXT_H_ diff --git a/src/lib/jexl/JexlManager.cpp b/src/lib/expressions/ExprManager.cpp similarity index 76% rename from src/lib/jexl/JexlManager.cpp rename to src/lib/expressions/ExprManager.cpp index 1d70241401..10739d6a1d 100644 --- a/src/lib/jexl/JexlManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -25,8 +25,8 @@ #include -#include "jexl/JexlManager.h" -#include "jexl/JexlResult.h" +#include "expressions/ExprManager.h" +#include "expressions/ExprResult.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" @@ -71,9 +71,9 @@ static const char* capturePythonError() /* **************************************************************************** * -* JexlManager::init - +* ExprManager::init - */ -void JexlManager::init(void) +void ExprManager::init(void) { pyjexlModule = NULL; jsonModule = NULL; @@ -85,7 +85,7 @@ void JexlManager::init(void) } Py_Initialize(); - LM_T(LmtJexl, ("Python interpreter has been initialized")); + LM_T(LmtExpr, ("Python interpreter has been initialized")); pyjexlModule = PyImport_ImportModule("pyjexl"); if (pyjexlModule == NULL) @@ -93,7 +93,7 @@ void JexlManager::init(void) const char* error = capturePythonError(); LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", error)); } - LM_T(LmtJexl, ("pyjexl module has been loaded")); + LM_T(LmtExpr, ("pyjexl module has been loaded")); jsonModule = PyImport_ImportModule("json"); if (jsonModule == NULL) @@ -101,7 +101,7 @@ void JexlManager::init(void) const char* error = capturePythonError(); LM_X(1, ("Fatal Error (error importing json module: %s)", error)); } - LM_T(LmtJexl, ("json module has been loaded")); + LM_T(LmtExpr, ("json module has been loaded")); jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); if (jexlEngine == NULL) @@ -109,40 +109,42 @@ void JexlManager::init(void) const char* error = capturePythonError(); LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", error)); } - LM_T(LmtJexl, ("jexl engine has been created")); + LM_T(LmtExpr, ("jexl engine has been created")); } /* **************************************************************************** * -* JexlManager::evaluate - +* ExprManager::evaluate - */ -JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _expression) +ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) { - JexlResult r; + ExprResult r; r.valueType = orion::ValueTypeNull; - LM_T(LmtJexl, ("evaluating JEXL expresion: <%s>", _expression.c_str())); + LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error building expression: %s", capturePythonError())); + LM_T(LmtExpr, ("error building expression: %s", capturePythonError())); return r; } - PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, jexlContextP->get()); + PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, exprContextObjectP->get()); Py_XDECREF(expression); if (result == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error evaluating expression: %s", capturePythonError())); + LM_T(LmtExpr, ("error evaluating expression: %s", capturePythonError())); return r; } r.fill(result); + + // FIXME PR: does this Py_XDECREF() recursively in the case of dicts or lists? Py_XDECREF(result); return r; @@ -152,28 +154,28 @@ JexlResult JexlManager::evaluate(JexlContext* jexlContextP, const std::string& _ /* **************************************************************************** * -* JexlManager::release - +* ExprManager::release - */ -void JexlManager::release(void) +void ExprManager::release(void) { if (jexlEngine != NULL) { Py_XDECREF(jexlEngine); - LM_T(LmtJexl, ("jexl engine has been freed")); + LM_T(LmtExpr, ("jexl engine has been freed")); } if (pyjexlModule != NULL) { Py_XDECREF(pyjexlModule); - LM_T(LmtJexl, ("pyjexl module has been freed")); + LM_T(LmtExpr, ("pyjexl module has been freed")); } if (jsonModule != NULL) { Py_XDECREF(jsonModule); - LM_T(LmtJexl, ("json module has been freed")); + LM_T(LmtExpr, ("json module has been freed")); } Py_Finalize(); - LM_T(LmtJexl, ("Python interpreter has been finalized")); + LM_T(LmtExpr, ("Python interpreter has been finalized")); } \ No newline at end of file diff --git a/src/lib/jexl/JexlManager.h b/src/lib/expressions/ExprManager.h similarity index 80% rename from src/lib/jexl/JexlManager.h rename to src/lib/expressions/ExprManager.h index adce908c7c..7466372b29 100644 --- a/src/lib/jexl/JexlManager.h +++ b/src/lib/expressions/ExprManager.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_JEXLMANAGER_H_ -#define SRC_LIB_JEXL_JEXLMANAGER_H_ +#ifndef SRC_LIB_JEXL_EXPRMANAGER_H_ +#define SRC_LIB_JEXL_EXPRMANAGER_H_ /* * @@ -29,14 +29,14 @@ #include #include -#include "jexl/JexlContext.h" -#include "jexl/JexlResult.h" +#include "expressions/ExprContext.h" +#include "expressions/ExprResult.h" /* **************************************************************************** * -* JexlManager - +* ExprManager - */ -class JexlManager +class ExprManager { private: PyObject* pyjexlModule; @@ -47,8 +47,8 @@ class JexlManager public: void init(void); - JexlResult evaluate(JexlContext* jexlContextP, const std::string& expression); + ExprResult evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); void release(void); }; -#endif // SRC_LIB_JEXL_JEXLMANAGER_H_ \ No newline at end of file +#endif // SRC_LIB_JEXL_EXPRMANAGER_H_ \ No newline at end of file diff --git a/src/lib/jexl/JexlResult.cpp b/src/lib/expressions/ExprResult.cpp similarity index 81% rename from src/lib/jexl/JexlResult.cpp rename to src/lib/expressions/ExprResult.cpp index 19f4060f6f..036651fdc4 100644 --- a/src/lib/jexl/JexlResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -23,7 +23,7 @@ * Author: Fermin Galan */ -#include "jexl/JexlResult.h" +#include "expressions/ExprResult.h" #include "common/string.h" #include "common/JsonHelper.h" @@ -115,7 +115,7 @@ static orion::ValueType getPyObjectType(PyObject* obj) * fill - * */ -void JexlResult::fill(PyObject* result) +void ExprResult::fill(PyObject* result) { // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; @@ -123,7 +123,7 @@ void JexlResult::fill(PyObject* result) // Special case: expresion evalutes to None if (result == Py_None) { - LM_T(LmtJexl, ("JexlResult is null")); + LM_T(LmtExpr, ("ExprResult is null")); valueType = orion::ValueTypeNull; return; } @@ -133,12 +133,12 @@ void JexlResult::fill(PyObject* result) if (valueType == orion::ValueTypeNumber) { numberValue = PyFloat_AsDouble(result); - LM_T(LmtJexl, ("JexlResult (double): %f", numberValue)); + LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); } else if (valueType == orion::ValueTypeBoolean) { boolValue = PyObject_IsTrue(result); - LM_T(LmtJexl, ("JexlResult (bool): %s", boolValue ? "true": "false")); + LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); } else if (valueType == orion::ValueTypeObject) { @@ -147,7 +147,8 @@ void JexlResult::fill(PyObject* result) Py_ssize_t pos = 0; while (PyDict_Next(result, &pos, &key, &value)) { - // FIXME PR: memory should be freed? + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() processDictItem(compoundValueP, key, value); } } @@ -157,7 +158,8 @@ void JexlResult::fill(PyObject* result) Py_ssize_t size = PyList_Size(result); for (Py_ssize_t ix = 0; ix < size; ++ix) { - // FIXME PR: memory should be freed? + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() processListItem(compoundValueP, PyList_GetItem(result, ix)); } } @@ -167,12 +169,12 @@ void JexlResult::fill(PyObject* result) if (str == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); valueType = orion::ValueTypeNull; } else { - LM_T(LmtJexl, ("JexlResult (string): %s", str)); + LM_T(LmtExpr, ("ExprResult (string): %s", str)); stringValue = std::string(str); } } @@ -186,7 +188,7 @@ void JexlResult::fill(PyObject* result) * * FIXME PR: maybe this should be static function out of the class? */ -void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* value) +void ExprResult::processListItem(orion::CompoundValueNode* parentP, PyObject* value) { orion::CompoundValueNode* nodeP; @@ -203,11 +205,11 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va if (str == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); } else { - LM_T(LmtJexl, ("processListITem (string): %s", str)); + LM_T(LmtExpr, ("processListITem (string): %s", str)); nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); parentP->add(nodeP); } @@ -215,14 +217,14 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va case orion::ValueTypeNumber: d = PyFloat_AsDouble(value); - LM_T(LmtJexl, ("processList (double): %f", d)); + LM_T(LmtExpr, ("processList (double): %f", d)); nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); parentP->add(nodeP); break; case orion::ValueTypeBoolean: b = PyObject_IsTrue(value); - LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); + LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); parentP->add(nodeP); break; @@ -237,8 +239,9 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va size = PyList_Size(value); for (Py_ssize_t ix = 0; ix < size; ++ix) { - processListItem(nodeP, PyList_GetItem(value, ix)); - // FIXME PR: memory should be freed? + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(nodeP, PyList_GetItem(value, ix)); } parentP->add(nodeP); break; @@ -248,8 +251,9 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va pos = 0; while (PyDict_Next(value, &pos, &keyAux, &valueAux)) { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() processDictItem(nodeP, keyAux, valueAux); - // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -271,13 +275,13 @@ void JexlResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va * * FIXME PR: maybe this should be static function out of the class? */ -void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) { const char * keyStr = PyUnicode_AsUTF8(key); if (keyStr == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); return; } @@ -295,11 +299,11 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke if (str == NULL) { // FIXME PR: use LM_E/LM_W? - LM_T(LmtJexl, ("error obtaning str representation (string): %s", capturePythonError())); + LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); } else { - LM_T(LmtJexl, ("processListITem (string): %s", str)); + LM_T(LmtExpr, ("processListITem (string): %s", str)); nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); parentP->add(nodeP); } @@ -307,14 +311,14 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke case orion::ValueTypeNumber: d = PyFloat_AsDouble(value); - LM_T(LmtJexl, ("processList (double): %f", d)); + LM_T(LmtExpr, ("processList (double): %f", d)); nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); parentP->add(nodeP); break; case orion::ValueTypeBoolean: b = PyObject_IsTrue(value); - LM_T(LmtJexl, ("JexlResult (bool): %s", b ? "true": "false")); + LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); parentP->add(nodeP); break; @@ -329,8 +333,9 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke size = PyList_Size(value); for (Py_ssize_t ix = 0; ix < size; ++ix) { - processListItem(nodeP, PyList_GetItem(value, ix)); - // FIXME PR: memory should be freed? + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(nodeP, PyList_GetItem(value, ix)); } parentP->add(nodeP); break; @@ -340,8 +345,9 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke pos = 0; while (PyDict_Next(value, &pos, &keyAux, &valueAux)) { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() processDictItem(nodeP, keyAux, valueAux); - // FIXME PR: memory should be freed? } parentP->add(nodeP); break; @@ -364,7 +370,7 @@ void JexlResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke * Pretty similar to ContextAttribute::toJsonValue() * */ -std::string JexlResult::toString(void) +std::string ExprResult::toString(void) { if (valueType == orion::ValueTypeNumber) { @@ -396,7 +402,7 @@ std::string JexlResult::toString(void) } else { - LM_E(("Runtime Error (not allowed type in JexlResult: %s)", valueTypeName(valueType))); + LM_E(("Runtime Error (not allowed type in ExprResult: %s)", valueTypeName(valueType))); return ""; } } @@ -405,9 +411,9 @@ std::string JexlResult::toString(void) /* **************************************************************************** * -* JexlResult::release - +* ExprResult::release - */ -void JexlResult::release(void) +void ExprResult::release(void) { if (compoundValueP != NULL) { diff --git a/src/lib/jexl/JexlResult.h b/src/lib/expressions/ExprResult.h similarity index 92% rename from src/lib/jexl/JexlResult.h rename to src/lib/expressions/ExprResult.h index cdf853c24a..bfa4bcbbb7 100644 --- a/src/lib/jexl/JexlResult.h +++ b/src/lib/expressions/ExprResult.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_JEXLRESULT_H_ -#define SRC_LIB_JEXL_JEXLRESULT_H_ +#ifndef SRC_LIB_JEXL_EXPRRESULT_H_ +#define SRC_LIB_JEXL_EXPRRESULT_H_ /* * @@ -35,9 +35,9 @@ /* **************************************************************************** * -* JexlResult - +* ExprResult - */ -class JexlResult +class ExprResult { public: // Similar to the fields used in ContextAttribute.h @@ -61,4 +61,4 @@ class JexlResult -#endif // #define SRC_LIB_JEXL_JEXLRESULT_H_ +#endif // #define SRC_LIB_JEXL_EXPRRESULT_H_ diff --git a/src/lib/jexl/jexlMgr.cpp b/src/lib/expressions/exprMgr.cpp similarity index 85% rename from src/lib/jexl/jexlMgr.cpp rename to src/lib/expressions/exprMgr.cpp index a5e8e2cd85..f56d51706b 100644 --- a/src/lib/jexl/jexlMgr.cpp +++ b/src/lib/expressions/exprMgr.cpp @@ -1,6 +1,6 @@ /* * -* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U * * This file is part of Orion Context Broker. * @@ -20,14 +20,14 @@ * For those usages not covered by this license please contact with * iot_support at tid dot es * -* Author: Ken Zangelin +* Author: Fermin Galan */ -#include "jexl/JexlManager.h" +#include "expressions/ExprManager.h" /* **************************************************************************** * -* jexlMgr - +* exprMgr - */ -JexlManager jexlMgr; +ExprManager exprMgr; diff --git a/src/lib/jexl/jexlMgr.h b/src/lib/expressions/exprMgr.h similarity index 80% rename from src/lib/jexl/jexlMgr.h rename to src/lib/expressions/exprMgr.h index 75b22a66a8..852e396e24 100644 --- a/src/lib/jexl/jexlMgr.h +++ b/src/lib/expressions/exprMgr.h @@ -1,9 +1,9 @@ -#ifndef SRC_LIB_JEXL_JEXLMGR_H_ -#define SRC_LIB_JEXL_JEXLMGR_H_ +#ifndef SRC_LIB_JEXL_EXPRMGR_H_ +#define SRC_LIB_JEXL_EXPRMGR_H_ /* * -* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U * * This file is part of Orion Context Broker. * @@ -25,7 +25,7 @@ * * Author: Fermin Galan */ -#include "jexl/JexlManager.h" +#include "expressions/ExprManager.h" @@ -33,6 +33,6 @@ * * metricsMgr - */ -extern JexlManager jexlMgr; +extern ExprManager exprMgr; -#endif // SRC_LIB_JEXL_JEXLMGR_H_ +#endif // SRC_LIB_JEXL_EXPRMGR_H_ diff --git a/src/lib/jexl/JexlContext.cpp b/src/lib/jexl/JexlContext.cpp deleted file mode 100644 index 82ab5e3dd4..0000000000 --- a/src/lib/jexl/JexlContext.cpp +++ /dev/null @@ -1,300 +0,0 @@ -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -// FIXME PR: use better names than JexlContext and JexlContextList. Also in LM_T -// FIXME PR: add methods should check error in Py_BuildValue() for NULL - -#include - -#include "logMsg/logMsg.h" -#include "jexl/JexlContext.h" - - - -/* **************************************************************************** -* -* JexlContext::JexlContext - -*/ -JexlContext::JexlContext -() -{ - jexl_context = PyDict_New(); - - if (jexl_context == NULL) - { - // FIXME PR: error control - // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) - } -} - - - -/* **************************************************************************** -* -* JexlContext::get - -*/ -PyObject* JexlContext::get(void) -{ - return jexl_context; -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, const std::string& _value) -{ - LM_T(LmtJexl, ("adding to JEXL context (string): %s=%s", key.c_str(), _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - PyDict_SetItemString(jexl_context, key.c_str(), value); - Py_DECREF(value); -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, double _value) -{ - LM_T(LmtJexl, ("adding to JEXL context (double): %s=%f", key.c_str(), _value)); - PyObject* value = Py_BuildValue("d", _value); - PyDict_SetItemString(jexl_context, key.c_str(), value); - Py_DECREF(value); -} - - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, bool _value) -{ - LM_T(LmtJexl, ("adding to JEXL context (bool): %s=%s", key.c_str(), _value? "true" : "false")); - if (_value) - { - PyDict_SetItemString(jexl_context, key.c_str(), Py_True); - } - else - { - PyDict_SetItemString(jexl_context, key.c_str(), Py_False); - } -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key) -{ - LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyDict_SetItemString(jexl_context, key.c_str(), Py_None); -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, JexlContext _jexlContext) -{ - // FIXME PR: implement a toString() method in JexlContext to be used here - //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyDict_SetItemString(jexl_context, key.c_str(), _jexlContext.get()); -} - - - -/* **************************************************************************** -* -* JexlContext::add - -*/ -void JexlContext::add(const std::string& key, JexlContextList jexlContextList) -{ - // FIXME PR: implement a toString() method in JexlContextList to be used here - //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyDict_SetItemString(jexl_context, key.c_str(), jexlContextList.get()); -} - - - -/* **************************************************************************** -* -* JexlContext::hasKey - -*/ -bool JexlContext::hasKey(const std::string& key) -{ - // Check if the key exists in the jexl_context dictionary - PyObject* keyObject = PyUnicode_FromString(key.c_str()); - int result = PyDict_Contains(jexl_context, keyObject); - Py_DECREF(keyObject); - - return result == 1; // Return true if key exists, false otherwise -} - - - -/* **************************************************************************** -* -* JexlContext::release - -*/ -void JexlContext::release(void) -{ - // FIXME PR: this is not correct. Recursively release of the dict object - //Py_XDECREF(jexl_context); -} - - -/* **************************************************************************** -* -* JexlContextList::JexlContextList - -*/ -JexlContextList::JexlContextList() -{ - jexl_context = PyList_New(0); - - if (jexl_context == NULL) - { - // FIXME PR: error control - // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) - } -} - - - -/* **************************************************************************** -* -* JexlContextList::get - -*/ -PyObject* JexlContextList::get(void) -{ - return jexl_context; -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(const std::string& _value) -{ - LM_T(LmtJexl, ("adding to JEXL context list (string): %s=", _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - PyList_Append(jexl_context, value); - Py_DECREF(value); -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(double _value) -{ - LM_T(LmtJexl, ("adding to JEXL context list (double): %f", _value)); - PyObject* value = Py_BuildValue("d", _value); - PyList_Append(jexl_context, value); - Py_DECREF(value); -} - - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(bool _value) -{ - LM_T(LmtJexl, ("adding to JEXL context list (bool): %s", _value? "true" : "false")); - if (_value) - { - PyList_Append(jexl_context, Py_True); - } - else - { - PyList_Append(jexl_context, Py_False); - } -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(void) -{ - LM_T(LmtJexl, ("adding to JEXL context list (none)")); - PyList_Append(jexl_context, Py_None); -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(JexlContext _jexlContextP) -{ - // FIXME PR: implement a toString() method in JexlContext to be used here - //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyList_Append(jexl_context, _jexlContextP.get()); -} - - - -/* **************************************************************************** -* -* JexlContextList::add - -*/ -void JexlContextList::add(JexlContextList _jexlContextList) -{ - // FIXME PR: implement a toString() method in JexlContextList to be used here - //LM_T(LmtJexl, ("adding to JEXL context (none): %s", key.c_str())); - PyList_Append(jexl_context, _jexlContextList.get()); -} - - - -/* **************************************************************************** -* -* JexlContextList::relesase - -*/ -void JexlContextList::release(void) -{ - // FIXME PR: this is not correct. Recursively release of the list object - Py_XDECREF(jexl_context); -} \ No newline at end of file diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index 8a5ca6a412..84a40bc6f8 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -131,7 +131,7 @@ typedef enum TraceLevels LmtNotImplemented, LmtCurlContext, LmtThreadpool, - LmtJexl, + LmtExpr, LmtOldInfo = 240, // old INFO traces moved to DEBUG in Orion 2.5.0 diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 725e159ed9..4bf143ba76 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1052,7 +1052,7 @@ void ContextAttribute::filterAndOrderMetadata * renderNgsiField true is used in custom notification payloads, which have some small differences * with regards to conventional rendering */ -std::string ContextAttribute::toJson(const std::vector& metadataFilter, bool renderNgsiField, JexlContext* jexlContextP) +std::string ContextAttribute::toJson(const std::vector& metadataFilter, bool renderNgsiField, ExprContextObject* exprContextObjectP) { JsonObjectHelper jh; @@ -1085,7 +1085,7 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi // of DB entities) may lead to NULL, so the check is needed if (childToRenderP != NULL) { - jh.addRaw("value", childToRenderP->toJson(jexlContextP)); + jh.addRaw("value", childToRenderP->toJson(exprContextObjectP)); } } else if (valueType == orion::ValueTypeNumber) @@ -1101,7 +1101,7 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi } else if (valueType == orion::ValueTypeString) { - jh.addRaw("value", smartStringValue(stringValue, jexlContextP, "null")); + jh.addRaw("value", smartStringValue(stringValue, exprContextObjectP, "null")); } else if (valueType == orion::ValueTypeBoolean) { @@ -1296,41 +1296,41 @@ std::string ContextAttribute::toJsonAsValue * * Pretty similar in structure to toJsonValue */ -void ContextAttribute::addToContext(JexlContext* jexlContextP) +void ContextAttribute::addToContext(ExprContextObject* exprContextObjectP) { if (compoundValueP != NULL) { if (valueType == orion::ValueTypeObject) { - jexlContextP->add(name, compoundValueP->toJexlContext()); + exprContextObjectP->add(name, compoundValueP->toExprContextObject()); } else // valueType == orion::ValueTypeVector { - jexlContextP->add(name, compoundValueP->toJexlContextList()); + exprContextObjectP->add(name, compoundValueP->toExprContextList()); } } else if (valueType == orion::ValueTypeNumber) { if ((type == DATE_TYPE) || (type == DATE_TYPE_ALT)) { - jexlContextP->add(name, toJsonString(isodate2str(numberValue))); + exprContextObjectP->add(name, toJsonString(isodate2str(numberValue))); } else // regular number { - jexlContextP->add(name, numberValue); + exprContextObjectP->add(name, numberValue); } } else if (valueType == orion::ValueTypeString) { - jexlContextP->add(name, toJsonString(stringValue)); + exprContextObjectP->add(name, toJsonString(stringValue)); } else if (valueType == orion::ValueTypeBoolean) { - jexlContextP->add(name, boolValue); + exprContextObjectP->add(name, boolValue); } else if (valueType == orion::ValueTypeNull) { - jexlContextP->add(name); + exprContextObjectP->add(name); } else if (valueType == orion::ValueTypeNotGiven) { diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index 6b65827d29..23b74c06f1 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -37,7 +37,7 @@ #include "parse/CompoundValueNode.h" #include "rest/HttpStatusCode.h" #include "mongoDriver/BSONObjBuilder.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" @@ -108,7 +108,7 @@ typedef struct ContextAttribute std::string toJsonV1AsNameString(bool comma); - std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, JexlContext* jexlContextP = NULL); + std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, ExprContextObject* exprContextObjectP = NULL); std::string toJsonValue(void); @@ -119,7 +119,7 @@ typedef struct ContextAttribute MimeType* outMimeTypeP, HttpStatusCode* scP); - void addToContext(JexlContext* jexlContextP); + void addToContext(ExprContextObject* exprContextObjectP); void release(void); std::string getName(void); diff --git a/src/lib/ngsi/ContextElementResponse.cpp b/src/lib/ngsi/ContextElementResponse.cpp index 262d0f2a4c..8f5dace5ca 100644 --- a/src/lib/ngsi/ContextElementResponse.cpp +++ b/src/lib/ngsi/ContextElementResponse.cpp @@ -215,12 +215,12 @@ std::string ContextElementResponse::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { std::string out; - out = entity.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, false, jexlContextP); + out = entity.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, false, exprContextObjectP); return out; } diff --git a/src/lib/ngsi/ContextElementResponse.h b/src/lib/ngsi/ContextElementResponse.h index 471a8e66e6..2394183081 100644 --- a/src/lib/ngsi/ContextElementResponse.h +++ b/src/lib/ngsi/ContextElementResponse.h @@ -33,7 +33,7 @@ #include "ngsi/StringList.h" #include "ngsi/ContextAttribute.h" #include "apiTypesV2/Entity.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" #include "mongoDriver/BSONObj.h" @@ -83,7 +83,7 @@ typedef struct ContextElementResponse const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContext); + ExprContextObject* exprContextObjectP); void applyUpdateOperators(void); diff --git a/src/lib/ngsi/ContextElementResponseVector.cpp b/src/lib/ngsi/ContextElementResponseVector.cpp index 03fd35741a..24f40121f4 100644 --- a/src/lib/ngsi/ContextElementResponseVector.cpp +++ b/src/lib/ngsi/ContextElementResponseVector.cpp @@ -119,14 +119,14 @@ std::string ContextElementResponseVector::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { JsonVectorHelper jvh; for (unsigned int ix = 0; ix < vec.size(); ++ix) { - jvh.addRaw(vec[ix]->toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP)); + jvh.addRaw(vec[ix]->toJson(renderFormat, attrsFilter, blacklist, metadataFilter, exprContextObjectP)); } return jvh.str(); diff --git a/src/lib/ngsi/ContextElementResponseVector.h b/src/lib/ngsi/ContextElementResponseVector.h index 16cb7afb7c..9a6102b990 100644 --- a/src/lib/ngsi/ContextElementResponseVector.h +++ b/src/lib/ngsi/ContextElementResponseVector.h @@ -31,7 +31,7 @@ #include "ngsi/ContextElementResponse.h" #include "apiTypesV2/EntityVector.h" #include "common/RenderFormat.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" @@ -61,7 +61,7 @@ typedef struct ContextElementResponseVector const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP); + ExprContextObject* exprContextObjectP); void push_back(ContextElementResponse* item); unsigned int size(void) const; ContextElementResponse* lookup(Entity* eP, HttpStatusCode code = SccNone); diff --git a/src/lib/ngsi10/NotifyContextRequest.cpp b/src/lib/ngsi10/NotifyContextRequest.cpp index 30122bfcb0..f1ae7167d9 100644 --- a/src/lib/ngsi10/NotifyContextRequest.cpp +++ b/src/lib/ngsi10/NotifyContextRequest.cpp @@ -80,7 +80,7 @@ std::string NotifyContextRequest::toJson const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP + ExprContextObject* exprContextObjectP ) { if ((renderFormat != NGSI_V2_NORMALIZED) && (renderFormat != NGSI_V2_KEYVALUES) && (renderFormat != NGSI_V2_VALUES) && (renderFormat != NGSI_V2_SIMPLIFIEDKEYVALUES) && (renderFormat != NGSI_V2_SIMPLIFIEDNORMALIZED)) @@ -100,7 +100,7 @@ std::string NotifyContextRequest::toJson else { std::string out; - out += contextElementResponseVector[0]->toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, jexlContextP); + out += contextElementResponseVector[0]->toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, exprContextObjectP); return out; } } @@ -114,7 +114,7 @@ std::string NotifyContextRequest::toJson else { std::string out; - out += contextElementResponseVector[0]->toJson(NGSI_V2_KEYVALUES, attrsFilter, blacklist, metadataFilter, jexlContextP); + out += contextElementResponseVector[0]->toJson(NGSI_V2_KEYVALUES, attrsFilter, blacklist, metadataFilter, exprContextObjectP); return out; } } @@ -123,7 +123,7 @@ std::string NotifyContextRequest::toJson JsonObjectHelper jh; jh.addString("subscriptionId", subscriptionId.get()); - jh.addRaw("data", contextElementResponseVector.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP)); + jh.addRaw("data", contextElementResponseVector.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, exprContextObjectP)); return jh.str(); } } diff --git a/src/lib/ngsi10/NotifyContextRequest.h b/src/lib/ngsi10/NotifyContextRequest.h index 80e4a19c8d..c0d179a1f3 100644 --- a/src/lib/ngsi10/NotifyContextRequest.h +++ b/src/lib/ngsi10/NotifyContextRequest.h @@ -53,7 +53,7 @@ typedef struct NotifyContextRequest const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - JexlContext* jexlContextP = NULL); + ExprContextObject* exprContextObjectP = NULL); std::string check(ApiVersion apiVersion, const std::string& predetectedError); void release(void); NotifyContextRequest* clone(void); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 4ab2b5262f..aedcaa1b21 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -121,7 +121,7 @@ static bool setPayload const std::string& notifPayload, const SubscriptionId& subscriptionId, Entity& en, - JexlContext* jexlContextP, + ExprContextObject* exprContextObjectP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -170,7 +170,7 @@ static bool setPayload } else { - if (!macroSubstitute(payloadP, notifPayload, jexlContextP, "", true)) + if (!macroSubstitute(payloadP, notifPayload, exprContextObjectP, "", true)) { return false; } @@ -194,39 +194,17 @@ static bool setPayload static bool setJsonPayload ( orion::CompoundValueNode* json, - JexlContext* jexlContextP, + ExprContextObject* exprContextObjectP, std::string* payloadP, std::string* mimeTypeP ) { - *payloadP = json->toJson(jexlContextP); + *payloadP = json->toJson(exprContextObjectP); *mimeTypeP = "application/json"; // this can be overriden by headers field return true; } -/* **************************************************************************** -* -* removeQuotes - -* -* Entity id and type are special. Different from a attribute, they are always -* strings and cannot take a number, boolean, etc. as value. -* -* FIXME PR: duplicated logic! (both " and ' in different points of the code) -*/ -inline std::string removeQuotes(std::string s) -{ - if (s[0] == '"') - { - return s.substr(1, s.size()-2); - } - else - { - return s; - } -} - - /* **************************************************************************** * @@ -239,7 +217,7 @@ static bool setNgsiPayload const Entity& ngsi, const SubscriptionId& subscriptionId, Entity& en, - JexlContext* jexlContextP, + ExprContextObject* exprContextObjectP, const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, @@ -258,7 +236,7 @@ static bool setNgsiPayload else { // If id is not found in the replacements macro, we use en.id. - effectiveId = removeQuotes(smartStringValue(ngsi.id, jexlContextP, '"' + en.id + '"')); + effectiveId = removeQuotes(smartStringValue(ngsi.id, exprContextObjectP, '"' + en.id + '"')); } std::string effectiveType; @@ -269,7 +247,7 @@ static bool setNgsiPayload else { // If type is not found in the replacements macro, we use en.type. - effectiveType = removeQuotes(smartStringValue(ngsi.type, jexlContextP, '"' + en.type + '"')); + effectiveType = removeQuotes(smartStringValue(ngsi.type, exprContextObjectP, '"' + en.type + '"')); } cer.entity.fill(effectiveId, effectiveType, en.isPattern, en.servicePath); @@ -295,11 +273,11 @@ static bool setNgsiPayload if ((renderFormat == NGSI_V2_SIMPLIFIEDNORMALIZED) || (renderFormat == NGSI_V2_SIMPLIFIEDKEYVALUES)) { - *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, jexlContextP); + *payloadP = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter, exprContextObjectP); } else { - *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, jexlContextP); + *payloadP = ncr.toJson(NGSI_V2_NORMALIZED, attrsFilter, blacklist, metadataFilter, exprContextObjectP); } return true; @@ -338,15 +316,15 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - JexlContext jexlContext; - jexlContext.add("id", en.id); - jexlContext.add("type", en.type); - jexlContext.add("service", tenant); - jexlContext.add("servicePath", en.servicePath); - jexlContext.add("authToken", xauthToken); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add("service", tenant); + exprContext.add("servicePath", en.servicePath); + exprContext.add("authToken", xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - en.attributeVector[ix]->addToContext(&jexlContext); + en.attributeVector[ix]->addToContext(&exprContext); } // @@ -373,10 +351,10 @@ static SenderThreadParams* buildSenderParamsCustom // 2. URL // std::string notifUrl = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.url : notification.mqttInfo.url); - if (macroSubstitute(&url, notifUrl, &jexlContext, "", true) == false) + if (macroSubstitute(&url, notifUrl, &exprContext, "", true) == false) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } @@ -390,27 +368,27 @@ static SenderThreadParams* buildSenderParamsCustom { bool includePayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.includePayload : notification.mqttInfo.includePayload); std::string notifPayload = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.payload : notification.mqttInfo.payload); - if (!setPayload(includePayload, notifPayload, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) + if (!setPayload(includePayload, notifPayload, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } } else if (customPayloadType == ngsiv2::CustomPayloadType::Json) { orion::CompoundValueNode* json = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.json : notification.mqttInfo.json); - setJsonPayload(json, &jexlContext, &payload, &mimeType); + setJsonPayload(json, &exprContext, &payload, &mimeType); renderFormat = NGSI_V2_CUSTOM; } else // customPayloadType == ngsiv2::CustomPayloadType::Ngsi { // Important to use const& for Entity here. Otherwise problems may occur in the object release logic const Entity& ngsi = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.ngsi : notification.mqttInfo.ngsi); - if (!setNgsiPayload(ngsi, subscriptionId, en, &jexlContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) + if (!setNgsiPayload(ngsi, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } mimeType = "application/json"; @@ -427,10 +405,10 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) + if ((macroSubstitute(&key, it->first, &exprContext, "", true) == false) || (macroSubstitute(&value, it->second, &exprContext, "", true) == false)) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } @@ -454,10 +432,10 @@ static SenderThreadParams* buildSenderParamsCustom std::string key = it->first; std::string value = it->second; - if ((macroSubstitute(&key, it->first, &jexlContext, "", true) == false) || (macroSubstitute(&value, it->second, &jexlContext, "", true) == false)) + if ((macroSubstitute(&key, it->first, &exprContext, "", true) == false) || (macroSubstitute(&value, it->second, &exprContext, "", true) == false)) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } @@ -489,7 +467,7 @@ static SenderThreadParams* buildSenderParamsCustom if (!parseUrl(url, host, port, uriPath, protocol)) { LM_E(("Runtime Error (not sending notification: malformed URL: '%s')", url.c_str())); - jexlContext.release(); + exprContext.release(); return NULL; } @@ -521,10 +499,10 @@ static SenderThreadParams* buildSenderParamsCustom // 8. Topic (only in the case of MQTT notifications) if (notification.type == ngsiv2::MqttNotification) { - if (macroSubstitute(&topic, notification.mqttInfo.topic, &jexlContext, "", true) == false) + if (macroSubstitute(&topic, notification.mqttInfo.topic, &exprContext, "", true) == false) { // Warning already logged in macroSubstitute() - jexlContext.release(); + exprContext.release(); return NULL; } } @@ -560,7 +538,7 @@ static SenderThreadParams* buildSenderParamsCustom snprintf(suffix, sizeof(suffix), "%u", correlatorCounter); paramsP->fiwareCorrelator = fiwareCorrelator + "; cbnotif=" + suffix; - jexlContext.release(); + exprContext.release(); return paramsP; } diff --git a/src/lib/parse/CompoundValueNode.cpp b/src/lib/parse/CompoundValueNode.cpp index f9e45c1555..fdf0fa2008 100644 --- a/src/lib/parse/CompoundValueNode.cpp +++ b/src/lib/parse/CompoundValueNode.cpp @@ -612,7 +612,7 @@ bool CompoundValueNode::equal(const orion::BSONElement& be) * CompoundValueNode:toJson * */ -std::string CompoundValueNode::toJson(JexlContext* jexlContextP) +std::string CompoundValueNode::toJson(ExprContextObject* exprContextObjectP) { std::string out; JsonVectorHelper jvh; @@ -621,7 +621,7 @@ std::string CompoundValueNode::toJson(JexlContext* jexlContextP) switch (valueType) { case orion::ValueTypeString: - return smartStringValue(stringValue, jexlContextP, "null"); + return smartStringValue(stringValue, exprContextObjectP, "null"); case orion::ValueTypeNumber: return double2string(numberValue); @@ -635,14 +635,14 @@ std::string CompoundValueNode::toJson(JexlContext* jexlContextP) case orion::ValueTypeVector: for (unsigned int ix = 0; ix < childV.size(); ix++) { - jvh.addRaw(childV[ix]->toJson(jexlContextP)); + jvh.addRaw(childV[ix]->toJson(exprContextObjectP)); } return jvh.str(); case orion::ValueTypeObject: for (unsigned int ix = 0; ix < childV.size(); ix++) { - joh.addRaw(childV[ix]->name, childV[ix]->toJson(jexlContextP)); + joh.addRaw(childV[ix]->name, childV[ix]->toJson(exprContextObjectP)); } return joh.str(); @@ -660,39 +660,39 @@ std::string CompoundValueNode::toJson(JexlContext* jexlContextP) /* **************************************************************************** * -* CompoundValueNode:toJexlContext +* CompoundValueNode:toExprContextObject * */ -JexlContext CompoundValueNode::toJexlContext(void) +ExprContextObject CompoundValueNode::toExprContextObject(void) { - JexlContext jc; + ExprContextObject co; for (uint64_t ix = 0; ix < childV.size(); ++ix) { CompoundValueNode* child = childV[ix]; switch (child->valueType) { case orion::ValueTypeString: - jc.add(child->name, child->stringValue); + co.add(child->name, child->stringValue); break; case orion::ValueTypeNumber: - jc.add(child->name, child->numberValue); + co.add(child->name, child->numberValue); break; case orion::ValueTypeBoolean: - jc.add(child->name, child->boolValue); + co.add(child->name, child->boolValue); break; case orion::ValueTypeNull: - jc.add(child->name); + co.add(child->name); break; case orion::ValueTypeVector: - jc.add(child->name, child->toJexlContextList()); + co.add(child->name, child->toExprContextList()); break; case orion::ValueTypeObject: - jc.add(child->name, child->toJexlContext()); + co.add(child->name, child->toExprContextObject()); break; case orion::ValueTypeNotGiven: @@ -703,46 +703,46 @@ JexlContext CompoundValueNode::toJexlContext(void) LM_E(("Runtime Error (value type unknown (%s))", name.c_str())); } } - return jc; + return co; } /* **************************************************************************** * -* CompoundValueNode:toJexlContextList +* CompoundValueNode:toExprContextList * */ -JexlContextList CompoundValueNode::toJexlContextList(void) +ExprContextList CompoundValueNode::toExprContextList(void) { - JexlContextList jcl; + ExprContextList cl; for (uint64_t ix = 0; ix < childV.size(); ++ix) { CompoundValueNode* child = childV[ix]; switch (child->valueType) { case orion::ValueTypeString: - jcl.add(child->stringValue); + cl.add(child->stringValue); break; case orion::ValueTypeNumber: - jcl.add(child->numberValue); + cl.add(child->numberValue); break; case orion::ValueTypeBoolean: - jcl.add(child->boolValue); + cl.add(child->boolValue); break; case orion::ValueTypeNull: - jcl.add(); + cl.add(); break; case orion::ValueTypeVector: - jcl.add(child->toJexlContextList()); + cl.add(child->toExprContextList()); break; case orion::ValueTypeObject: - jcl.add(child->toJexlContext()); + cl.add(child->toExprContextObject()); break; case orion::ValueTypeNotGiven: @@ -753,7 +753,7 @@ JexlContextList CompoundValueNode::toJexlContextList(void) LM_E(("Runtime Error (value type unknown)")); } } - return jcl; + return cl; } diff --git a/src/lib/parse/CompoundValueNode.h b/src/lib/parse/CompoundValueNode.h index a77eb1a393..bd15aefe64 100644 --- a/src/lib/parse/CompoundValueNode.h +++ b/src/lib/parse/CompoundValueNode.h @@ -35,7 +35,7 @@ #include "mongoDriver/BSONElement.h" -#include "jexl/JexlContext.h" +#include "expressions/ExprContext.h" namespace orion @@ -119,10 +119,10 @@ class CompoundValueNode bool equal(const orion::BSONElement& be); std::string finish(void); - std::string toJson(JexlContext* jexlContextP = NULL); + std::string toJson(ExprContextObject* exprContextObjectP = NULL); - JexlContext toJexlContext(void); - JexlContextList toJexlContextList(void); + ExprContextObject toExprContextObject(void); + ExprContextList toExprContextList(void); void shortShow(const std::string& indent); void show(const std::string& indent); From 49a3694ff2639006c8d3a93c42d15ac8f9230d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 16:34:58 +0100 Subject: [PATCH 169/390] FIX refactor common code and fix utest --- src/lib/expressions/CMakeLists.txt | 2 + src/lib/expressions/ExprContext.h | 6 +- src/lib/expressions/ExprManager.cpp | 52 +--------------- src/lib/expressions/ExprManager.h | 7 +-- src/lib/expressions/ExprResult.cpp | 39 +----------- src/lib/expressions/ExprResult.h | 6 +- src/lib/expressions/exprCommon.cpp | 60 +++++++++++++++++++ src/lib/expressions/exprCommon.h | 41 +++++++++++++ src/lib/expressions/exprMgr.h | 6 +- test/unittests/CMakeLists.txt | 1 + .../common/commonMacroSubstitute_test.cpp | 36 +++++++---- test/unittests/main_UnitTest.cpp | 2 + 12 files changed, 144 insertions(+), 114 deletions(-) create mode 100644 src/lib/expressions/exprCommon.cpp create mode 100644 src/lib/expressions/exprCommon.h diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index 4f938e3634..bfdd9da83c 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -21,6 +21,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES + exprCommon.cpp ExprManager.cpp ExprContext.cpp ExprResult.cpp @@ -28,6 +29,7 @@ SET (SOURCES ) SET (HEADERS + exprCommon.h ExprManager.h ExprContext.h ExprResult.h diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 4db34f562c..365eb65905 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_EXPRCONTEXT_H_ -#define SRC_LIB_JEXL_EXPRCONTEXT_H_ +#ifndef SRC_LIB_EXPRESSIONS_EXPRCONTEXT_H_ +#define SRC_LIB_EXPRESSIONS_EXPRCONTEXT_H_ /* * @@ -73,4 +73,4 @@ class ExprContextList }; -#endif // #define SRC_LIB_JEXL_EXPRCONTEXT_H_ +#endif // SRC_LIB_EXPRESSIONS_EXPRCONTEXT_H_ diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 10739d6a1d..985f1a802a 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -27,48 +27,13 @@ #include "expressions/ExprManager.h" #include "expressions/ExprResult.h" +#include "expressions/exprCommon.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" -/* **************************************************************************** -* -* capturePythonError - -*/ -static const char* capturePythonError() -{ - if (PyErr_Occurred()) - { - PyObject* ptype; - PyObject* pvalue; - PyObject* ptraceback; - - // Fetch the exception type, value, and traceback - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - - if (pvalue != NULL) - { - PyObject* str_obj = PyObject_Str(pvalue); - const char* error_message = PyUnicode_AsUTF8(str_obj); - - // Release the Python objects - Py_XDECREF(str_obj); - Py_XDECREF(ptype); - Py_XDECREF(pvalue); - Py_XDECREF(ptraceback); - - return error_message; - } - } - - PyErr_Clear(); - return ""; -} - - - /* **************************************************************************** * * ExprManager::init - @@ -76,7 +41,6 @@ static const char* capturePythonError() void ExprManager::init(void) { pyjexlModule = NULL; - jsonModule = NULL; jexlEngine = NULL; if (sem_init(&sem, 0, 1) == -1) @@ -95,14 +59,6 @@ void ExprManager::init(void) } LM_T(LmtExpr, ("pyjexl module has been loaded")); - jsonModule = PyImport_ImportModule("json"); - if (jsonModule == NULL) - { - const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error importing json module: %s)", error)); - } - LM_T(LmtExpr, ("json module has been loaded")); - jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); if (jexlEngine == NULL) { @@ -170,12 +126,6 @@ void ExprManager::release(void) LM_T(LmtExpr, ("pyjexl module has been freed")); } - if (jsonModule != NULL) - { - Py_XDECREF(jsonModule); - LM_T(LmtExpr, ("json module has been freed")); - } - Py_Finalize(); LM_T(LmtExpr, ("Python interpreter has been finalized")); } \ No newline at end of file diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 7466372b29..92525900e0 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_EXPRMANAGER_H_ -#define SRC_LIB_JEXL_EXPRMANAGER_H_ +#ifndef SRC_LIB_EXPRESSIONS_EXPRMANAGER_H_ +#define SRC_LIB_EXPRESSIONS_EXPRMANAGER_H_ /* * @@ -42,7 +42,6 @@ class ExprManager PyObject* pyjexlModule; PyObject* jexlEngine; PyObject* jsonModule; - //PyObject* customJsonSerializer; sem_t sem; public: @@ -51,4 +50,4 @@ class ExprManager void release(void); }; -#endif // SRC_LIB_JEXL_EXPRMANAGER_H_ \ No newline at end of file +#endif // SRC_LIB_EXPRESSIONS_EXPRMANAGER_H_ \ No newline at end of file diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 036651fdc4..71dcd6c336 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -24,6 +24,7 @@ */ #include "expressions/ExprResult.h" +#include "expressions/exprCommon.h" #include "common/string.h" #include "common/JsonHelper.h" @@ -31,44 +32,6 @@ -/* **************************************************************************** -* -* capturePythonError - -* -* FIXME PR: duplicate code. Unify -*/ -static const char* capturePythonError() -{ - if (PyErr_Occurred()) - { - PyObject* ptype; - PyObject* pvalue; - PyObject* ptraceback; - - // Fetch the exception type, value, and traceback - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - - if (pvalue != NULL) - { - PyObject* str_obj = PyObject_Str(pvalue); - const char* error_message = PyUnicode_AsUTF8(str_obj); - - // Release the Python objects - Py_XDECREF(str_obj); - Py_XDECREF(ptype); - Py_XDECREF(pvalue); - Py_XDECREF(ptraceback); - - return error_message; - } - } - - PyErr_Clear(); - return ""; -} - - - /* **************************************************************************** * * getPyObjectType - diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index bfa4bcbbb7..c29ffd5c49 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_EXPRRESULT_H_ -#define SRC_LIB_JEXL_EXPRRESULT_H_ +#ifndef SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ +#define SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ /* * @@ -61,4 +61,4 @@ class ExprResult -#endif // #define SRC_LIB_JEXL_EXPRRESULT_H_ +#endif // SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ diff --git a/src/lib/expressions/exprCommon.cpp b/src/lib/expressions/exprCommon.cpp new file mode 100644 index 0000000000..1372d73eaa --- /dev/null +++ b/src/lib/expressions/exprCommon.cpp @@ -0,0 +1,60 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "expressions/exprCommon.h" + +/* **************************************************************************** +* +* capturePythonError - +*/ +const char* capturePythonError() +{ + if (PyErr_Occurred()) + { + PyObject* ptype; + PyObject* pvalue; + PyObject* ptraceback; + + // Fetch the exception type, value, and traceback + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + + if (pvalue != NULL) + { + PyObject* strObj = PyObject_Str(pvalue); + const char* errorMessage = PyUnicode_AsUTF8(strObj); + + // Release the Python objects + Py_XDECREF(strObj); + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + + return errorMessage; + } + } + + PyErr_Clear(); + return ""; +} \ No newline at end of file diff --git a/src/lib/expressions/exprCommon.h b/src/lib/expressions/exprCommon.h new file mode 100644 index 0000000000..7b9667d1b4 --- /dev/null +++ b/src/lib/expressions/exprCommon.h @@ -0,0 +1,41 @@ +#ifndef SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ +#define SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include + + + +/* **************************************************************************** +* +* capturePythonError - +*/ +extern const char* capturePythonError(); + + + +#endif // SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ diff --git a/src/lib/expressions/exprMgr.h b/src/lib/expressions/exprMgr.h index 852e396e24..5b57b3feb8 100644 --- a/src/lib/expressions/exprMgr.h +++ b/src/lib/expressions/exprMgr.h @@ -1,5 +1,5 @@ -#ifndef SRC_LIB_JEXL_EXPRMGR_H_ -#define SRC_LIB_JEXL_EXPRMGR_H_ +#ifndef SRC_LIB_EXPRESSIONS_EXPRMGR_H_ +#define SRC_LIB_EXPRESSIONS_EXPRMGR_H_ /* * @@ -35,4 +35,4 @@ */ extern ExprManager exprMgr; -#endif // SRC_LIB_JEXL_EXPRMGR_H_ +#endif // SRC_LIB_EXPRESSIONS_EXPRMGR_H_ diff --git a/test/unittests/CMakeLists.txt b/test/unittests/CMakeLists.txt index af8fbaf709..a929c69b39 100644 --- a/test/unittests/CMakeLists.txt +++ b/test/unittests/CMakeLists.txt @@ -217,6 +217,7 @@ include_directories("${PROJECT_SOURCE_DIR}/test") # Needed for the new C driver # FIXME: why do we need this declared here if they are also declared # in the main CMakeFiles.txt? +include_directories("/usr/include/python3.11/") include_directories("/usr/local/include/libmongoc-1.0") include_directories("/usr/local/include/libbson-1.0") diff --git a/test/unittests/common/commonMacroSubstitute_test.cpp b/test/unittests/common/commonMacroSubstitute_test.cpp index 40812fb6fb..be51a1e8af 100644 --- a/test/unittests/common/commonMacroSubstitute_test.cpp +++ b/test/unittests/common/commonMacroSubstitute_test.cpp @@ -49,9 +49,12 @@ TEST(commonMacroSubstitute, simple) const char* correct = "Entity E1/T1, attribute 'attr1'"; std::string result; - std::map replacements; - buildReplacementsMap(en, "", "", &replacements); - b = macroSubstitute(&result, s1, &replacements, ""); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add(caP->name, caP->stringValue); + + b = macroSubstitute(&result, s1, &exprContext, "", true); EXPECT_TRUE(b); EXPECT_STREQ(correct, result.c_str()); } @@ -88,9 +91,12 @@ TEST(commonMacroSubstitute, withRealloc) std::string correct = std::string(base) + "Now, finally something to substitute: Entity E1/T1, attribute 'attr1'"; std::string result; - std::map replacements; - buildReplacementsMap(en, "", "", &replacements); - b = macroSubstitute(&result, s1, &replacements, ""); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add(caP->name, caP->stringValue); + + b = macroSubstitute(&result, s1, &exprContext, "", true); EXPECT_TRUE(b); EXPECT_STREQ(correct.c_str(), result.c_str()); } @@ -120,9 +126,12 @@ TEST(commonMacroSubstitute, bufferTooBigInitially) // correct = std::string(base) + "EntityId000001/EntityType000001"; std::string result; - std::map replacements; - buildReplacementsMap(en, "", "", &replacements); - b = macroSubstitute(&result, s1, &replacements, ""); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add(caP->name, caP->stringValue); + + b = macroSubstitute(&result, s1, &exprContext, "", true); EXPECT_FALSE(b); EXPECT_STREQ("", result.c_str()); @@ -158,9 +167,12 @@ TEST(commonMacroSubstitute, bufferTooBigAfterSubstitution) // correct = std::string(base) + "EntityId000001/EntityType000001"; // > 8MB after substitutions std::string result; - std::map replacements; - buildReplacementsMap(en, "", "", &replacements); - b = macroSubstitute(&result, s1, &replacements, ""); + ExprContextObject exprContext; + exprContext.add("id", en.id); + exprContext.add("type", en.type); + exprContext.add(caP->name, caP->stringValue); + + b = macroSubstitute(&result, s1, &exprContext, "", true); EXPECT_FALSE(b); EXPECT_STREQ("", result.c_str()); diff --git a/test/unittests/main_UnitTest.cpp b/test/unittests/main_UnitTest.cpp index 7d7b9bc991..8fa3ad41b7 100644 --- a/test/unittests/main_UnitTest.cpp +++ b/test/unittests/main_UnitTest.cpp @@ -42,6 +42,7 @@ #include "mongoBackend/MongoGlobal.h" #include "ngsiNotify/Notifier.h" #include "alarmMgr/alarmMgr.h" +#include "expressions/exprMgr.h" #include "logSummary/logSummary.h" #include "unittests/unittest.h" @@ -162,6 +163,7 @@ int main(int argC, char** argV) // Note that disableRetryTries, multitenancy and mutex time stats are disabled for unit test mongo init mongoInit(dbURI, dbHost, rplSet, dbName, user, pwd, authMech, authDb, dbSSL, false, false, dbTimeout, writeConcern, dbPoolSize, false); alarmMgr.init(false); + exprMgr.init(); logSummaryInit(&lsPeriod); // setupDatabase(); FIXME #3775: pending on mongo unit test re-enabling From 8417f19b558c763f1dceecbc427142b6c93cb3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 17:01:23 +0100 Subject: [PATCH 170/390] FIX enable venv in ci image utest --- ci/deb/makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/deb/makefile b/ci/deb/makefile index 4cbc001c5d..3e8217c3d3 100644 --- a/ci/deb/makefile +++ b/ci/deb/makefile @@ -69,6 +69,9 @@ install_unit: install unit: @echo '------------------------------------- make unit ----------------------------------------' + # ft_env needed for pyjexl module. Very important to "chain" the two commands with '; \' + # otherwise it seems unitTest doesn't get the venv and it uses default Python in the system + . /opt/ft_env/bin/activate ; \ BUILD_UNIT/test/unittests/unitTest -t 0-255 -dbhost ${MONGO_HOST} --gtest_output=xml:/tmp/builder/logs/unit.xml build_functional: prepare From 8cebfddeba47103a502bf6f3704294d427dc21d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 17:06:24 +0100 Subject: [PATCH 171/390] FIX improvement --- src/lib/expressions/ExprContext.cpp | 147 +++++++++++++++++++++++----- src/lib/expressions/ExprContext.h | 7 +- src/lib/expressions/ExprManager.cpp | 6 +- src/lib/expressions/ExprResult.cpp | 12 +-- 4 files changed, 134 insertions(+), 38 deletions(-) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 8c05fd5764..c5070d486f 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -23,12 +23,11 @@ * Author: Fermin Galan */ -// FIXME PR: add methods should check error in Py_BuildValue() for NULL - #include #include "logMsg/logMsg.h" #include "expressions/ExprContext.h" +#include "expressions/ExprCommon.h" @@ -36,15 +35,15 @@ * * ExprContextObject::ExprContextObject - */ -ExprContextObject::ExprContextObject -() +ExprContextObject::ExprContextObject() { jexlContext = PyDict_New(); if (jexlContext == NULL) { - // FIXME PR: error control - // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + // Note that this ruins the object. We log this situation just one time here, but we include a NULL check + // in every other method to be safer + LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); } } @@ -67,8 +66,17 @@ PyObject* ExprContextObject::get(void) */ void ExprContextObject::add(const std::string& key, const std::string& _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); + return; + } PyDict_SetItemString(jexlContext, key.c_str(), value); Py_DECREF(value); } @@ -81,8 +89,17 @@ void ExprContextObject::add(const std::string& key, const std::string& _value) */ void ExprContextObject::add(const std::string& key, double _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); PyObject* value = Py_BuildValue("d", _value); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); + return; + } PyDict_SetItemString(jexlContext, key.c_str(), value); Py_DECREF(value); } @@ -96,6 +113,10 @@ void ExprContextObject::add(const std::string& key, double _value) */ void ExprContextObject::add(const std::string& key, bool _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value? "true" : "false")); if (_value) { @@ -115,6 +136,10 @@ void ExprContextObject::add(const std::string& key, bool _value) */ void ExprContextObject::add(const std::string& key) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); PyDict_SetItemString(jexlContext, key.c_str(), Py_None); } @@ -127,8 +152,11 @@ void ExprContextObject::add(const std::string& key) */ void ExprContextObject::add(const std::string& key, ExprContextObject exprContextObject) { - // FIXME PR: implement a toString() method in ExprContextObject to be used here - //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (object): %s=%s", key.c_str(), exprContextObject.toString().c_str())); PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.get()); } @@ -140,8 +168,11 @@ void ExprContextObject::add(const std::string& key, ExprContextObject exprContex */ void ExprContextObject::add(const std::string& key, ExprContextList exprContextList) { - // FIXME PR: implement a toString() method in ExprContextList to be used here - //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (list): %s=%s", key.c_str(), exprContextList.toString().c_str())); PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); } @@ -149,16 +180,20 @@ void ExprContextObject::add(const std::string& key, ExprContextList exprContextL /* **************************************************************************** * -* ExprContextObject::hasKey - +* ExprContextObject::toString - */ -bool ExprContextObject::hasKey(const std::string& key) +std::string ExprContextObject::toString(void) { - // Check if the key exists in the jexlContext dictionary - PyObject* keyObject = PyUnicode_FromString(key.c_str()); - int result = PyDict_Contains(jexlContext, keyObject); - Py_DECREF(keyObject); - - return result == 1; // Return true if key exists, false otherwise + const char* str = PyUnicode_AsUTF8(jexlContext); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + return ""; + } + else + { + return std::string(str); + } } @@ -169,11 +204,16 @@ bool ExprContextObject::hasKey(const std::string& key) */ void ExprContextObject::release(void) { + if (jexlContext == NULL) + { + return; + } // FIXME PR: this is not correct. Recursively release of the dict object //Py_XDECREF(jexlContext); } + /* **************************************************************************** * * ExprContextList::ExprContextList - @@ -184,8 +224,9 @@ ExprContextList::ExprContextList() if (jexlContext == NULL) { - // FIXME PR: error control - // Note that this ruins the object... eg. add methods are not calleable (maybe should inclue a NULL check in all them) + // Note that this ruins the object. We log this situation just one time here, but we include a NULL check + // in every other method to be safer + LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); } } @@ -208,8 +249,17 @@ PyObject* ExprContextList::get(void) */ void ExprContextList::add(const std::string& _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context list (string): %s=", _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); + return; + } PyList_Append(jexlContext, value); Py_DECREF(value); } @@ -222,8 +272,17 @@ void ExprContextList::add(const std::string& _value) */ void ExprContextList::add(double _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context list (double): %f", _value)); PyObject* value = Py_BuildValue("d", _value); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); + return; + } PyList_Append(jexlContext, value); Py_DECREF(value); } @@ -237,6 +296,10 @@ void ExprContextList::add(double _value) */ void ExprContextList::add(bool _value) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value? "true" : "false")); if (_value) { @@ -256,6 +319,10 @@ void ExprContextList::add(bool _value) */ void ExprContextList::add(void) { + if (jexlContext == NULL) + { + return; + } LM_T(LmtExpr, ("adding to expression context list (none)")); PyList_Append(jexlContext, Py_None); } @@ -268,8 +335,11 @@ void ExprContextList::add(void) */ void ExprContextList::add(ExprContextObject exprContextObject) { - // FIXME PR: implement a toString() method in ExprContext to be used here - //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context list (object): %s", exprContextObject.toString().c_str())); PyList_Append(jexlContext, exprContextObject.get()); } @@ -281,19 +351,46 @@ void ExprContextList::add(ExprContextObject exprContextObject) */ void ExprContextList::add(ExprContextList exprContextList) { - // FIXME PR: implement a toString() method in ExprContextList to be used here - //LM_T(LmtExpr, ("adding to expression context (none): %s", key.c_str())); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context list (list): %s", exprContextList.toString().c_str())); PyList_Append(jexlContext, exprContextList.get()); } +/* **************************************************************************** +* +* ExprContextList::toString - +*/ +std::string ExprContextList::toString(void) +{ + const char* str = PyUnicode_AsUTF8(jexlContext); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + return ""; + } + else + { + return std::string(str); + } +} + + + /* **************************************************************************** * * ExprContextList::relesase - */ void ExprContextList::release(void) { + if (jexlContext == NULL) + { + return; + } // FIXME PR: this is not correct. Recursively release of the list object - Py_XDECREF(jexlContext); + //Py_XDECREF(jexlContext); } \ No newline at end of file diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 365eb65905..6b41dc7bca 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -50,7 +50,9 @@ class ExprContextObject void add(const std::string& key); void add(const std::string& key, ExprContextObject exprContextObject); void add(const std::string& key, ExprContextList exprContextList); - bool hasKey(const std::string& key); + + std::string toString(void); + void release(void); }; @@ -69,6 +71,9 @@ class ExprContextList void add(void); void add(ExprContextObject exprContextObject); void add(ExprContextList exprContextList); + + std::string toString(void); + void release(void); }; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 985f1a802a..d6aa98d5bd 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -84,8 +84,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st PyObject* expression = Py_BuildValue("s", _expression.c_str()); if (expression == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error building expression: %s", capturePythonError())); + LM_W(("error building JEXL expression: %s", capturePythonError())); return r; } @@ -93,8 +92,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st Py_XDECREF(expression); if (result == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error evaluating expression: %s", capturePythonError())); + LM_W(("error evaluating JEXL expression: %s", capturePythonError())); return r; } diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 71dcd6c336..f2b8d944c6 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -131,8 +131,7 @@ void ExprResult::fill(PyObject* result) const char* str = PyUnicode_AsUTF8(result); if (str == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); valueType = orion::ValueTypeNull; } else @@ -167,8 +166,7 @@ void ExprResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va str = PyUnicode_AsUTF8(value); if (str == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); } else { @@ -243,8 +241,7 @@ void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke const char * keyStr = PyUnicode_AsUTF8(key); if (keyStr == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); return; } @@ -261,8 +258,7 @@ void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke str = PyUnicode_AsUTF8(value); if (str == NULL) { - // FIXME PR: use LM_E/LM_W? - LM_T(LmtExpr, ("error obtaning str representation (string): %s", capturePythonError())); + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); } else { From 0beb957c2c1ebb8f56189bedc25e57a98c273607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 17:34:44 +0100 Subject: [PATCH 172/390] FIX refactor --- src/lib/expressions/ExprContext.cpp | 2 +- src/lib/expressions/ExprResult.cpp | 148 ++++++++++++++-------------- src/lib/expressions/ExprResult.h | 3 - 3 files changed, 75 insertions(+), 78 deletions(-) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index c5070d486f..41ec5b3ddb 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -27,7 +27,7 @@ #include "logMsg/logMsg.h" #include "expressions/ExprContext.h" -#include "expressions/ExprCommon.h" +#include "expressions/exprCommon.h" diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index f2b8d944c6..986ca52e53 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -72,85 +72,15 @@ static orion::ValueType getPyObjectType(PyObject* obj) } - -/* **************************************************************************** -* -* fill - -* -*/ -void ExprResult::fill(PyObject* result) -{ - // If nothing changes, the returned value would be null (failsafe) - valueType = orion::ValueTypeNull; - - // Special case: expresion evalutes to None - if (result == Py_None) - { - LM_T(LmtExpr, ("ExprResult is null")); - valueType = orion::ValueTypeNull; - return; - } - - // Other not null types - valueType = getPyObjectType(result); - if (valueType == orion::ValueTypeNumber) - { - numberValue = PyFloat_AsDouble(result); - LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); - } - else if (valueType == orion::ValueTypeBoolean) - { - boolValue = PyObject_IsTrue(result); - LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); - } - else if (valueType == orion::ValueTypeObject) - { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(result, &pos, &key, &value)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(compoundValueP, key, value); - } - } - else if (valueType == orion::ValueTypeVector) - { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); - Py_ssize_t size = PyList_Size(result); - for (Py_ssize_t ix = 0; ix < size; ++ix) - { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(compoundValueP, PyList_GetItem(result, ix)); - } - } - else if (valueType == orion::ValueTypeString) - { - const char* str = PyUnicode_AsUTF8(result); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtExpr, ("ExprResult (string): %s", str)); - stringValue = std::string(str); - } - } -} - +static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); // forward declaration /* **************************************************************************** * * processListItem - * -* FIXME PR: maybe this should be static function out of the class? */ -void ExprResult::processListItem(orion::CompoundValueNode* parentP, PyObject* value) +void processListItem(orion::CompoundValueNode* parentP, PyObject* value) { orion::CompoundValueNode* nodeP; @@ -234,9 +164,8 @@ void ExprResult::processListItem(orion::CompoundValueNode* parentP, PyObject* va * * processDictItem - * -* FIXME PR: maybe this should be static function out of the class? */ -void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) { const char * keyStr = PyUnicode_AsUTF8(key); if (keyStr == NULL) @@ -322,6 +251,77 @@ void ExprResult::processDictItem(orion::CompoundValueNode* parentP, PyObject* ke +/* **************************************************************************** +* +* fill - +* +*/ +void ExprResult::fill(PyObject* result) +{ + // If nothing changes, the returned value would be null (failsafe) + valueType = orion::ValueTypeNull; + + // Special case: expresion evalutes to None + if (result == Py_None) + { + LM_T(LmtExpr, ("ExprResult is null")); + valueType = orion::ValueTypeNull; + return; + } + + // Other not null types + valueType = getPyObjectType(result); + if (valueType == orion::ValueTypeNumber) + { + numberValue = PyFloat_AsDouble(result); + LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); + } + else if (valueType == orion::ValueTypeBoolean) + { + boolValue = PyObject_IsTrue(result); + LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); + } + else if (valueType == orion::ValueTypeObject) + { + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(result, &pos, &key, &value)) + { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() + processDictItem(compoundValueP, key, value); + } + } + else if (valueType == orion::ValueTypeVector) + { + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); + Py_ssize_t size = PyList_Size(result); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(compoundValueP, PyList_GetItem(result, ix)); + } + } + else if (valueType == orion::ValueTypeString) + { + const char* str = PyUnicode_AsUTF8(result); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtExpr, ("ExprResult (string): %s", str)); + stringValue = std::string(str); + } + } +} + + + /* **************************************************************************** * * toString - diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index c29ffd5c49..df1ec0f597 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -52,9 +52,6 @@ class ExprResult void fill(PyObject* result); - void processListItem(orion::CompoundValueNode* parentP, PyObject* item); - void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); - std::string toString(void); void release(void); }; From 6f77ee597953e6f0c1e69de5b417bbd99cc542c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 19 Feb 2024 17:55:40 +0100 Subject: [PATCH 173/390] FIX indent --- src/lib/apiTypesV2/Entity.cpp | 4 ++-- src/lib/apiTypesV2/Entity.h | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/apiTypesV2/Entity.cpp b/src/lib/apiTypesV2/Entity.cpp index f864e0bcb2..f51cbd6b35 100644 --- a/src/lib/apiTypesV2/Entity.cpp +++ b/src/lib/apiTypesV2/Entity.cpp @@ -308,7 +308,7 @@ std::string Entity::toJson bool blacklist, const std::vector& metadataFilter, bool renderNgsiField, - ExprContextObject* exprContextObjectP + ExprContextObject* exprContextObjectP ) { std::vector orderedAttrs; @@ -444,7 +444,7 @@ std::string Entity::toJsonNormalized const std::vector& orderedAttrs, const std::vector& metadataFilter, bool renderNgsiField, - ExprContextObject* exprContextObjectP + ExprContextObject* exprContextObjectP ) { JsonObjectHelper jh; diff --git a/src/lib/apiTypesV2/Entity.h b/src/lib/apiTypesV2/Entity.h index f8acc3af97..9d534d52ed 100644 --- a/src/lib/apiTypesV2/Entity.h +++ b/src/lib/apiTypesV2/Entity.h @@ -91,8 +91,8 @@ class Entity const std::vector& attrsFilter, bool blacklist, const std::vector& metadataFilter, - bool renderNgsiField = false, - ExprContextObject* exprContext = NULL); + bool renderNgsiField = false, + ExprContextObject* exprContextObjectP = NULL); std::string toJson(RenderFormat renderFormat, bool renderNgsiField = false); @@ -145,8 +145,8 @@ class Entity std::string toJsonKeyvalues(const std::vector& orderedAttrs); std::string toJsonNormalized(const std::vector& orderedAttrs, const std::vector& metadataFilter, - bool renderNgsiField = false, - ExprContextObject* exprContext = NULL); + bool renderNgsiField = false, + ExprContextObject* exprContextObject = NULL); }; #endif // SRC_LIB_APITYPESV2_ENTITY_H_ From b56cd7e9062214c35a8ddcc519fcd5eb8bd31ee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 21 Feb 2024 13:25:21 +0100 Subject: [PATCH 174/390] FIX add libpython to version operation --- .../cases/0000_version_operation/version_via_rest.test | 1 + test/functionalTest/cases/0501_cors/version_request.test | 3 +++ .../cases/1916_fiware_correlator/fiware_correlator.test | 2 ++ test/unittests/serviceRoutines/versionTreat_test.cpp | 2 ++ 4 files changed, 8 insertions(+) diff --git a/test/functionalTest/cases/0000_version_operation/version_via_rest.test b/test/functionalTest/cases/0000_version_operation/version_via_rest.test index ddd58071be..e4fd1517f7 100644 --- a/test/functionalTest/cases/0000_version_operation/version_via_rest.test +++ b/test/functionalTest/cases/0000_version_operation/version_via_rest.test @@ -64,6 +64,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/functionalTest/cases/0501_cors/version_request.test b/test/functionalTest/cases/0501_cors/version_request.test index 2cdcafe8a1..57b963b40a 100644 --- a/test/functionalTest/cases/0501_cors/version_request.test +++ b/test/functionalTest/cases/0501_cors/version_request.test @@ -94,6 +94,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -130,6 +131,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -164,6 +166,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test index 5e52e7eea5..01cd198b53 100644 --- a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test +++ b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test @@ -73,6 +73,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -107,6 +108,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", + "libpython": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/unittests/serviceRoutines/versionTreat_test.cpp b/test/unittests/serviceRoutines/versionTreat_test.cpp index 9950e5d6f0..fd8a4e2e4b 100644 --- a/test/unittests/serviceRoutines/versionTreat_test.cpp +++ b/test/unittests/serviceRoutines/versionTreat_test.cpp @@ -94,6 +94,8 @@ TEST(versionTreat, ok) EXPECT_TRUE(strstr(out.c_str(), "boost") != NULL); EXPECT_TRUE(strstr(out.c_str(), "libcurl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "libmicrohttpd") != NULL); + EXPECT_TRUE(strstr(out.c_str(), "libmosquitto") != NULL); + EXPECT_TRUE(strstr(out.c_str(), "libpython") != NULL); EXPECT_TRUE(strstr(out.c_str(), "openssl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "rapidjson") != NULL); EXPECT_TRUE(strstr(out.c_str(), "mongoc") != NULL); From df3e8ddc3bed53fe17d9c7fc2801e4d819949cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 21 Feb 2024 13:51:30 +0100 Subject: [PATCH 175/390] FIX version operation --- src/lib/serviceRoutines/versionTreat.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index 981c81f2b6..5ea06218d9 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -46,6 +46,7 @@ #include #include #include +#include /* **************************************************************************** * @@ -70,6 +71,7 @@ std::string libVersions(void) std::string mhd = " \"libmicrohttpd\": "; std::string ssl = " \"openssl\": "; std::string rjson = " \"rapidjson\": "; + std::string python = " \"libpython\": "; std::string mongo = " \"mongoc\": "; std::string bson = " \"bson\": "; @@ -84,10 +86,17 @@ std::string libVersions(void) char mosqVersion[16]; snprintf(mosqVersion, sizeof(mosqVersion), "%d.%d.%d", mosqMayor, mosqMinor, mosqRevision); + char pyVersion[16]; + snprintf(pyVersion, sizeof(pyVersion), "%d.%d.%d", + (PY_VERSION_HEX >> 24) & 0xFF, + (PY_VERSION_HEX >> 16) & 0xFF, + (PY_VERSION_HEX >> 8) & 0xFF); + total += boost + "\"" + BOOST_LIB_VERSION "\"" + ",\n"; total += curl + "\"" + curlVersion + "\"" + ",\n"; total += mosq + "\"" + mosqVersion + "\"" + ",\n"; total += mhd + "\"" + MHD_get_version() + "\"" + ",\n"; + total += python + "\"" + pyVersion + "\"" + ",\n"; #ifdef OLD_SSL_VERSION_FORMAT // Needed by openssl 1.1.1n in Debian 11 and before total += ssl + "\"" + SHLIB_VERSION_NUMBER "\"" + ",\n"; From a8411ab4b7baf20c074843db8bc46bd3895e0c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 21 Feb 2024 16:06:33 +0100 Subject: [PATCH 176/390] FIX new ftest --- ...c.test => jexl_basic_attrs_in_update.test} | 12 +- .../jexl_basic_attrs_not_in_update.test | 173 ++++++++++++++ .../jexl_expr_attrs_weird_syntax.test | 213 ++++++++++++++++++ .../jexl_json_navigation.test | 199 ++++++++++++++++ 4 files changed, 591 insertions(+), 6 deletions(-) rename test/functionalTest/cases/4004_jexl_expressions_in_subs/{jexl_basic.test => jexl_basic_attrs_in_update.test} (94%) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test similarity index 94% rename from test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test rename to test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test index 209b61474c..5e031e80e7 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification +JEXL expression in custom notification (source attributes in udpate) --SHELL-INIT-- dbInit CB @@ -35,7 +35,7 @@ accumulatorStart --pretty-print # 02. Create entity E1 with A=1 and B=2 # 03. Update entity E1 with A=foo and B=bar # 04. Update entity E1 with A=2.1 and B=-3.8 -# 05. Dump accumulator and see two notifications (sum: 3 and sum: foobar) +# 05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7) # @@ -120,8 +120,8 @@ echo echo -echo "05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1,7)" -echo "=================================================================================" +echo "05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7)" +echo "===============================================================================" accumulatorDump echo echo @@ -164,8 +164,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1,7) -================================================================================= +05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7) +=============================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 221 diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test new file mode 100644 index 0000000000..eaef9821eb --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test @@ -0,0 +1,173 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (source attributes not in udpate) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression sum: A+B +# 02. Create entity E1 with A=1 and B=2 +# 03. Update entity E1 with C=foo to trigger notification +# 04. Dump accumulator and see notifications (sum: 3) +# + + +echo "01. Create custom sub with custom expression sum: A+B" +echo "=====================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ], + "condition": { + "attrs": [ "C" ] + } + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "sum": { + "value": "${A+B}", + "type": "Calculated" + } + } + }, + "attrs": [ "sum" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1 and B=2" +echo "=====================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with C=foo to trigger notification" +echo "=======================================================" +payload='{ + "C": { + "value": "foo", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (sum: 3)" +echo "===================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: A+B +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 and B=2 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with C=foo to trigger notification +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (sum: 3) +=================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 129 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test new file mode 100644 index 0000000000..3390b28e82 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -0,0 +1,213 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Legacy expression using attributes with weird syntax + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression: A-B and A:B +# 02. Create entity E1 with A-B 1 and A:B 2 +# 03. Update entity E1 with A-B 3 and A:B 4 +# 04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null) +# + + +echo "01. Create custom sub with custom expression: A-B and A:B" +echo "=========================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "R1": { + "value": "${A-B}", + "type": "Calculated" + }, + "R2": { + "value": "${A:B}", + "type": "Calculated" + } + } + }, + "attrs": [ "R1", "R2" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A-B 1 and A:B 2" +echo "=========================================" +payload='{ + "id": "E1", + "type": "T", + "A-B": { + "value": 1, + "type": "Number" + }, + "A:B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A-B 3 and A:B 4" +echo "=========================================" +payload='{ + "A-B": { + "value": 3, + "type": "Number" + }, + "A:B": { + "value": 4, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null)" +echo "==================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression: A-B and A:B +========================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A-B 1 and A:B 2 +========================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A-B 3 and A:B 4 +========================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null) +================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 185 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "R1": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "R2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 185 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "R1": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "R2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test new file mode 100644 index 0000000000..236b872365 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test @@ -0,0 +1,199 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (source attributes not in udpate) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression cal: A[0] + A[1].A1 + B.B1[2].C +# 02. Create entity E1 with A and B so cal result in 60 +# 03. Update entity E1 with A and B so cal result in 6 +# 04. Dump accumulator and see notifications (cal 60 and cal 6) +# + + +echo "01. Create custom sub with custom expression cal: A[0] + A[1].A1 + B.B1[2].C" +echo "============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "cal": { + "value": "${A[0] + A[1].A1 + B.B1[2].C}", + "type": "Calculated" + } + } + }, + "attrs": [ "cal" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A and B so cal result in 60" +echo "=====================================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": [ 10, { "A1": 20 } ], + "type": "StructuredValue" + }, + "B": { + "value": { "B1": [ 11, 22, { "C": 30 }]}, + "type": "StructuredValue" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A and B so cal result in 6" +echo "====================================================" +payload='{ + "A": { + "value": [ 1, { "A1": 2 } ], + "type": "StructuredValue" + }, + "B": { + "value": { "B1": [ 11, 22, { "C": 3 }]}, + "type": "StructuredValue" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (cal 60 and cal 6)" +echo "=============================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression cal: A[0] + A[1].A1 + B.B1[2].C +============================================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A and B so cal result in 60 +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A and B so cal result in 6 +==================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (cal 60 and cal 6) +============================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 130 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": 60 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 129 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": 6 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 574be0475e5c7ccadee21fafee9432915cec1e5e Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Fri, 23 Feb 2024 22:56:32 +0900 Subject: [PATCH 177/390] (JP) FIX deprecated old db CLI (#4505) --- doc/manuals.jp/admin/cli.md | 23 ---------------------- doc/manuals.jp/admin/database_admin.md | 17 ++++++++-------- doc/manuals.jp/admin/diagnosis.md | 2 +- doc/manuals.jp/admin/sanity_check.md | 13 ++++--------- doc/manuals.jp/admin/watchdog.md | 2 +- doc/manuals.jp/deprecated.md | 27 ++++++++++++++++++++++++++ doc/manuals.jp/devel/cookbook.md | 2 +- docker/README.jp.md | 10 +++++----- docker/docker_swarm.jp.md | 2 +- docker/raspberry_pi.jp.md | 2 +- 10 files changed, 50 insertions(+), 50 deletions(-) diff --git a/doc/manuals.jp/admin/cli.md b/doc/manuals.jp/admin/cli.md index 076b43d4e3..bdc50ba14a 100644 --- a/doc/manuals.jp/admin/cli.md +++ b/doc/manuals.jp/admin/cli.md @@ -34,23 +34,7 @@ broker はデフォルトでバックグラウンドで実行されるため、 - **-db ** : 使用する MogoDB データベース、または (`-multiservice` を使用している場合) サービス単位/テナント単位のデータベースのプレフィックス ([マルチ・テナンシー](../orion-api.md#multi-tenancy)のセクションを参照してください) です。このフィールドは最大10文字までです - **-dbURI ** : 使用する MongoDB を URI で指定します。 URI に文字列 `${PWD}` がある場合は `-dbpwd` または環境変数 `ORION_MONGO_PASSWORD` で指定したパスワードで置き換えられます。 - このオプションは `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL`, `-dbDisableRetryWrites` と組み合わせできません。（組み合わせた場合、Orion は起動時にエラーで終了します） -- **-dbhost ** : 使用する MongoDB のホストとポートです。たとえば、`-dbhost localhost:12345` です -- **-rplSet ** : 指定すれば、Orion CB が MongoDB レプリカセット (スタンドアロン MongoDB インスタンスではなく) に接続されます。使用するレプリカセットの名前は、パラメータの値です。この場合、-dbhost パラメーターは、レプリカ・セットのシードとして使用されるホスト ("," で区切られた) のリストにすることができます -- **-dbTimeout ** : レプリカセット (-rplSet) を使用する場合にのみ使用され、それ以外の場合は無視されます。レプリカセットへの接続のタイムアウトをミリ秒単位で指定します -- **-dbuser ** : 使用する MongoDB ユーザ。MongoDB が認証を使用しない場合、このオプションは避けなければなりません。[データベース認証セクション](database_admin.md#database-authorization)を参照してください - **-dbpwd ** : 使用する MongoDB パスワード。MongoDB が認証を使用しない場合、このオプションは避けなければなりません。[データベース認証セクション](database_admin.md#database-authorization)を参照してください -- **-dbAuthMech **. `-dbuser` と `-dbpwd` を提供する場合に使用する MongoDB - 認証メカニズム。代替手段はSCRAM-SHA-1 または SCRAM-SHA-256 です。 -- **-dbAuthDb ** : `-dbuser` と `-dbpwd` を提供する場合に認証に使用するデータベース - を指定します。 -- **-dbSSL** : MongoDB への接続で SSL を有効にします。MongoDB サーバまたはレプリカ・セットが - SSL を使用している場合は、このオプションを使用する必要があります (または、逆に、MongoDB - サーバまたはレプリカ・セットが SSL を使用していない場合は、このオプションを使用する必要は - ありません)。現在、制限があることに注意してください。この場合、Orion は `tlsAllowInvalidCertificates=true` - を使用するため、MongoDB サーバで使用される証明書は検証されません。 -- **-dbDisableRetryWrites** : DB 接続で retryWrite パラメータを false に設定します - (古い MongoDB インスタンスとの互換性を維持するためにのみで、通常は推奨されません) - **-dbPoolSize ** : データベース・コネクション・プール プールのデフォルトサイズは10接続です - **-writeConcern <0|1>** : MongoDB の書き込み操作に対する確認を指定 : 確認 (1) または未確認 (0)。デフォルトは 1です - **-https** : セキュアな HTTP モードで作業します (`-cert` および `-key` を参照) @@ -143,15 +127,8 @@ Orion は、環境変数を使用した引数の受け渡しをサポートし | ORION_PORT | port | | ORION_PID_PATH | pidpath | | ORION_MONGO_URI | dbURI | -| ORION_MONGO_HOST | dbhost | -| ORION_MONGO_REPLICA_SET | rplSet | -| ORION_MONGO_USER | dbuser | | ORION_MONGO_PASSWORD | dbpwd | -| ORION_MONGO_AUTH_MECH | dbAuthMech | -| ORION_MONGO_AUTH_SOURCE | dbAuthDb | -| ORION_MONGO_SSL | dbSSL | | ORION_MONGO_DB | db | -| ORION_MONGO_TIMEOUT | dbTimeout | | ORION_MONGO_POOL_SIZE | dbPoolSize | | ORION_USEIPV4 | ipv4 | | ORION_USEIPV6 | ipv6 | diff --git a/doc/manuals.jp/admin/database_admin.md b/doc/manuals.jp/admin/database_admin.md index c3c6b28da3..eb0138dc87 100644 --- a/doc/manuals.jp/admin/database_admin.md +++ b/doc/manuals.jp/admin/database_admin.md @@ -54,27 +54,28 @@ mongorestore --host --db dump/ ## データベースの認証 -MongoDB の認証は `-db, `-dbuser` と `-dbpwd` オプションで設定されます ([コマンドライン・オプションのセクション](cli.md)を参照)。考慮するいくつかの異なるケースがあります : +MongoDB の認証は `-dbURI` および `-dbpwd` オプションを使用して設定されます ([コマンドライン・オプションのセクションを参照](cli.md))。考慮すべきいくつかの異なるケースがあります: -- MongoDB インスタンス/クラスタが認証を使用していない場合は、`-dbuser` と `-dbpwd` オプションは使用しないでください -- `-dbAuthMech` で認証メカニズムを指定できます +- MongoDB インスタンス/クラスタが認証を使用していない場合は、`-dbpwd` を使用せず、`username:${PWD}@` のパートを省略した `-dbURI` を使用してください +- `authMechanism` オプションを使用して、`-dbURI` 内で認証メカニズムを指定できます - MongoDB インスタンス/クラスタが認可を使用している場合は、次のようになります : - - Orion をシングルサービス/テナントモードで実行している場合 (つまり `-multiservice` でない場合)、1つのデータベース (-db オプションで指定されたもの) のみを使用しているので、認証は、そのデータベースで `-dbuser` と `-dbpwd` を使用して行われます - - Orion をマルチサービス/テナントモードで実行している場合 (つまり `-multiservice` の場合)、認証は、`admin` データベースで `-dbuser` と `-dbpwd` を使用して行われます。[このドキュメントの後半](#multiservicemultitenant-database-separation)で説明するように、マルチサービス/テナントモードでは、Orion はいくつかのデータベース (潜在的にオンザフライで作成される可能性があります) を使用します。`admin` データベース上での認証は、それらの全てで許可します - - とにかく、上記のデフォルトを `-dbAuthDb` でオーバーライドして、 + - `-dbURI` では、`username:${PWD}@` のパートを使用する必要があります。`${PWD}` は `dbpwd` パラメータの値に置き換えられます + - Orion をシングルサービス/テナントモードで実行している場合 (つまり `-multiservice` でない場合)、1つのデータベース (`-db` オプションで指定されたもの) であり、認証は、`-dbURI` および `-dbpwd` で指定されたユーザ名を使用してそのデータベース内で行われます + - Orion をマルチサービス/テナントモードで実行している場合 (つまり `-multiservice` の場合)、認証は、`admin` データベースで `-dbURI` で指定されたユーザー名と `-dbpwd` を使用して行われます。[このドキュメントの後半](#multiservicemultitenant-database-separation)で説明するように、マルチサービス/テナントモードでは、Orion はいくつかのデータベース (潜在的にオンザフライで作成される可能性があります) を使用します。`admin` データベース上での認証は、それらの全てで許可します + - とにかく、上記のデフォルトを `-dbURI` の `defaultauthdb` でオーバーライドして、 必要な認証 DB を指定できます 次の例を考えてみましょう。 MongoDB の構成がそうである場合、通常は以下を使用してアクセスします : ``` -mongo "mongodb://example1.net:27017,example2.net:27017,example3.net:27017/orion?replicaSet=rs0" --ssl --authenticationDatabase admin --username orion --password orionrules +mongosh mongodb://orion@orionrules:example1.net:27017,example2.net:27017,example3.net:27017/admin?replicaSet=rs0&tls=true&tlsAllowInvalidCertificates=true ``` Context Broker CLI パラメーターの同等の接続は次のようになります : ``` --dbhost examples1.net:27017,example2.net:27017,example3.net:27017 -rplSet rs0 -dbSSL -dbAuthDb admin -dbuser orion -dbpwd orionrules +-dbURI mongodb://orion@${PWD}:example1.net:27017,example2.net:27017,example3.net:27017/admin?replicaSet=rs0&tls=true&tlsAllowInvalidCertificates=true -dbpwd orionrules ``` diff --git a/doc/manuals.jp/admin/diagnosis.md b/doc/manuals.jp/admin/diagnosis.md index 03e082444b..b52e185dcb 100644 --- a/doc/manuals.jp/admin/diagnosis.md +++ b/doc/manuals.jp/admin/diagnosis.md @@ -220,7 +220,7 @@ Orion Context Broker は、次のフローを使用します : } ``` -どちらの場合も、MonogDB への接続が正しく構成されていることを確認してください。特に、[コマンドラインから実行する](cli.md)場合は "-dbhost" オプションです。また、シャーディングを使用しているかどうかによって異なりますが、mongod/mongos プロセスが起動していることです。 +どちらの場合も、MonogDB への接続が正しく構成されていることを確認してください。特に、[コマンドラインから実行する](cli.md)場合は `-dbURI` オプションです。また、シャーディングを使用しているかどうかによって異なりますが、mongod/mongos プロセスが起動していることです。 MongoDB が停止していることが問題の場合は、Orion Context Broker は、準備ができたらデータベースに再接続できることに注意してください。つまり、データベースに再接続するために broker を再起動する必要はありません。 diff --git a/doc/manuals.jp/admin/sanity_check.md b/doc/manuals.jp/admin/sanity_check.md index d1288f93cd..bf89c90187 100644 --- a/doc/manuals.jp/admin/sanity_check.md +++ b/doc/manuals.jp/admin/sanity_check.md @@ -45,7 +45,7 @@ curl --header 'Accept: application/json' localhost:1026/version ``` $ ps ax | grep contextBroker - 8517 ? Ssl 8:58 /usr/bin/contextBroker -port 1026 -logDir /var/log/contextBroker -pidpath /var/log/contextBroker/contextBroker.pid -dbhost localhost -db orion + 8517 ? Ssl 8:58 /usr/bin/contextBroker -port 1026 -logDir /var/log/contextBroker -pidpath /var/log/contextBroker/contextBroker.pid -dbURI mongodb://localhost/ -db orion ``` [トップ](#top) @@ -63,25 +63,20 @@ Orion Context Broker は、デフォルト・ポートとして TCP 1026を使 Orion Context Broker は MongoDB データベースを使用します。このパラメーターは、 コマンドライン・オプションを使用して提供されます : -* `-dbhost` +* `-dbUri` * `-db` -* `-dbuser` * `-dbpwd` -* `-dbAuthMech` -* `-dbAuthDb` -* `-dbSSL` -* `-dbDisableRetryWrites` -* `-dbTimeout` * `-dbPoolSize` * `-writeConcern` `-dbuser`, `-dbpwd`, `-dbAuthMech` および `-dbAuthDb` は、MongoDB が認証を使用 して、つまり `--auth` で実行される場合にのみ使用されることを注意して ください。 +`-dbpwd` は、MongoDB が認証を使用して実行される場合、つまり `--auth` を使用して実行される場合にのみ使用されることに注意してください。 mongo コンソールを使用してデータベースが動作していることを確認できます : ``` -mongo / +mongosh / ``` mongo コンソールの次のコマンドを使用して、broker が使用するさまざまなコレクションをチェックすることができます。ただし、最初にドキュメントを挿入するときに broker がコレクションを作成するので、broker を初めて実行する、またはデータベースがクリーンアップされている、broker がまだリクエストを受信していない場合、コレクションは存在しません。特定の瞬間に実際のコレクションリストを取得するために `show collections` を使用します。 diff --git a/doc/manuals.jp/admin/watchdog.md b/doc/manuals.jp/admin/watchdog.md index c3d6bc7afa..1546d3a861 100644 --- a/doc/manuals.jp/admin/watchdog.md +++ b/doc/manuals.jp/admin/watchdog.md @@ -72,7 +72,7 @@ monit が正常に動作していることを確認するには、プロセス # ps -ef | grep contextBroker 500 27175 1 0 21:06 ? 00:00:00 monit -v -c /home/localadmin/monit_CB/monitBROKER.conf -d 10 -p /var/log/contextBroker/monit.pid - 500 27205 1 0 21:06 ? 00:00:00 /usr/bin/contextBroker -port 1026 -logDir /var/log/contextBroker -pidpath /var/log/contextBroker/contextBroker.pid -dbhost localhost -db orion; + 500 27205 1 0 21:06 ? 00:00:00 /usr/bin/contextBroker -port 1026 -logDir /var/log/contextBroker -pidpath /var/log/contextBroker/contextBroker.pid -dbURI mongodb://localhost/ -db orion; そして、contextBroker を kill します。例 : diff --git a/doc/manuals.jp/deprecated.md b/doc/manuals.jp/deprecated.md index 3b81eacd34..a515871299 100644 --- a/doc/manuals.jp/deprecated.md +++ b/doc/manuals.jp/deprecated.md @@ -8,6 +8,7 @@ 推奨されなくなった機能のリストと、廃止された機能のバージョンは次のとおりです : +* Orion 3.12.0 での CLI パラメータ (および関連する環境変数): `-dbhost`、`-rplSet`、`-dbTimeout`、`-dbuser`、`-dbAuthMech`、`-dbAuthDb`、`-dbSSL`、および `-dbDisableRetryWrites`。MongoDB URI を構築するために必要な情報が必要な場合は、[このセクション](#mapping-to-mongouri-from-old-cli-parameters) をチェックして、代わりに `dbURI` を使用してください * Orion 3.10.0 での `geo:point`, `geo:line`, `geo:box` および `geo:polygon` 属性タイプ。代わりに `geo:json` を使用してください * Orion 3.8.0 での `GET /v2` 操作。この操作はかなり役に立たず、実際には使用されません。 * Orion 3.1.0 のサブスクリプションでの初期通知 (`skipInitialNotification` オプションと共に)。(Orion 3.2.0 で削除)。初期通知の @@ -52,6 +53,31 @@ * Configuration Manager のロールは、0.21.0 で非推奨になり、0.25.0 で削除されました * Associations は、0.21.0 で非推奨になり、0.25.0 で削除されました + + +### 古い CLI パラメータから MongoURI へのマッピング + +次の CLI パラメータがあることを考慮します: + +* `-dbhost HOST` +* `-rplSet RPLSET` +* `-dbTimeout TIMEOUT` +* `-dbuser USER` +* `-dbpass PASS` +* `-dbAuthMech AUTHMECH` +* `-dbAuthDb AUTHDB` +* `-dbSSL` +* `-dbDisableRetryWrites` + +結果の MongoURI (つまり、`-dbURI` の値) は次のようになります: + +> mongodb://[USER:PASS@]HOST/[AUTHDB][?replicaSet=RPLSET[&authMechanism=AUTHMECH][&tls=true&tlsAllowInvalidCertificates=true][&retryWrites=false][&connectTimeoutMS=TIMEOUT] +Notes: + +* `-dbSSL` が使用される場合、`&tls=true&tlsAllowInvalidCertificates=true` トークンが追加されます +* `-dbDisableRetryWrites` が使用される場合、`&retryWrites=false` トークンが追加されます +* その他の `[...]` は、対応するパラメータが使用されているかどうかに応じて、オプションのトークンを意味します + ## 非推奨の警告をログに記録 @@ -77,6 +103,7 @@ | **削除された機能** | **機能をサポートする Orion ラスト・バージョン** | **バージョンのリリース日** | |--------------------------------------------------------------------------------------|-------------------------------------------------|----------------------------| +| CLI `-dbhost`、`-rplSet`、`-dbTimeout`、`-dbuser`、`-dbAuthMech`、`-dbAuthDb`、`-dbSSL`、および `-dbDisableRetryWrites` (および関連する環境変数) | まだ定義されていません | まだ定義されていません | | `POST /v2/entities` オペレーションの `attributes` フィールド | まだ定義されていません | まだ定義されていません | | `APPEND`, `UPDATE`, など。`POST /v2/op/update` でのアクション・タイプ | まだ定義されていません | まだ定義されていません | | URI パラメータでの `dateCreated` および `dateModified` | まだ定義されていません | まだ定義されていません | diff --git a/doc/manuals.jp/devel/cookbook.md b/doc/manuals.jp/devel/cookbook.md index f25aa69f26..633a7e25eb 100644 --- a/doc/manuals.jp/devel/cookbook.md +++ b/doc/manuals.jp/devel/cookbook.md @@ -20,7 +20,7 @@ Orion に新しい CLI パラメータを追加するのは簡単です。これ * 新しい CLI パラメーターの値を保持する**変数**、および * `PaArgument` ベクトルの**新しい項目** `paArgs` -新しいCLIパラメーターが `-v` (verbose) のようなブール値の場合、`bool` 変数が必要です。`-dbHost ` のようなテキスト・パラメータの場合は、char-vector が使用されるなどです。 +新しいCLIパラメーターが `-v` (verbose) のようなブール値の場合、`bool` 変数が必要です。`-dbURI ` のようなテキスト・パラメータの場合は、char-vector が使用されるなどです。 最も簡単な方法は、同じタイプの古い CLI パラメータを単純にコピーすることです。 diff --git a/docker/README.jp.md b/docker/README.jp.md index c36ae73140..e1080f2888 100644 --- a/docker/README.jp.md +++ b/docker/README.jp.md @@ -33,7 +33,7 @@ Orion Context Broker を試してみたいし、データベースについて - "1026:1026" depends_on: - mongo - command: -dbhost mongo + command: -dbURI mongodb://mongo mongo: image: mongo:6.0 @@ -79,7 +79,7 @@ Orion Context Broker を試してみたいし、データベースについて そして、このコマンドで Orion を実行します - sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 fiware/orion -dbhost mongodb + sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 fiware/orion -dbURI mongodb://mongodb すべてが動作することを確認します。 @@ -91,7 +91,7 @@ Orion Context Broker を試してみたいし、データベースについて 別の MongoDB インスタンスに接続する場合は、前のコマンドの**代わりに**、次のコマンドを実行します - sudo docker run -d --name orion1 -p 1026:1026 fiware/orion -dbhost + sudo docker run -d --name orion1 -p 1026:1026 fiware/orion -dbURI mongodb:// すべてが動作することを確認します。 @@ -111,10 +111,10 @@ Orion Context Broker を試してみたいし、データベースについて * 手動で MongoDB を別のコンテナで実行します : 1. `sudo docker run --name mongodb -d mongo:6.0` 2. `sudo docker build -t orion .` - 3. `sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 orion -dbhost mongodb`. + 3. `sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 orion -dbURI mongodb://mongodb`. * 手動で MongoDB ホストを見つける場所を指定します : 1. `sudo docker build -t orion .` - 2. `sudo docker run -d --name orion1 -p 1026:1026 orion -dbhost `. + 2. `sudo docker run -d --name orion1 -p 1026:1026 orion -dbURI mongodb://`. すべてが動作することを確認します diff --git a/docker/docker_swarm.jp.md b/docker/docker_swarm.jp.md index 6b35f7920d..494fc4ac3d 100644 --- a/docker/docker_swarm.jp.md +++ b/docker/docker_swarm.jp.md @@ -172,7 +172,7 @@ MongoDB ReplicaSet を Docker Swarm にデプロイする方法の詳細につ image: fiware/orion:latest ports: - "1026:1026" - command: -logLevel DEBUG -dbhost mongo_mongo -rplSet rs -dbTimeout 10000 + command: -logLevel DEBUG -dbURI mongodb://mongo_mongo/?replicaSet=rs&connectTimeoutMS=10000 deploy: replicas: 2 networks: diff --git a/docker/raspberry_pi.jp.md b/docker/raspberry_pi.jp.md index 6a2e961cc7..03ee1ec375 100644 --- a/docker/raspberry_pi.jp.md +++ b/docker/raspberry_pi.jp.md @@ -47,7 +47,7 @@ services: - "1026:1026" depends_on: - mongo - command: -dbhost mongo + command: -dbURI mongodb://mongo mongo: image: mongo:6.0 From 92e92896358aaa61e150581758862f5562e6713f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 26 Feb 2024 17:22:28 +0100 Subject: [PATCH 178/390] ADD exprLang --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/admin/database_model.md | 1 + doc/manuals/orion-api.md | 2 + src/lib/apiTypesV2/HttpInfo.cpp | 8 +- src/lib/apiTypesV2/HttpInfo.h | 2 + src/lib/apiTypesV2/MqttInfo.cpp | 8 +- src/lib/apiTypesV2/MqttInfo.h | 2 + src/lib/common/macroSubstitute.cpp | 10 +- src/lib/expressions/ExprContext.cpp | 187 +++-- src/lib/expressions/ExprContext.h | 13 +- src/lib/expressions/ExprManager.cpp | 58 +- src/lib/jsonParseV2/parseSubscription.cpp | 40 + .../mongoBackend/MongoCommonSubscription.cpp | 7 + src/lib/mongoBackend/dbConstants.h | 1 + src/lib/ngsiNotify/Notifier.cpp | 4 +- .../notification_templates_cache_refresh.test | 3 +- ...s_with_decoded_chars_in_notifications.test | 3 +- .../http_info_for_sub_update.test | 6 +- .../custom_url_validity_check.test | 3 +- ...tom_notification_http_json_basic_crud.test | 25 +- ...stom_notification_http_json_constants.test | 4 +- ...m_notification_http_json_replacements.test | 4 +- ...ication_http_json_replacements_inside.test | 4 +- ...n_http_json_replacements_inside_multi.test | 4 +- ...n_http_json_replacements_intermediate.test | 4 +- ...tom_notification_mqtt_json_basic_crud.test | 25 +- ...stom_notification_mqtt_json_constants.test | 6 +- ...m_notification_mqtt_json_replacements.test | 4 +- ...ication_mqtt_json_replacements_inside.test | 4 +- ...n_mqtt_json_replacements_inside_multi.test | 4 +- ...n_mqtt_json_replacements_intermediate.test | 4 +- .../self_notification_multi_hop.test | 9 +- .../self_notification_one_hop.test | 3 +- .../mqtt_custom_subscription_basic_crud.test | 8 +- .../mqtt_custom_subscription_update.test | 12 +- .../3001_mqtt/mqtt_notification_custom.test | 3 +- ...tification_problem_with_custom_header.test | 4 +- .../null_payload_get.test | 6 +- .../per_sub_httpTimeout_crud_custom.test | 15 +- ...r_sub_httpTimeout_notification_custom.test | 6 +- .../mqtt_custom_notifications_auth.test | 12 +- ...http_custom_exprlang_field_basic_crud.test | 664 +++++++++++++++++ .../jexl_expr_attrs_weird_syntax.test | 12 +- .../legacy_expr_attrs_weird_syntax.test | 216 ++++++ ...mqtt_custom_exprlang_field_basic_crud.test | 699 ++++++++++++++++++ ...m_notification_http_ngsi_attr_no_type.test | 3 +- ...tom_notification_http_ngsi_basic_crud.test | 25 +- ...cation_http_ngsi_basic_crud_compounds.test | 21 +- ...fication_http_ngsi_basic_crud_partial.test | 21 +- ...m_notification_mqtt_ngsi_attr_no_type.test | 3 +- ...tom_notification_mqtt_ngsi_basic_crud.test | 25 +- ...cation_mqtt_ngsi_basic_crud_compounds.test | 21 +- ...fication_mqtt_ngsi_basic_crud_partial.test | 21 +- .../mqttCustom_retain_basic_crud.test | 30 +- .../ngsi_patching_special_attr_types.test | 6 +- 55 files changed, 2091 insertions(+), 205 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index fdb575ad63..daa5db26e3 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ +- Add: exprLang field in custom notifications (#4004) - Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) - Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) diff --git a/doc/manuals/admin/database_model.md b/doc/manuals/admin/database_model.md index ae438dbef6..1c3ad517c6 100644 --- a/doc/manuals/admin/database_model.md +++ b/doc/manuals/admin/database_model.md @@ -329,6 +329,7 @@ notifications. It is a number between 0 and 1800000. If defined to 0 or omitted, in the Orion API. More detail of this functionality [here](../orion-api.md#ngsi-payload-patching). The value of this field is an object with a `attrs` key which value is a simplified version of `attrs` in [the entities collection](#entities-collection). +- **exprLang**: optional field to store the expression language. Only for custom subscriptions. If omitted `legacy` is assumed. - **lastFailure**: the time (as integer number, meaning seconds) when last notification failure occurred. Not present if the subscription has never failed. - **lastFailureReason**: text describing the cause of the last failure. diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 7b66992569..4fbc8d5ab0 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -3946,6 +3946,7 @@ A `httpCustom` object contains the following subfields. | `payload` | ✓ | string | Text-based payload to be used in notifications. In case of empty string or omitted, the default payload (see [Notification Messages](#notification-messages) sections) is used. If `null`, notification will not include any payload. | | `json` | ✓ | object | JSON-based payload to be used in notifications. See [JSON Payloads](#json-payloads) section for more details. | | `ngsi` | ✓ | object | NGSI patching for payload to be used in notifications. See [NGSI payload patching](#ngsi-payload-patching) section for more details. | +| `exprLang` | ✓ | string | Expression language to use, either `legacy` or `jexl` (default is `jexl`) | | `timeout` | ✓ | number | Maximum time (in milliseconds) the subscription waits for the response. The maximum value allowed for this parameter is 1800000 (30 minutes). If `timeout` is defined to 0 or omitted, then the value passed as `-httpTimeout` CLI parameter is used. See section in the [Command line options](admin/cli.md#command-line-options) for more details. | `payload`, `json` or `ngsi` cannot be used at the same time, they are mutually exclusive. @@ -3967,6 +3968,7 @@ A `mqttCustom` object contains the following subfields. | `payload` | ✓ | string | Text-based payload to be used in notifications. In case of empty string or omitted, the default payload (see [Notification Messages](#notification-messages) sections) is used. If `null`, notification will not include any payload. | | `json` | ✓ | object | JSON-based payload to be used in notifications. See [JSON Payloads](#json-payloads) section for more details. | | `ngsi` | ✓ | object | NGSI patching for payload to be used in notifications. See [NGSI payload patching](#ngsi-payload-patching) section for more details. | +| `exprLang` | ✓ | string | Expression language to use, either `legacy` or `jexl` (default is `jexl`) | `payload`, `json` or `ngsi` cannot be used at the same time, they are mutually exclusive. diff --git a/src/lib/apiTypesV2/HttpInfo.cpp b/src/lib/apiTypesV2/HttpInfo.cpp index 4f5e1fd80e..cc4c000283 100644 --- a/src/lib/apiTypesV2/HttpInfo.cpp +++ b/src/lib/apiTypesV2/HttpInfo.cpp @@ -106,6 +106,8 @@ std::string HttpInfo::toJson() { jh.addRaw("headers", objectToJson(headers)); } + + jh.addString("exprLang", this->exprLang); } return jh.str(); @@ -131,7 +133,7 @@ void HttpInfo::fill(const orion::BSONObj& bo) if (bo.hasField(CSUB_NGSI)) n++; if (n > 1) { - LM_E(("custom notification must not have more than one payload related field")); + LM_E(("Runtime Error (custom notification must not have more than one payload related field)")); return; } @@ -232,6 +234,9 @@ void HttpInfo::fill(const orion::BSONObj& bo) orion::BSONObj headers = getObjectFieldF(bo, CSUB_HEADERS); headers.toStringMap(&this->headers); } + + // expression language used in custom notifications + this->exprLang = bo.hasField(CSUB_EXPRLANG) ? getStringFieldF(bo, CSUB_EXPRLANG) : "legacy"; } } @@ -252,6 +257,7 @@ void HttpInfo::fill(const HttpInfo& _httpInfo) this->custom = _httpInfo.custom; this->includePayload = _httpInfo.includePayload; this->timeout = _httpInfo.timeout; + this->exprLang = _httpInfo.exprLang; this->json = _httpInfo.json == NULL? NULL : _httpInfo.json->clone(); diff --git a/src/lib/apiTypesV2/HttpInfo.h b/src/lib/apiTypesV2/HttpInfo.h index 0af3d273ef..8d32651374 100644 --- a/src/lib/apiTypesV2/HttpInfo.h +++ b/src/lib/apiTypesV2/HttpInfo.h @@ -57,6 +57,8 @@ struct HttpInfo bool includePayload; long long timeout; + std::string exprLang; + HttpInfo(); std::string toJson(); diff --git a/src/lib/apiTypesV2/MqttInfo.cpp b/src/lib/apiTypesV2/MqttInfo.cpp index d46b9e4507..c8829b0bb2 100644 --- a/src/lib/apiTypesV2/MqttInfo.cpp +++ b/src/lib/apiTypesV2/MqttInfo.cpp @@ -102,6 +102,8 @@ std::string MqttInfo::toJson() jh.addRaw("ngsi", this->ngsi.toJson(NGSI_V2_NORMALIZED, true)); break; } + + jh.addString("exprLang", this->exprLang); } return jh.str(); @@ -143,7 +145,7 @@ void MqttInfo::fill(const orion::BSONObj& bo) if (bo.hasField(CSUB_NGSI)) n++; if (n > 1) { - LM_E(("custom notification must not have more than one payload related field")); + LM_E(("Runtime Error (custom notification must not have more than one payload related field)")); return; } @@ -216,6 +218,9 @@ void MqttInfo::fill(const orion::BSONObj& bo) this->ngsi.attributeVector.fill(getObjectFieldF(ngsiObj, ENT_ATTRS)); } } + + // expression language used in custom notifications + this->exprLang = bo.hasField(CSUB_EXPRLANG) ? getStringFieldF(bo, CSUB_EXPRLANG) : "legacy"; } } @@ -238,6 +243,7 @@ void MqttInfo::fill(const MqttInfo& _mqttInfo) this->providedAuth = _mqttInfo.providedAuth; this->user = _mqttInfo.user; this->passwd = _mqttInfo.passwd; + this->exprLang = _mqttInfo.exprLang; this->json = _mqttInfo.json == NULL ? NULL : _mqttInfo.json->clone(); diff --git a/src/lib/apiTypesV2/MqttInfo.h b/src/lib/apiTypesV2/MqttInfo.h index 738e73c6a9..bbc3b9260d 100644 --- a/src/lib/apiTypesV2/MqttInfo.h +++ b/src/lib/apiTypesV2/MqttInfo.h @@ -53,6 +53,8 @@ struct MqttInfo CustomPayloadType payloadType; bool includePayload; + std::string exprLang; + bool providedAuth; std::string user; std::string passwd; diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 3c0fc7e105..e1619ca77c 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -60,7 +60,15 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e } else { - return r.toString(); + // in legacy mode an extra remove quotes step is needed + if (exprContextObjectP->isLegacy()) + { + return removeQuotes(r.toString()); + } + else + { + return r.toString(); + } } } else if (exprContextObjectP != NULL) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 41ec5b3ddb..87f1de9f9d 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -25,6 +25,7 @@ #include +#include "common/string.h" #include "logMsg/logMsg.h" #include "expressions/ExprContext.h" #include "expressions/exprCommon.h" @@ -35,15 +36,20 @@ * * ExprContextObject::ExprContextObject - */ -ExprContextObject::ExprContextObject() +ExprContextObject::ExprContextObject(bool _legacy) { - jexlContext = PyDict_New(); + legacy = _legacy; - if (jexlContext == NULL) + if (!legacy) { - // Note that this ruins the object. We log this situation just one time here, but we include a NULL check - // in every other method to be safer - LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); + jexlContext = PyDict_New(); + + if (jexlContext == NULL) + { + // Note that this ruins the object. We log this situation just one time here, but we include a NULL check + // in every other method to be safer + LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); + } } } @@ -51,34 +57,52 @@ ExprContextObject::ExprContextObject() /* **************************************************************************** * -* ExprContextObject::get - +* ExprContextObject::getJexlContext - */ -PyObject* ExprContextObject::get(void) +PyObject* ExprContextObject::getJexlContext(void) { return jexlContext; } +/* **************************************************************************** +* +* ExprContextObject::getJexlContext - +*/ +std::map* ExprContextObject::getMap(void) +{ + return &repl; +} + + + /* **************************************************************************** * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, const std::string& _value) +void ExprContextObject::add(const std::string &key, const std::string &_value) { - if (jexlContext == NULL) + if (legacy) { - return; + repl.insert(std::pair(key, _value)); } - LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - if (value == NULL) + else { - LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); - return; + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); + PyObject* value = Py_BuildValue("s", _value.c_str()); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); + return; + } + PyDict_SetItemString(jexlContext, key.c_str(), value); + Py_DECREF(value); } - PyDict_SetItemString(jexlContext, key.c_str(), value); - Py_DECREF(value); } @@ -87,44 +111,57 @@ void ExprContextObject::add(const std::string& key, const std::string& _value) * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, double _value) +void ExprContextObject::add(const std::string &key, double _value) { - if (jexlContext == NULL) + if (legacy) { - return; + repl.insert(std::pair(key, double2string(_value))); } - LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); - PyObject* value = Py_BuildValue("d", _value); - if (value == NULL) + else { - LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); - return; + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); + PyObject* value = Py_BuildValue("d", _value); + if (value == NULL) + { + LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); + return; + } + PyDict_SetItemString(jexlContext, key.c_str(), value); + Py_DECREF(value); } - PyDict_SetItemString(jexlContext, key.c_str(), value); - Py_DECREF(value); } - /* **************************************************************************** * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, bool _value) +void ExprContextObject::add(const std::string &key, bool _value) { - if (jexlContext == NULL) + if (legacy) { - return; - } - LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value? "true" : "false")); - if (_value) - { - PyDict_SetItemString(jexlContext, key.c_str(), Py_True); + repl.insert(std::pair(key, _value? "true": "false")); } else { - PyDict_SetItemString(jexlContext, key.c_str(), Py_False); + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); + if (_value) + { + PyDict_SetItemString(jexlContext, key.c_str(), Py_True); + } + else + { + PyDict_SetItemString(jexlContext, key.c_str(), Py_False); + } } } @@ -134,14 +171,21 @@ void ExprContextObject::add(const std::string& key, bool _value) * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key) +void ExprContextObject::add(const std::string &key) { - if (jexlContext == NULL) + if (legacy) { - return; + repl.insert(std::pair(key, "null")); + } + else + { + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), Py_None); } - LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), Py_None); } @@ -150,14 +194,22 @@ void ExprContextObject::add(const std::string& key) * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, ExprContextObject exprContextObject) +void ExprContextObject::add(const std::string &key, ExprContextObject exprContextObject) { - if (jexlContext == NULL) + if (legacy) { - return; + // FIXME PR: not sure if this is going to work... + repl.insert(std::pair(key, exprContextObject.toString())); + } + else + { + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (object): %s=%s", key.c_str(), exprContextObject.toString().c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.getJexlContext()); } - LM_T(LmtExpr, ("adding to expression context object (object): %s=%s", key.c_str(), exprContextObject.toString().c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.get()); } @@ -166,18 +218,36 @@ void ExprContextObject::add(const std::string& key, ExprContextObject exprContex * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string& key, ExprContextList exprContextList) +void ExprContextObject::add(const std::string &key, ExprContextList exprContextList) { - if (jexlContext == NULL) + if (legacy) { - return; + // FIXME PR: not sure if this is going to work... + repl.insert(std::pair(key, exprContextList.toString())); + } + else + { + if (jexlContext == NULL) + { + return; + } + LM_T(LmtExpr, ("adding to expression context object (list): %s=%s", key.c_str(), exprContextList.toString().c_str())); + PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); } - LM_T(LmtExpr, ("adding to expression context object (list): %s=%s", key.c_str(), exprContextList.toString().c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); } +/* **************************************************************************** +* +* ExprContextObject::isLegacy - +*/ +bool ExprContextObject::isLegacy(void) +{ + return legacy; +} + + /* **************************************************************************** * * ExprContextObject::toString - @@ -247,7 +317,7 @@ PyObject* ExprContextList::get(void) * * ExprContextList::add - */ -void ExprContextList::add(const std::string& _value) +void ExprContextList::add(const std::string &_value) { if (jexlContext == NULL) { @@ -255,7 +325,7 @@ void ExprContextList::add(const std::string& _value) } LM_T(LmtExpr, ("adding to expression context list (string): %s=", _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); - if (value == NULL) + if (value == NULL) { LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); return; @@ -278,7 +348,7 @@ void ExprContextList::add(double _value) } LM_T(LmtExpr, ("adding to expression context list (double): %f", _value)); PyObject* value = Py_BuildValue("d", _value); - if (value == NULL) + if (value == NULL) { LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); return; @@ -289,7 +359,6 @@ void ExprContextList::add(double _value) - /* **************************************************************************** * * ExprContextList::add - @@ -300,7 +369,7 @@ void ExprContextList::add(bool _value) { return; } - LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value? "true" : "false")); + LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value ? "true" : "false")); if (_value) { PyList_Append(jexlContext, Py_True); @@ -340,7 +409,7 @@ void ExprContextList::add(ExprContextObject exprContextObject) return; } LM_T(LmtExpr, ("adding to expression context list (object): %s", exprContextObject.toString().c_str())); - PyList_Append(jexlContext, exprContextObject.get()); + PyList_Append(jexlContext, exprContextObject.getJexlContext()); } diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 6b41dc7bca..19c165f276 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -28,6 +28,7 @@ #include #include +#include class ExprContextList; // forward declaration @@ -38,12 +39,16 @@ class ExprContextList; // forward declaration class ExprContextObject { private: - PyObject* jexlContext; + bool legacy; + PyObject* jexlContext; // used in regular (i.e. not legacy) mode + std::map repl; // used in legacy mode public: - ExprContextObject(); + ExprContextObject(bool legacy = false); + + PyObject* getJexlContext(void); + std::map* getMap(void); - PyObject* get(void); void add(const std::string& key, const std::string& value); void add(const std::string& key, double value); void add(const std::string& key, bool value); @@ -51,6 +56,8 @@ class ExprContextObject void add(const std::string& key, ExprContextObject exprContextObject); void add(const std::string& key, ExprContextList exprContextList); + bool isLegacy(void); + std::string toString(void); void release(void); diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index d6aa98d5bd..bfe6d14b81 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -54,16 +54,14 @@ void ExprManager::init(void) pyjexlModule = PyImport_ImportModule("pyjexl"); if (pyjexlModule == NULL) { - const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", error)); + LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", capturePythonError())); } LM_T(LmtExpr, ("pyjexl module has been loaded")); jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); if (jexlEngine == NULL) { - const char* error = capturePythonError(); - LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", error)); + LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", capturePythonError())); } LM_T(LmtExpr, ("jexl engine has been created")); } @@ -79,28 +77,46 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st ExprResult r; r.valueType = orion::ValueTypeNull; - LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - - PyObject* expression = Py_BuildValue("s", _expression.c_str()); - if (expression == NULL) + if (exprContextObjectP->isLegacy()) { - LM_W(("error building JEXL expression: %s", capturePythonError())); - return r; - } + // std::map based evaluation. Only pure replacement is supported + LM_T(LmtExpr, ("evaluating legacy expresion: <%s>", _expression.c_str())); + + std::map* replacementsP = exprContextObjectP->getMap(); - PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, exprContextObjectP->get()); - Py_XDECREF(expression); - if (result == NULL) + std::map::iterator iter = replacementsP->find(_expression); + if (iter != replacementsP->end()) + { + r.valueType = orion::ValueTypeString; + r.stringValue = iter->second; + } + } + else { - LM_W(("error evaluating JEXL expression: %s", capturePythonError())); - return r; + // JEXL based evaluation + LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); + + PyObject* expression = Py_BuildValue("s", _expression.c_str()); + if (expression == NULL) + { + LM_W(("error building JEXL expression: %s", capturePythonError())); + return r; + } + + PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, exprContextObjectP->getJexlContext()); + Py_XDECREF(expression); + if (result == NULL) + { + LM_W(("error evaluating JEXL expression: %s", capturePythonError())); + return r; + } + + r.fill(result); + + // FIXME PR: does this Py_XDECREF() recursively in the case of dicts or lists? + Py_XDECREF(result); } - r.fill(result); - - // FIXME PR: does this Py_XDECREF() recursively in the case of dicts or lists? - Py_XDECREF(result); - return r; } diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index 4207dfa023..cea25d5b52 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -1031,6 +1031,26 @@ static std::string parseNotification(ConnectionInfo* ciP, SubscriptionUpdate* su return r; } + // exprLang + Opt exprLangOpt = getStringOpt(httpCustom, "exprLang", "exprLang httpCustom notification"); + std::string exprLang = "jexl"; + + if (!exprLangOpt.ok()) + { + return badInput(ciP, exprLangOpt.error); + } + + if (exprLangOpt.given) + { + exprLang = exprLangOpt.value; + if ((exprLang != "jexl") && (exprLang != "legacy")) + { + return badInput(ciP, "not valid exprLang, valid ones are jexl or legacy"); + } + } + + subsP->notification.httpInfo.exprLang = exprLang; + subsP->notification.httpInfo.custom = true; } else if (notification.HasMember("mqtt")) @@ -1141,6 +1161,26 @@ static std::string parseNotification(ConnectionInfo* ciP, SubscriptionUpdate* su return r; } + // exprLang + Opt exprLangOpt = getStringOpt(mqttCustom, "exprLang", "exprLang mqttCustom notification"); + std::string exprLang = "jexl"; + + if (!exprLangOpt.ok()) + { + return badInput(ciP, exprLangOpt.error); + } + + if (exprLangOpt.given) + { + exprLang = exprLangOpt.value; + if ((exprLang != "jexl") && (exprLang != "legacy")) + { + return badInput(ciP, "not valid exprLang, valid ones are jexl or legacy"); + } + } + + subsP->notification.mqttInfo.exprLang = exprLang; + subsP->notification.mqttInfo.custom = true; } diff --git a/src/lib/mongoBackend/MongoCommonSubscription.cpp b/src/lib/mongoBackend/MongoCommonSubscription.cpp index dd6bcf3101..a58ea5f55b 100644 --- a/src/lib/mongoBackend/MongoCommonSubscription.cpp +++ b/src/lib/mongoBackend/MongoCommonSubscription.cpp @@ -183,6 +183,9 @@ static void setCustomHttpInfo(const HttpInfo& httpInfo, orion::BSONObjBuilder* b b->append(CSUB_NGSI, bob.obj()); } + + b->append(CSUB_EXPRLANG, httpInfo.exprLang); + LM_T(LmtMongo, ("Subscription exprLang: %s", httpInfo.exprLang.c_str())); } @@ -255,6 +258,9 @@ static void setCustomMqttInfo(const ngsiv2::MqttInfo& mqttInfo, orion::BSONObjBu b->append(CSUB_NGSI, bob.obj()); } + + b->append(CSUB_EXPRLANG, mqttInfo.exprLang); + LM_T(LmtMongo, ("Subscription exprLang: %s", mqttInfo.exprLang.c_str())); } @@ -626,6 +632,7 @@ void setOnlyChanged(const Subscription& sub, orion::BSONObjBuilder* b) } + /* **************************************************************************** * * setCovered - diff --git a/src/lib/mongoBackend/dbConstants.h b/src/lib/mongoBackend/dbConstants.h index b55cf29925..241507efd9 100644 --- a/src/lib/mongoBackend/dbConstants.h +++ b/src/lib/mongoBackend/dbConstants.h @@ -119,6 +119,7 @@ #define CSUB_NGSI "ngsi" #define CSUB_BLACKLIST "blacklist" #define CSUB_ONLYCHANGED "onlyChanged" +#define CSUB_EXPRLANG "exprLang" #define CSUB_COVERED "covered" #define CSUB_NOTIFYONMETADATACHANGE "notifyOnMetadataChange" #define CSUB_LASTFAILURE "lastFailure" diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index aedcaa1b21..3f8516798f 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -315,8 +315,10 @@ static SenderThreadParams* buildSenderParamsCustom std::map headers; Entity& en = notifyCerP->entity; + const std::string exprLang = notification.type == ngsiv2::HttpNotification ? notification.httpInfo.exprLang : notification.mqttInfo.exprLang; + // Used by several macroSubstitute() calls along this function - ExprContextObject exprContext; + ExprContextObject exprContext(exprLang == "legacy"); exprContext.add("id", en.id); exprContext.add("type", en.type); exprContext.add("service", tenant); diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test index 853d3e6abd..f1b0c00cbb 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test @@ -157,7 +157,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 695 +Content-Length: 713 [ { @@ -167,6 +167,7 @@ Content-Length: 695 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "A1": "${A1}", "A2": "${A2}", diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test index 9a548cec40..a31092de4a 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test @@ -137,7 +137,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 550 +Content-Length: 568 { "id": "REGEX([0-9a-f]{24})", @@ -146,6 +146,7 @@ Content-Length: 550 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "hs1": "1", "hs2": "${A1}" diff --git a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test index 7a4956a172..2bdcc9c72c 100644 --- a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test +++ b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test @@ -173,7 +173,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 495 +Content-Length: 513 [ { @@ -183,6 +183,7 @@ Content-Length: 495 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-Type": "application/json", "H1": "${A1}" @@ -231,7 +232,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 497 +Content-Length: 515 [ { @@ -241,6 +242,7 @@ Content-Length: 497 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-Type": "application/json", "H2": "${A2}" diff --git a/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test b/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test index 57a7a22c99..5969637f5d 100644 --- a/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test +++ b/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test @@ -142,7 +142,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 286 +Content-Length: 304 [ { @@ -152,6 +152,7 @@ Content-Length: 286 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "method": "PUT", "url": "${abc}" }, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test index c50d612856..89828afff7 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test @@ -254,7 +254,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 325 +Content-Length: 343 { "id": "REGEX([0-9a-f]{24})", @@ -263,6 +263,7 @@ Content-Length: 325 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "x1": 1, "x2": 2 @@ -301,7 +302,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -310,6 +311,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -345,7 +347,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 321 +Content-Length: 339 { "id": "REGEX([0-9a-f]{24})", @@ -354,6 +356,7 @@ Content-Length: 321 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "yy", "zz" @@ -392,7 +395,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 317 +Content-Length: 335 { "id": "REGEX([0-9a-f]{24})", @@ -401,6 +404,7 @@ Content-Length: 317 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ 1, 2, @@ -440,7 +444,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -449,6 +453,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -486,7 +491,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -495,6 +500,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -530,7 +536,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 334 +Content-Length: 352 { "id": "REGEX([0-9a-f]{24})", @@ -539,6 +545,7 @@ Content-Length: 334 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "z4": true, "z5": [ @@ -572,7 +579,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 655 +Content-Length: 691 [ { @@ -582,6 +589,7 @@ Content-Length: 655 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -608,6 +616,7 @@ Content-Length: 655 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "z4": true, "z5": [ diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test index 2bb34d3d32..d49ac8a108 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test @@ -180,7 +180,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 880 +Content-Length: 916 [ { @@ -190,6 +190,7 @@ Content-Length: 880 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "array": [ "22", @@ -245,6 +246,7 @@ Content-Length: 880 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "foo", 10, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test index 08f26a3682..e0a2d267ec 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 802 +Content-Length: 838 [ { @@ -197,6 +197,7 @@ Content-Length: 802 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "array": "${array}", "bool": "${bool}", @@ -230,6 +231,7 @@ Content-Length: 802 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "${text}", "${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test index 758775b4e3..e1b86b3f9f 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 815 +Content-Length: 851 [ { @@ -197,6 +197,7 @@ Content-Length: 815 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "A": "ns:${text}", "B": "ns:${number}", @@ -230,6 +231,7 @@ Content-Length: 815 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "ns:${text}", "ns:${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test index 95f7928e31..191994521c 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 921 +Content-Length: 957 [ { @@ -197,6 +197,7 @@ Content-Length: 921 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "A": "ns:${text}:${object}", "B": "ns:${number}:${array}", @@ -230,6 +231,7 @@ Content-Length: 921 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "ns:${text}:${object}", "ns:${number}:${array}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test index 7a19421ab8..568d36fc19 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test @@ -157,7 +157,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 715 +Content-Length: 751 [ { @@ -167,6 +167,7 @@ Content-Length: 715 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": { "x": { "x1": "${A}", @@ -202,6 +203,7 @@ Content-Length: 715 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "json": [ "22", { diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test index 0c24073d89..dccebf8230 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test @@ -261,7 +261,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 358 +Content-Length: 376 { "id": "REGEX([0-9a-f]{24})", @@ -270,6 +270,7 @@ Content-Length: 358 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "x1": 1, "x2": 2 @@ -311,7 +312,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -320,6 +321,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -358,7 +360,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 354 +Content-Length: 372 { "id": "REGEX([0-9a-f]{24})", @@ -367,6 +369,7 @@ Content-Length: 354 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "yy", "zz" @@ -408,7 +411,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 350 +Content-Length: 368 { "id": "REGEX([0-9a-f]{24})", @@ -417,6 +420,7 @@ Content-Length: 350 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ 1, 2, @@ -459,7 +463,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -468,6 +472,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -508,7 +513,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -517,6 +522,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -555,7 +561,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 367 +Content-Length: 385 { "id": "REGEX([0-9a-f]{24})", @@ -564,6 +570,7 @@ Content-Length: 367 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "z4": true, "z5": [ @@ -600,7 +607,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 721 +Content-Length: 757 [ { @@ -610,6 +617,7 @@ Content-Length: 721 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -639,6 +647,7 @@ Content-Length: 721 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "z4": true, "z5": [ diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test index d8fe2d436c..e84b665130 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test @@ -186,7 +186,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 946 +Content-Length: 982 [ { @@ -196,6 +196,7 @@ Content-Length: 946 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "array": [ "22", @@ -223,7 +224,7 @@ Content-Length: 946 "y1", "y2" ] - }, + }, "text": "foo" }, "qos": 0, @@ -254,6 +255,7 @@ Content-Length: 946 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "foo", 10, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test index 954ed3bdba..993180a8ce 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test @@ -193,7 +193,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 868 +Content-Length: 904 [ { @@ -203,6 +203,7 @@ Content-Length: 868 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "array": "${array}", "bool": "${bool}", @@ -239,6 +240,7 @@ Content-Length: 868 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "${text}", "${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test index 3a6f4a15c5..7133b2e675 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test @@ -192,7 +192,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 881 +Content-Length: 917 [ { @@ -202,6 +202,7 @@ Content-Length: 881 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "A": "ns:${text}", "B": "ns:${number}", @@ -238,6 +239,7 @@ Content-Length: 881 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "ns:${text}", "ns:${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test index 940492dc04..58c287e5d5 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test @@ -193,7 +193,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 987 +Content-Length: 1023 [ { @@ -203,6 +203,7 @@ Content-Length: 987 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "A": "ns:${text}:${object}", "B": "ns:${number}:${array}", @@ -239,6 +240,7 @@ Content-Length: 987 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "ns:${text}:${object}", "ns:${number}:${array}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test index 020245e27b..2fc44f8b6b 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test @@ -163,7 +163,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 781 +Content-Length: 817 [ { @@ -173,6 +173,7 @@ Content-Length: 781 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": { "x": { "x1": "${A}", @@ -211,6 +212,7 @@ Content-Length: 781 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "json": [ "22", { diff --git a/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test b/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test index 2fe079ec78..6fa972f30b 100644 --- a/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test +++ b/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test @@ -373,7 +373,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 564 +Content-Length: 582 { "id": "REGEX([0-9a-f]{24})", @@ -382,6 +382,7 @@ Content-Length: 564 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-type": "application/json" }, @@ -415,7 +416,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 564 +Content-Length: 582 { "id": "REGEX([0-9a-f]{24})", @@ -424,6 +425,7 @@ Content-Length: 564 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-type": "application/json" }, @@ -457,7 +459,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 564 +Content-Length: 582 { "id": "REGEX([0-9a-f]{24})", @@ -466,6 +468,7 @@ Content-Length: 564 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-type": "application/json" }, diff --git a/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test b/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test index ffaffdb044..12727d8b09 100644 --- a/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test +++ b/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test @@ -128,7 +128,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 566 +Content-Length: 584 [ { @@ -138,6 +138,7 @@ Content-Length: 566 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "headers": { "Content-type": "application/json" }, diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test index d482101617..36ca87149f 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test @@ -282,7 +282,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 2428 +Content-Length: 2536 [ { @@ -295,6 +295,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "temperature:${temperature}", "qos": 0, "retain": false, @@ -327,6 +328,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "temperature:${temperature}", "qos": 2, "retain": false, @@ -359,6 +361,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "/orion/${dest}", @@ -390,6 +393,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 2, "retain": false, "topic": "/orion/${dest}", @@ -421,6 +425,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": null, "qos": 0, "retain": false, @@ -453,6 +458,7 @@ Content-Length: 2428 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": null, "qos": 2, "retain": false, diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test index 531e251861..1d65654747 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test @@ -204,7 +204,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 389 +Content-Length: 407 { "description": "Original sub", @@ -214,6 +214,7 @@ Content-Length: 389 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "t:${t}", "qos": 2, "retain": false, @@ -252,7 +253,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 389 +Content-Length: 407 { "description": "Modified sub", @@ -262,6 +263,7 @@ Content-Length: 389 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "t:${t}", "qos": 2, "retain": false, @@ -300,7 +302,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 382 +Content-Length: 400 { "description": "Modified sub", @@ -310,6 +312,7 @@ Content-Length: 382 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": null, "qos": 0, "retain": false, @@ -392,7 +395,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 372 +Content-Length: 390 { "description": "Modified sub", @@ -402,6 +405,7 @@ Content-Length: 372 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 1, "retain": false, "topic": "/orionbk/${d}", diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test b/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test index ecc29badba..597cfa5457 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test @@ -245,7 +245,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 509 +Content-Length: 527 [ { @@ -258,6 +258,7 @@ Content-Length: 509 "lastNotification": "REGEX(.*)", "lastSuccess": "REGEX(.*)", "mqttCustom": { + "exprLang": "jexl", "payload": "{ %22A%22: %22${A}%22 }", "qos": 0, "retain": false, diff --git a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test b/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test index b4e9cc5695..9e63211726 100644 --- a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test +++ b/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test @@ -159,7 +159,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1012 +Content-Length: 1048 [ { @@ -171,6 +171,7 @@ Content-Length: 1012 "attrsFormat": "legacy", "covered": false, "httpCustom": { + "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "lastNotification": "REGEX(.*)", @@ -201,6 +202,7 @@ Content-Length: 1012 "attrsFormat": "legacy", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "{ %22A1%22: %22Value of A1: ${A1}%22 }", "url": "http://localhost:9997/notify" }, diff --git a/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test b/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test index 0955bd6c86..13e3163639 100644 --- a/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test +++ b/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test @@ -189,7 +189,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1200 +Content-Length: 1272 [ { @@ -199,6 +199,7 @@ Content-Length: 1200 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "abc", "url": "http://localhost:9997/notify" }, @@ -224,6 +225,7 @@ Content-Length: 1200 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "onlyChangedAttrs": false @@ -248,6 +250,7 @@ Content-Length: 1200 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": null, "url": "http://localhost:9997/notify" }, @@ -273,6 +276,7 @@ Content-Length: 1200 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "onlyChangedAttrs": false diff --git a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test index d874dbefdd..badbe98993 100644 --- a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test +++ b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test @@ -327,7 +327,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 379 { "description": "HTTP sub", @@ -337,6 +337,7 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 1000, "url": "http://localhost:1234" @@ -373,7 +374,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 379 { "description": "HTTP sub", @@ -383,6 +384,7 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 2000, "url": "http://localhost:1234" @@ -419,7 +421,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 346 +Content-Length: 364 { "description": "HTTP sub", @@ -429,6 +431,7 @@ Content-Length: 346 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "url": "http://localhost:1234" }, @@ -464,7 +467,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 379 { "description": "HTTP sub", @@ -474,6 +477,7 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 1000, "url": "http://localhost:1234" @@ -520,7 +524,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 346 +Content-Length: 364 { "description": "HTTP sub", @@ -530,6 +534,7 @@ Content-Length: 346 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "url": "http://localhost:1234" }, diff --git a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test index 0937cb5cec..bbdcdaebb0 100644 --- a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test +++ b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test @@ -175,7 +175,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 496 +Content-Length: 514 { "description": "HTTP sub", @@ -185,6 +185,7 @@ Content-Length: 496 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 12000, "url": "http://localhost:9997/noresponse" @@ -217,7 +218,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 532 +Content-Length: 550 { "description": "HTTP sub", @@ -228,6 +229,7 @@ Content-Length: 532 "covered": false, "failsCounter": 1, "httpCustom": { + "exprLang": "jexl", "payload": "Custom payload", "timeout": 8000, "url": "http://localhost:9997/noresponse" diff --git a/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test b/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test index 59a65b45f0..c50332dc8f 100644 --- a/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test +++ b/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test @@ -317,7 +317,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 366 +Content-Length: 384 { "id": "REGEX([0-9a-f\-]{24})", @@ -326,6 +326,7 @@ Content-Length: 366 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, @@ -371,7 +372,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 367 +Content-Length: 385 { "id": "REGEX([0-9a-f\-]{24})", @@ -380,6 +381,7 @@ Content-Length: 367 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, @@ -425,7 +427,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 334 +Content-Length: 352 { "id": "REGEX([0-9a-f\-]{24})", @@ -434,6 +436,7 @@ Content-Length: 334 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "/orion", @@ -477,7 +480,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 366 +Content-Length: 384 { "id": "REGEX([0-9a-f\-]{24})", @@ -486,6 +489,7 @@ Content-Length: 366 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test new file mode 100644 index 0000000000..656e55da9b --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test @@ -0,0 +1,664 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Basic CRUD for exprLang field (http variant) + +--SHELL-INIT-- +dbInit CB +brokerStart CB 0-255 + +--SHELL-- + +# +# 01. Create subscription with invalid exprLang, see error +# 02. Create subscription with exprLang jexl +# 03a. Get subscription and see exprLang jexl +# 03b. Get subscription (as a list) and see exprLang jexl +# 04. Update subscription with exprLang legacy +# 05a. Get subscription and see exprLang legacy +# 05b. Get subscription (as a list) and see exprLang legacy +# 06. Update subscription without exprLang +# 07a. Get subscription and see exprLang jexl +# 07b. Get subscription (as a list) and see exprLang jexl +# 08. Update subscription with exprLang legacy +# 09a. Get subscription and see exprLang legacy +# 09b. Get subscription (as a list) and see exprLang legacy +# 10. Update subscription with invalid exprLang, see error +# 11a. Get subscription and see exprLang legacy +# 11b. Get subscription (as a list) and see exprLang legacy +# + +echo "01. Create subscription with invalid exprLang, see error" +echo "========================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": "foo" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create subscription with exprLang jexl" +echo "==========================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": "jexl" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "03a. Get subscription and see exprLang jexl" +echo "===========================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "03b. Get subscription (as a list) and see exprLang jexl" +echo "=======================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "04. Update subscription with exprLang legacy" +echo "============================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "05a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "05b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "06. Update subscription without exprLang" +echo "========================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "topic": "orion" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "07a. Get subscription and see exprLang jexl" +echo "===========================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "07b. Get subscription (as a list) and see exprLang jexl" +echo "=======================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "08. Update subscription with exprLang legacy" +echo "============================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "09a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "09b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "10. Update subscription with invalid exprLang, see error" +echo "========================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://localhost:9998", + "exprLang": 42 + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "11a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "11b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + +--REGEXPECT-- +01. Create subscription with invalid exprLang, see error +======================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 88 + +{ + "description": "not valid exprLang, valid ones are jexl or legacy", + "error": "BadRequest" +} + + +02. Create subscription with exprLang jexl +========================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03a. Get subscription and see exprLang jexl +=========================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 320 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "jexl", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +03b. Get subscription (as a list) and see exprLang jexl +======================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "jexl", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +04. Update subscription with exprLang legacy +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +05b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 324 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +06. Update subscription without exprLang +======================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +07a. Get subscription and see exprLang jexl +=========================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 320 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "jexl", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +07b. Get subscription (as a list) and see exprLang jexl +======================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "jexl", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +08. Update subscription with exprLang legacy +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +09a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +09b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 324 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +10. Update subscription with invalid exprLang, see error +======================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 87 + +{ + "description": "exprLang httpCustom notification is not a string", + "error": "BadRequest" +} + + +11a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 322 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +11b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 324 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "exprLang": "legacy", + "url": "http://localhost:9998" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +--TEARDOWN-- +brokerStop CB +dbDrop CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test index 3390b28e82..cf42ab7ebe 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -31,15 +31,17 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with custom expression: A-B and A:B +# Same as legacy_expr_attrs_weird_syntax.test, but using jexl expression language +# +# 01. 01. Create custom sub with custom expression: R1=A-B and R2=A:B # 02. Create entity E1 with A-B 1 and A:B 2 # 03. Update entity E1 with A-B 3 and A:B 4 # 04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null) # -echo "01. Create custom sub with custom expression: A-B and A:B" -echo "=========================================================" +echo "01. Create custom sub with custom expression: R1=A-B and R2=A:B" +echo "===============================================================" payload='{ "subject": { "entities": [ @@ -115,8 +117,8 @@ echo --REGEXPECT-- -01. Create custom sub with custom expression: A-B and A:B -========================================================= +01. Create custom sub with custom expression: R1=A-B and R2=A:B +=============================================================== HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test new file mode 100644 index 0000000000..15da6fc5bb --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test @@ -0,0 +1,216 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Legacy expression using attributes with weird syntax + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# Same as jexl_expr_attrs_weird_syntax.test, but using legacy expression language +# +# 01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B +# 02. Create entity E1 with A-B 1 and A:B 2 +# 03. Update entity E1 with A-B 3 and A:B 4 +# 04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4) +# + + +echo "01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B" +echo "======================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "exprLang": "legacy", + "ngsi": { + "R1": { + "value": "${A-B}", + "type": "Calculated" + }, + "R2": { + "value": "${A:B}", + "type": "Calculated" + } + } + }, + "attrs": [ "R1", "R2" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A-B 1 and A:B 2" +echo "=========================================" +payload='{ + "id": "E1", + "type": "T", + "A-B": { + "value": 1, + "type": "Number" + }, + "A:B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A-B 3 and A:B 4" +echo "=========================================" +payload='{ + "A-B": { + "value": 3, + "type": "Number" + }, + "A:B": { + "value": 4, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4)" +echo "======================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B +====================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A-B 1 and A:B 2 +========================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A-B 3 and A:B 4 +========================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4) +====================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 179 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "R1": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "R2": { + "metadata": {}, + "type": "Calculated", + "value": 2 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 179 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "R1": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "R2": { + "metadata": {}, + "type": "Calculated", + "value": 4 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test new file mode 100644 index 0000000000..b155766fea --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test @@ -0,0 +1,699 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Basic CRUD for exprLang field (mqtt variant) + +--SHELL-INIT-- +dbInit CB +brokerStart CB 0-255 + +--SHELL-- + +# +# 01. Create subscription with invalid exprLang, see error +# 02. Create subscription with exprLang jexl +# 03a. Get subscription and see exprLang jexl +# 03b. Get subscription (as a list) and see exprLang jexl +# 04. Update subscription with exprLang legacy +# 05a. Get subscription and see exprLang legacy +# 05b. Get subscription (as a list) and see exprLang legacy +# 06. Update subscription without exprLang +# 07a. Get subscription and see exprLang jexl +# 07b. Get subscription (as a list) and see exprLang jexl +# 08. Update subscription with exprLang legacy +# 09a. Get subscription and see exprLang legacy +# 09b. Get subscription (as a list) and see exprLang legacy +# 10. Update subscription with invalid exprLang, see error +# 11a. Get subscription and see exprLang legacy +# 11b. Get subscription (as a list) and see exprLang legacy +# + +echo "01. Create subscription with invalid exprLang, see error" +echo "========================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": "foo" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create subscription with exprLang jexl" +echo "==========================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": "jexl" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "03a. Get subscription and see exprLang jexl" +echo "===========================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "03b. Get subscription (as a list) and see exprLang jexl" +echo "=======================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "04. Update subscription with exprLang legacy" +echo "============================================" +payload='{ + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "05a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "05b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "06. Update subscription without exprLang" +echo "========================================" +payload='{ + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "07a. Get subscription and see exprLang jexl" +echo "===========================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "07b. Get subscription (as a list) and see exprLang jexl" +echo "=======================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "08. Update subscription with exprLang legacy" +echo "============================================" +payload='{ + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "09a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "09b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + + +echo "10. Update subscription with invalid exprLang, see error" +echo "========================================================" +payload='{ + "notification": { + "mqttCustom": { + "url": "mqtt://localhost:1883", + "topic": "orion", + "exprLang": 42 + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "11a. Get subscription and see exprLang legacy" +echo "=============================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "11b. Get subscription (as a list) and see exprLang legacy" +echo "=========================================================" +orionCurl --url /v2/subscriptions +echo +echo + +--REGEXPECT-- +01. Create subscription with invalid exprLang, see error +======================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 88 + +{ + "description": "not valid exprLang, valid ones are jexl or legacy", + "error": "BadRequest" +} + + +02. Create subscription with exprLang jexl +========================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03a. Get subscription and see exprLang jexl +=========================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 359 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "jexl", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +03b. Get subscription (as a list) and see exprLang jexl +======================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "jexl", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +04. Update subscription with exprLang legacy +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +05b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 363 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +06. Update subscription without exprLang +======================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +07a. Get subscription and see exprLang jexl +=========================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 359 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "jexl", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +07b. Get subscription (as a list) and see exprLang jexl +======================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "jexl", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +08. Update subscription with exprLang legacy +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +09a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +09b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 363 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +10. Update subscription with invalid exprLang, see error +======================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 87 + +{ + "description": "exprLang mqttCustom notification is not a string", + "error": "BadRequest" +} + + +11a. Get subscription and see exprLang legacy +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 361 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } +} + + +11b. Get subscription (as a list) and see exprLang legacy +========================================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 363 + +[ + { + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "mqttCustom": { + "exprLang": "legacy", + "qos": 0, + "retain": false, + "topic": "orion", + "url": "mqtt://localhost:1883" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + } + } +] + + +--TEARDOWN-- +brokerStop CB +dbDrop CB diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test index 4f2b9778b6..ed8fee4246 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test @@ -110,7 +110,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 567 +Content-Length: 585 { "id": "REGEX([0-9a-f]{24})", @@ -119,6 +119,7 @@ Content-Length: 567 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "SomeType", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test index 9d671877f0..71d19596e5 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test @@ -275,7 +275,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 365 +Content-Length: 383 { "id": "REGEX([0-9a-f]{24})", @@ -284,6 +284,7 @@ Content-Length: 365 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -326,7 +327,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -335,6 +336,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -370,7 +372,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 365 +Content-Length: 383 { "id": "REGEX([0-9a-f]{24})", @@ -379,6 +381,7 @@ Content-Length: 365 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -421,7 +424,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 365 +Content-Length: 383 { "id": "REGEX([0-9a-f]{24})", @@ -430,6 +433,7 @@ Content-Length: 365 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "C": { "type": "Number", @@ -472,7 +476,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -481,6 +485,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -518,7 +523,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -527,6 +532,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -562,7 +568,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 365 +Content-Length: 383 { "id": "REGEX([0-9a-f]{24})", @@ -571,6 +577,7 @@ Content-Length: 365 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", @@ -605,7 +612,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 686 +Content-Length: 722 [ { @@ -615,6 +622,7 @@ Content-Length: 686 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -641,6 +649,7 @@ Content-Length: 686 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test index a34bfa6d96..3ee0ec39fb 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test @@ -293,7 +293,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 414 +Content-Length: 432 { "id": "REGEX([0-9a-f]{24})", @@ -302,6 +302,7 @@ Content-Length: 414 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "StructuredValue", @@ -353,7 +354,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -362,6 +363,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -397,7 +399,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 415 +Content-Length: 433 { "id": "REGEX([0-9a-f]{24})", @@ -406,6 +408,7 @@ Content-Length: 415 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "StructuredValue", @@ -461,7 +464,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 411 +Content-Length: 429 { "id": "REGEX([0-9a-f]{24})", @@ -470,6 +473,7 @@ Content-Length: 411 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "C": { "type": "StructuredValue", @@ -521,7 +525,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -530,6 +534,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -567,7 +572,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -576,6 +581,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -611,7 +617,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 417 +Content-Length: 435 { "id": "REGEX([0-9a-f]{24})", @@ -620,6 +626,7 @@ Content-Length: 417 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "StructuredValue", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test index 2e60112a6e..d6330c8c1f 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test @@ -259,7 +259,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 353 +Content-Length: 371 { "id": "REGEX([0-9a-f]{24})", @@ -268,6 +268,7 @@ Content-Length: 353 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -309,7 +310,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -318,6 +319,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -353,7 +355,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 343 +Content-Length: 361 { "id": "REGEX([0-9a-f]{24})", @@ -362,6 +364,7 @@ Content-Length: 343 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -402,7 +405,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 333 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -411,6 +414,7 @@ Content-Length: 333 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "id": "E3", "type": "T3" @@ -449,7 +453,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -458,6 +462,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -495,7 +500,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 318 +Content-Length: 336 { "id": "REGEX([0-9a-f]{24})", @@ -504,6 +509,7 @@ Content-Length: 318 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -539,7 +545,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 355 +Content-Length: 373 { "id": "REGEX([0-9a-f]{24})", @@ -548,6 +554,7 @@ Content-Length: 355 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test index 8043fd7c4f..0f62f2759f 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test @@ -111,7 +111,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 600 +Content-Length: 618 { "id": "REGEX([0-9a-f]{24})", @@ -120,6 +120,7 @@ Content-Length: 600 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "SomeType", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test index c08b1ab6ca..d8421cc232 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test @@ -282,7 +282,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 398 +Content-Length: 416 { "id": "REGEX([0-9a-f]{24})", @@ -291,6 +291,7 @@ Content-Length: 398 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -336,7 +337,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -345,6 +346,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -383,7 +385,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 398 +Content-Length: 416 { "id": "REGEX([0-9a-f]{24})", @@ -392,6 +394,7 @@ Content-Length: 398 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -437,7 +440,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 398 +Content-Length: 416 { "id": "REGEX([0-9a-f]{24})", @@ -446,6 +449,7 @@ Content-Length: 398 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "C": { "type": "Number", @@ -491,7 +495,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -500,6 +504,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -540,7 +545,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -549,6 +554,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -587,7 +593,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 398 +Content-Length: 416 { "id": "REGEX([0-9a-f]{24})", @@ -596,6 +602,7 @@ Content-Length: 398 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", @@ -633,7 +640,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 752 +Content-Length: 788 [ { @@ -643,6 +650,7 @@ Content-Length: 752 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -672,6 +680,7 @@ Content-Length: 752 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test index fcbbb8f196..2eaf4e7c1e 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test @@ -300,7 +300,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 447 +Content-Length: 465 { "id": "REGEX([0-9a-f]{24})", @@ -309,6 +309,7 @@ Content-Length: 447 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "StructuredValue", @@ -363,7 +364,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -372,6 +373,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -410,7 +412,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 448 +Content-Length: 466 { "id": "REGEX([0-9a-f]{24})", @@ -419,6 +421,7 @@ Content-Length: 448 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "StructuredValue", @@ -477,7 +480,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 444 +Content-Length: 462 { "id": "REGEX([0-9a-f]{24})", @@ -486,6 +489,7 @@ Content-Length: 444 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "C": { "type": "StructuredValue", @@ -540,7 +544,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -549,6 +553,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -589,7 +594,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -598,6 +603,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -636,7 +642,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 450 +Content-Length: 468 { "id": "REGEX([0-9a-f]{24})", @@ -645,6 +651,7 @@ Content-Length: 450 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "StructuredValue", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test index 4ed1f9ce81..0ea36db1d8 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test @@ -266,7 +266,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 386 +Content-Length: 404 { "id": "REGEX([0-9a-f]{24})", @@ -275,6 +275,7 @@ Content-Length: 386 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -319,7 +320,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -328,6 +329,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -366,7 +368,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 376 +Content-Length: 394 { "id": "REGEX([0-9a-f]{24})", @@ -375,6 +377,7 @@ Content-Length: 376 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -418,7 +421,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 366 +Content-Length: 384 { "id": "REGEX([0-9a-f]{24})", @@ -427,6 +430,7 @@ Content-Length: 366 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "id": "E3", "type": "T3" @@ -468,7 +472,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -477,6 +481,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -517,7 +522,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 369 { "id": "REGEX([0-9a-f]{24})", @@ -526,6 +531,7 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -564,7 +570,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 388 +Content-Length: 406 { "id": "REGEX([0-9a-f]{24})", @@ -573,6 +579,7 @@ Content-Length: 388 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test b/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test index 9aff1e0a20..4054d27904 100644 --- a/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test +++ b/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test @@ -262,7 +262,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 340 +Content-Length: 358 { "id": "REGEX([0-9a-f]{24})", @@ -271,6 +271,7 @@ Content-Length: 340 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -300,7 +301,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 342 +Content-Length: 360 [ { @@ -310,6 +311,7 @@ Content-Length: 342 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -348,7 +350,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 341 +Content-Length: 359 { "id": "REGEX([0-9a-f]{24})", @@ -357,6 +359,7 @@ Content-Length: 341 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -386,7 +389,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 343 +Content-Length: 361 [ { @@ -396,6 +399,7 @@ Content-Length: 343 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -434,7 +438,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 341 +Content-Length: 359 { "id": "REGEX([0-9a-f]{24})", @@ -443,6 +447,7 @@ Content-Length: 341 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -472,7 +477,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 343 +Content-Length: 361 [ { @@ -482,6 +487,7 @@ Content-Length: 343 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -520,7 +526,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 340 +Content-Length: 358 { "id": "REGEX([0-9a-f]{24})", @@ -529,6 +535,7 @@ Content-Length: 340 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -558,7 +565,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 342 +Content-Length: 360 [ { @@ -568,6 +575,7 @@ Content-Length: 342 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -612,7 +620,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 340 +Content-Length: 358 { "id": "REGEX([0-9a-f]{24})", @@ -621,6 +629,7 @@ Content-Length: 340 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -650,7 +659,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 342 +Content-Length: 360 [ { @@ -660,6 +669,7 @@ Content-Length: 342 "attrsFormat": "normalized", "covered": false, "mqttCustom": { + "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", diff --git a/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test b/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test index 8352a77b7b..191958e178 100644 --- a/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test +++ b/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test @@ -146,7 +146,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 524 +Content-Length: 542 { "description": "DateTime test", @@ -159,6 +159,7 @@ Content-Length: 524 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "dateobservedto": { "type": "DateTime", @@ -205,7 +206,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 528 +Content-Length: 546 { "description": "DateTime test", @@ -218,6 +219,7 @@ Content-Length: 528 "attrsFormat": "normalized", "covered": false, "httpCustom": { + "exprLang": "jexl", "ngsi": { "dateobservedto2": { "type": "DateTime", From f2a601afd91ff749119ddbb09953252700e0b711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 26 Feb 2024 18:30:57 +0100 Subject: [PATCH 179/390] FIX temporarly force legacy mode in ExprContext to check all ftest in legacy mode --- src/lib/ngsiNotify/Notifier.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 3f8516798f..01fbcc4ec0 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -318,7 +318,9 @@ static SenderThreadParams* buildSenderParamsCustom const std::string exprLang = notification.type == ngsiv2::HttpNotification ? notification.httpInfo.exprLang : notification.mqttInfo.exprLang; // Used by several macroSubstitute() calls along this function - ExprContextObject exprContext(exprLang == "legacy"); + // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode + //ExprContextObject exprContext(exprLang == "legacy"); + ExprContextObject exprContext(true); exprContext.add("id", en.id); exprContext.add("type", en.type); exprContext.add("service", tenant); From e581530e3be29fa0aa2226c80563df8d70fac83c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 27 Feb 2024 12:27:33 +0100 Subject: [PATCH 180/390] FIX to reference presentation and version numbers --- README.md | 2 +- doc/manuals.jp/devel/README.md | 2 +- doc/manuals/devel/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 70ad7e6957..838d95c675 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ recommended to have a look to the brief ### Introductory presentations - Orion Context Broker - [(en)](https://www.slideshare.net/slideshows/orion-context-broker-introduction-20240115/265436732) + [(en)](https://www.slideshare.net/slideshows/orion-context-broker-introduction-20240227/266516824) [(jp)](https://www.slideshare.net/slideshows/orion-context-broker-20240116/265462443) - NGSIv2 Overview for Developers That Already Know NGSIv1 [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220523) diff --git a/doc/manuals.jp/devel/README.md b/doc/manuals.jp/devel/README.md index 07dfc9889e..603da17228 100644 --- a/doc/manuals.jp/devel/README.md +++ b/doc/manuals.jp/devel/README.md @@ -1,6 +1,6 @@ # 開発マニュアル -*注 : このドキュメントでは、リリース 3.11.x の Orion Context Broker について説明しています。* +*注 : このドキュメントでは、リリース 3.12.x の Orion Context Broker について説明しています。* ## 対象読者 diff --git a/doc/manuals/devel/README.md b/doc/manuals/devel/README.md index 34046787f8..87fc4f4e02 100644 --- a/doc/manuals/devel/README.md +++ b/doc/manuals/devel/README.md @@ -1,6 +1,6 @@ # Development Manual -*Note: This document describes Orion Context Broker as of release 3.11.x.* +*Note: This document describes Orion Context Broker as of release 3.12.x.* ## Intended audience The intended audience of this manual is developers that need to understand the internals of the Orion Context Broker From ce510784e647d3b7a9b2daba40e28fd1913cfb78 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 27 Feb 2024 21:19:24 +0900 Subject: [PATCH 181/390] (JP) Update links to docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 838d95c675..9cfac12e21 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ recommended to have a look to the brief - Orion Context Broker [(en)](https://www.slideshare.net/slideshows/orion-context-broker-introduction-20240227/266516824) - [(jp)](https://www.slideshare.net/slideshows/orion-context-broker-20240116/265462443) + [(jp)](https://www.slideshare.net/slideshows/orion-context-broker-introduction-20240227/266517846) - NGSIv2 Overview for Developers That Already Know NGSIv1 [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220523) [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220526) From a307e4c100c7642cb7352a89ddccabb9957061cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 27 Feb 2024 13:37:12 +0100 Subject: [PATCH 182/390] FIX add comment about Python interpreter in walkthrough_apiv2.md --- doc/manuals/user/walkthrough_apiv2.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manuals/user/walkthrough_apiv2.md b/doc/manuals/user/walkthrough_apiv2.md index 9ffbf6407a..a9f78876f1 100644 --- a/doc/manuals/user/walkthrough_apiv2.md +++ b/doc/manuals/user/walkthrough_apiv2.md @@ -210,7 +210,9 @@ Some additional remarks: different port. - In order to pretty-print JSON in responses, you can use Python with - msjon.tool (examples along with tutorial are using this style): + msjon.tool (examples along with tutorial are using this style). Replace + `python` by the particular Python executable in your case (e.g. in some + cases it could be `| python3 -mjson.tool`). ``` (curl ... | python -mjson.tool) < Date: Tue, 27 Feb 2024 22:12:05 +0900 Subject: [PATCH 183/390] Fix typo --- doc/manuals/user/walkthrough_apiv2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/user/walkthrough_apiv2.md b/doc/manuals/user/walkthrough_apiv2.md index a9f78876f1..568b208a7f 100644 --- a/doc/manuals/user/walkthrough_apiv2.md +++ b/doc/manuals/user/walkthrough_apiv2.md @@ -210,7 +210,7 @@ Some additional remarks: different port. - In order to pretty-print JSON in responses, you can use Python with - msjon.tool (examples along with tutorial are using this style). Replace + mjson.tool (examples along with tutorial are using this style). Replace `python` by the particular Python executable in your case (e.g. in some cases it could be `| python3 -mjson.tool`). From ca2ab8e44f669b23d8413c1ea921f92309564bbf Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 27 Feb 2024 22:12:26 +0900 Subject: [PATCH 184/390] (JP) FIX add comment about Python interpreter in walkthrough_apiv2.md (#4524) --- doc/manuals.jp/user/walkthrough_apiv2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals.jp/user/walkthrough_apiv2.md b/doc/manuals.jp/user/walkthrough_apiv2.md index 433f3231b0..9ea87203cc 100644 --- a/doc/manuals.jp/user/walkthrough_apiv2.md +++ b/doc/manuals.jp/user/walkthrough_apiv2.md @@ -166,7 +166,7 @@ curl ... -H 'Content-Type: application/json' ... - 例では、broker がポート1026をリッスンしていると仮定しています。異なるポートを使用している場合は、curl コマンドラインでこれを調整してください -- レスポンスで JSON をきれいに印刷するために、Python を msjon.tool とともに使用することができます (チュートリアルの例ではこのスタイルを使用しています) : +- レスポンスで JSON をきれいに印刷するために、Python を mjson.tool とともに使用することができます (チュートリアルの例ではこのスタイルを使用しています) : `python` をあなたのケースによって特定の Python 実行可能ファイルに置き換えます (たとえば、ケースによっては `| python3 -mjson.tool` になる可能性があります)。 ``` (curl ... | python -mjson.tool) < Date: Wed, 28 Feb 2024 19:00:10 +0100 Subject: [PATCH 185/390] wip --- src/lib/common/macroSubstitute.cpp | 10 +++- src/lib/expressions/ExprContext.cpp | 87 ++++++++++------------------- src/lib/expressions/ExprContext.h | 6 +- src/lib/expressions/ExprResult.cpp | 4 +- src/lib/ngsi/ContextAttribute.cpp | 21 ++++++- src/lib/ngsi/ContextAttribute.h | 2 +- src/lib/ngsiNotify/Notifier.cpp | 4 +- 7 files changed, 65 insertions(+), 69 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index e1619ca77c..23b631a710 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -60,7 +60,7 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e } else { - // in legacy mode an extra remove quotes step is needed + // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { return removeQuotes(r.toString()); @@ -80,6 +80,7 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // error already logged in macroSubstitute, using stringValue itself as failsafe effectiveValue = stringValue; } + // toJsonString will stringfly JSON values in macros return '"' + toJsonString(effectiveValue) + '"'; } @@ -108,6 +109,13 @@ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, c else { std::string s = r.toString(); + + // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. + if (exprContextObjectP->isLegacy()) + { + s = removeQuotes(s); + } + if (raw) { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 87f1de9f9d..43737af844 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -81,11 +81,18 @@ std::map* ExprContextObject::getMap(void) * * ExprContextObject::add - */ -void ExprContextObject::add(const std::string &key, const std::string &_value) +void ExprContextObject::add(const std::string &key, const std::string &_value, bool raw) { if (legacy) { - repl.insert(std::pair(key, _value)); + std::string value = _value; + if (!raw) + { + // This is the case of regular string. The raw case is for JSON generated from compound values + value = '"' + _value + '"'; + } + LM_T(LmtExpr, ("adding to legacy expression context object (string): %s=%s", key.c_str(), value.c_str())); + repl.insert(std::pair(key, value)); } else { @@ -93,7 +100,7 @@ void ExprContextObject::add(const std::string &key, const std::string &_value) { return; } - LM_T(LmtExpr, ("adding to expression context object (string): %s=%s", key.c_str(), _value.c_str())); + LM_T(LmtExpr, ("adding to JEXL expression context object (string): %s=%s", key.c_str(), _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); if (value == NULL) { @@ -115,6 +122,7 @@ void ExprContextObject::add(const std::string &key, double _value) { if (legacy) { + LM_T(LmtExpr, ("adding to legacy expression context object (double): %s=%f", key.c_str(), _value)); repl.insert(std::pair(key, double2string(_value))); } else @@ -123,7 +131,7 @@ void ExprContextObject::add(const std::string &key, double _value) { return; } - LM_T(LmtExpr, ("adding to expression context object (double): %s=%f", key.c_str(), _value)); + LM_T(LmtExpr, ("adding to JEXL expression context object (double): %s=%f", key.c_str(), _value)); PyObject* value = Py_BuildValue("d", _value); if (value == NULL) { @@ -145,6 +153,7 @@ void ExprContextObject::add(const std::string &key, bool _value) { if (legacy) { + LM_T(LmtExpr, ("adding to legacy expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); repl.insert(std::pair(key, _value? "true": "false")); } else @@ -153,7 +162,7 @@ void ExprContextObject::add(const std::string &key, bool _value) { return; } - LM_T(LmtExpr, ("adding to expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); + LM_T(LmtExpr, ("adding to JEXL expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); if (_value) { PyDict_SetItemString(jexlContext, key.c_str(), Py_True); @@ -175,6 +184,7 @@ void ExprContextObject::add(const std::string &key) { if (legacy) { + LM_T(LmtExpr, ("adding to legacy expression context object (none): %s", key.c_str())); repl.insert(std::pair(key, "null")); } else @@ -183,7 +193,7 @@ void ExprContextObject::add(const std::string &key) { return; } - LM_T(LmtExpr, ("adding to expression context object (none): %s", key.c_str())); + LM_T(LmtExpr, ("adding to JEXL expression context object (none): %s", key.c_str())); PyDict_SetItemString(jexlContext, key.c_str(), Py_None); } } @@ -198,8 +208,7 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex { if (legacy) { - // FIXME PR: not sure if this is going to work... - repl.insert(std::pair(key, exprContextObject.toString())); + LM_E(("Runtime Error (this method must not be invoked in legacy mode)")); } else { @@ -207,7 +216,8 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex { return; } - LM_T(LmtExpr, ("adding to expression context object (object): %s=%s", key.c_str(), exprContextObject.toString().c_str())); + // FIXME P3: improve this implemeting a toString() method for ExprContextObject + LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=[object]", key.c_str()); PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.getJexlContext()); } } @@ -222,8 +232,7 @@ void ExprContextObject::add(const std::string &key, ExprContextList exprContextL { if (legacy) { - // FIXME PR: not sure if this is going to work... - repl.insert(std::pair(key, exprContextList.toString())); + LM_E(("Runtime Error (this method must not be invoked in legacy mode)")); } else { @@ -231,7 +240,8 @@ void ExprContextObject::add(const std::string &key, ExprContextList exprContextL { return; } - LM_T(LmtExpr, ("adding to expression context object (list): %s=%s", key.c_str(), exprContextList.toString().c_str())); + // FIXME P3: improve this implemeting a toString() method for ExprContextList + LM_T(LmtExpr, ("adding to JEXL expression context object (list): %s=[list]", key.c_str())); PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); } } @@ -248,25 +258,6 @@ bool ExprContextObject::isLegacy(void) } -/* **************************************************************************** -* -* ExprContextObject::toString - -*/ -std::string ExprContextObject::toString(void) -{ - const char* str = PyUnicode_AsUTF8(jexlContext); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - return ""; - } - else - { - return std::string(str); - } -} - - /* **************************************************************************** * @@ -323,7 +314,7 @@ void ExprContextList::add(const std::string &_value) { return; } - LM_T(LmtExpr, ("adding to expression context list (string): %s=", _value.c_str())); + LM_T(LmtExpr, ("adding to JEXL expression context list (string): %s=", _value.c_str())); PyObject* value = Py_BuildValue("s", _value.c_str()); if (value == NULL) { @@ -346,7 +337,7 @@ void ExprContextList::add(double _value) { return; } - LM_T(LmtExpr, ("adding to expression context list (double): %f", _value)); + LM_T(LmtExpr, ("adding to JEXL expression context list (double): %f", _value)); PyObject* value = Py_BuildValue("d", _value); if (value == NULL) { @@ -369,7 +360,7 @@ void ExprContextList::add(bool _value) { return; } - LM_T(LmtExpr, ("adding to expression context list (bool): %s", _value ? "true" : "false")); + LM_T(LmtExpr, ("adding to JEXL expression context list (bool): %s", _value ? "true" : "false")); if (_value) { PyList_Append(jexlContext, Py_True); @@ -392,7 +383,7 @@ void ExprContextList::add(void) { return; } - LM_T(LmtExpr, ("adding to expression context list (none)")); + LM_T(LmtExpr, ("adding to JEXL expression context list (none)")); PyList_Append(jexlContext, Py_None); } @@ -408,7 +399,8 @@ void ExprContextList::add(ExprContextObject exprContextObject) { return; } - LM_T(LmtExpr, ("adding to expression context list (object): %s", exprContextObject.toString().c_str())); + // FIXME P3: improve this implemeting a toString() method for ExprContextObject + LM_T(LmtExpr, ("adding to JEXL expression context list (object): [object]"))); PyList_Append(jexlContext, exprContextObject.getJexlContext()); } @@ -424,32 +416,13 @@ void ExprContextList::add(ExprContextList exprContextList) { return; } - LM_T(LmtExpr, ("adding to expression context list (list): %s", exprContextList.toString().c_str())); + // FIXME P3: improve this implemeting a toString() method for ExprContextList + LM_T(LmtExpr, ("adding to JEXL expression context list (list): [list]")); PyList_Append(jexlContext, exprContextList.get()); } -/* **************************************************************************** -* -* ExprContextList::toString - -*/ -std::string ExprContextList::toString(void) -{ - const char* str = PyUnicode_AsUTF8(jexlContext); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - return ""; - } - else - { - return std::string(str); - } -} - - - /* **************************************************************************** * * ExprContextList::relesase - diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 19c165f276..a3068b3138 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -49,7 +49,7 @@ class ExprContextObject PyObject* getJexlContext(void); std::map* getMap(void); - void add(const std::string& key, const std::string& value); + void add(const std::string& key, const std::string& value, bool raw = false); void add(const std::string& key, double value); void add(const std::string& key, bool value); void add(const std::string& key); @@ -58,8 +58,6 @@ class ExprContextObject bool isLegacy(void); - std::string toString(void); - void release(void); }; @@ -79,8 +77,6 @@ class ExprContextList void add(ExprContextObject exprContextObject); void add(ExprContextList exprContextList); - std::string toString(void); - void release(void); }; diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 986ca52e53..a57cbc9ab4 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -341,7 +341,9 @@ std::string ExprResult::toString(void) } else if (valueType == orion::ValueTypeString) { - return "\"" + toJsonString(stringValue) + "\""; + // FIXME PR: does this break the no legacy + //return "\"" + toJsonString(stringValue) + "\""; + return "\"" + stringValue + "\""; } else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) { diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 4bf143ba76..77e86480a9 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1296,17 +1296,32 @@ std::string ContextAttribute::toJsonAsValue * * Pretty similar in structure to toJsonValue */ -void ContextAttribute::addToContext(ExprContextObject* exprContextObjectP) +void ContextAttribute::addToContext(ExprContextObject* exprContextObjectP, bool legacy) { if (compoundValueP != NULL) { + // In legacy expression, objects are vector are strings to be stored in a std::map if (valueType == orion::ValueTypeObject) { - exprContextObjectP->add(name, compoundValueP->toExprContextObject()); + if (legacy) + { + exprContextObjectP->add(name, compoundValueP->toJson(), true); + } + else + { + exprContextObjectP->add(name, compoundValueP->toExprContextObject()); + } } else // valueType == orion::ValueTypeVector { - exprContextObjectP->add(name, compoundValueP->toExprContextList()); + if (legacy) + { + exprContextObjectP->add(name, compoundValueP->toJson(), true); + } + else + { + exprContextObjectP->add(name, compoundValueP->toExprContextList()); + } } } else if (valueType == orion::ValueTypeNumber) diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index 23b74c06f1..caa36cbfe5 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -119,7 +119,7 @@ typedef struct ContextAttribute MimeType* outMimeTypeP, HttpStatusCode* scP); - void addToContext(ExprContextObject* exprContextObjectP); + void addToContext(ExprContextObject* exprContextObjectP, bool legacy); void release(void); std::string getName(void); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 01fbcc4ec0..8c09270602 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -328,7 +328,9 @@ static SenderThreadParams* buildSenderParamsCustom exprContext.add("authToken", xauthToken); for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - en.attributeVector[ix]->addToContext(&exprContext); + // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode + //en.attributeVector[ix]->addToContext(&exprContext, exprLang == "legacy"); + en.attributeVector[ix]->addToContext(&exprContext, true); } // From 7ebe97f8ddc13436c66ead53460fd4776e923e34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 29 Feb 2024 10:04:09 +0100 Subject: [PATCH 186/390] Step: 3.11.0-next -> 3.12.0 --- CHANGES_NEXT_RELEASE | 5 ----- Changelog | 7 +++++++ README.md | 2 +- src/app/contextBroker/version.h | 2 +- src/lib/common/defaultValues.h | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 24f4ac02e7..e69de29bb2 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,5 +0,0 @@ -- Fix: service path levels with 0 length should not be allowed (#4495) -- Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) -- Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) -- Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) -- Deprecate: `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` CLI parameters along with associated env vars (use `-dbURI` instead`) \ No newline at end of file diff --git a/Changelog b/Changelog index dd05d8f08c..4ca4aa8522 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,10 @@ +3.12.0 (February 29th, 2024) + +- Fix: service path levels with 0 length should not be allowed (#4495) +- Fix: changed the default value of `-dbTimeout` to 0 to resolve conflict with `-dbURI` (#4496) +- Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) +- Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) +- Deprecate: `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` CLI parameters along with associated env vars (use `-dbURI` instead`) 3.11.0 (January 29th, 2024) - Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388) diff --git a/README.md b/README.md index 9cfac12e21..20c0575c56 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![Support badge](https://img.shields.io/badge/tag-fiware--orion-orange.svg?logo=stackoverflow)](http://stackoverflow.com/questions/tagged/fiware-orion) [![NGSI v2](https://img.shields.io/badge/NGSI-V2-red.svg)](doc/manuals/orion-api.md)
-[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion.svg)](https://fiware-orion.rtfd.io) +[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion/3.12.0.svg)](https://fiware-orion.rtfd.io/en/3.12.0/) ![Compliance Tests](https://github.com/telefonicaid/fiware-orion/workflows/Compliance%20Tests/badge.svg) ![Unit Tests](https://github.com/telefonicaid/fiware-orion/workflows/Unit%20Tests/badge.svg) ![Functional Tests](https://github.com/telefonicaid/fiware-orion/workflows/Functional%20Tests/badge.svg) diff --git a/src/app/contextBroker/version.h b/src/app/contextBroker/version.h index 9e3680ba6d..d2fde8d6ad 100644 --- a/src/app/contextBroker/version.h +++ b/src/app/contextBroker/version.h @@ -28,6 +28,6 @@ -#define ORION_VERSION "3.11.0-next" +#define ORION_VERSION "3.12.0" #endif // SRC_APP_CONTEXTBROKER_VERSION_H_ diff --git a/src/lib/common/defaultValues.h b/src/lib/common/defaultValues.h index 95998c82df..2fd823a03e 100644 --- a/src/lib/common/defaultValues.h +++ b/src/lib/common/defaultValues.h @@ -41,6 +41,6 @@ * * API Documentation - The link to the the GEri documentation, either in the gh-pages (.github.io/) inside the fiware organization in GitHub or ReadTheDocs manual. */ -#define API_DOC "https://fiware-orion.rtfd.io/" +#define API_DOC "https://fiware-orion.rtfd.io/en/3.12.0/" #endif // SRC_LIB_COMMON_DEFAULTVALUES_H_ From 1bd44af754df21fcbaf32832e9090e05fcfb4d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 29 Feb 2024 10:41:12 +0100 Subject: [PATCH 187/390] Step: 3.12.0 -> 3.12.0-next --- README.md | 2 +- src/app/contextBroker/version.h | 2 +- src/lib/common/defaultValues.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 20c0575c56..9cfac12e21 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![Support badge](https://img.shields.io/badge/tag-fiware--orion-orange.svg?logo=stackoverflow)](http://stackoverflow.com/questions/tagged/fiware-orion) [![NGSI v2](https://img.shields.io/badge/NGSI-V2-red.svg)](doc/manuals/orion-api.md)
-[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion/3.12.0.svg)](https://fiware-orion.rtfd.io/en/3.12.0/) +[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion.svg)](https://fiware-orion.rtfd.io) ![Compliance Tests](https://github.com/telefonicaid/fiware-orion/workflows/Compliance%20Tests/badge.svg) ![Unit Tests](https://github.com/telefonicaid/fiware-orion/workflows/Unit%20Tests/badge.svg) ![Functional Tests](https://github.com/telefonicaid/fiware-orion/workflows/Functional%20Tests/badge.svg) diff --git a/src/app/contextBroker/version.h b/src/app/contextBroker/version.h index d2fde8d6ad..26b18df3d6 100644 --- a/src/app/contextBroker/version.h +++ b/src/app/contextBroker/version.h @@ -28,6 +28,6 @@ -#define ORION_VERSION "3.12.0" +#define ORION_VERSION "3.12.0-next" #endif // SRC_APP_CONTEXTBROKER_VERSION_H_ diff --git a/src/lib/common/defaultValues.h b/src/lib/common/defaultValues.h index 2fd823a03e..95998c82df 100644 --- a/src/lib/common/defaultValues.h +++ b/src/lib/common/defaultValues.h @@ -41,6 +41,6 @@ * * API Documentation - The link to the the GEri documentation, either in the gh-pages (.github.io/) inside the fiware organization in GitHub or ReadTheDocs manual. */ -#define API_DOC "https://fiware-orion.rtfd.io/en/3.12.0/" +#define API_DOC "https://fiware-orion.rtfd.io/" #endif // SRC_LIB_COMMON_DEFAULTVALUES_H_ From abefe2b35d5446e109fa08517aa710b119d42acb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 29 Feb 2024 10:46:06 +0100 Subject: [PATCH 188/390] FIX typo in Changelog file --- Changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog b/Changelog index 4ca4aa8522..91315cc289 100644 --- a/Changelog +++ b/Changelog @@ -5,6 +5,7 @@ - Fix: wrong INFO startup log showing ORION_MONGO_TIMEOUT, ORION_IN_REQ_PAYLOAD_MAX_SIZE and ORION_OUT_REQ_MSG_MAX_SIZE env var values (#4496) - Fix: return 400 Bad Request when subject.entities exists but it is an empty array (#4499) - Deprecate: `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` CLI parameters along with associated env vars (use `-dbURI` instead`) + 3.11.0 (January 29th, 2024) - Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388) From 82941cc6b6868a452ff8cc674260936d5a0b2a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 29 Feb 2024 12:24:30 +0100 Subject: [PATCH 189/390] REMOVE deprecated db related CLIs --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/deprecated.md | 4 +- src/app/contextBroker/contextBroker.cpp | 24 +---- src/lib/mongoBackend/MongoGlobal.cpp | 16 ---- src/lib/mongoBackend/MongoGlobal.h | 8 -- src/lib/mongoDriver/mongoConnectionPool.cpp | 100 ++------------------ src/lib/mongoDriver/mongoConnectionPool.h | 8 -- test/unittests/main_UnitTest.cpp | 18 +--- 8 files changed, 16 insertions(+), 163 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e69de29bb2..59f9251202 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -0,0 +1 @@ +- Remove: `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` CLI parameters along with associated env vars, already deprecated in Orion 3.12.0 (use `-dbURI` instead`) \ No newline at end of file diff --git a/doc/manuals/deprecated.md b/doc/manuals/deprecated.md index 9b289331b4..5ed729988b 100644 --- a/doc/manuals/deprecated.md +++ b/doc/manuals/deprecated.md @@ -17,7 +17,7 @@ A list of deprecated features and the version in which they were deprecated foll * CLI parameters (and associated env vars): `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` in Orion 3.12.0. Use `dbURI` instead, - checking [this section](#mapping-to-mongouri-from-old-cli-parameters) if you need to know hot to build the MongoDB URI. + checking [this section](#mapping-to-mongouri-from-old-cli-parameters) if you need to know hot to build the MongoDB URI (removed in Orion 3.13.0). * `geo:point`, `geo:line`, `geo:box` and `geo:polygon` attribute types in Orion 3.10.0. Use `geo:json` instead. * `GET /v2` operation in Orion 3.8.0. This operation is pretty useless and not actually used. * Initial notification in subscriptions (along with `skipInitialNotification` option) in Orion 3.1.0. @@ -134,12 +134,12 @@ The following table provides information about the last Orion version supporting | **Removed feature** | **Last Orion version supporting feature** | **That version release date** | |----------------------------------------------------------------------------|-------------------------------------------|---------------------------------| -| CLI `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` (and associated env vars) | Not yet defined | Not yet defined | | `attributes` field in `POST /v2/entities` operation | Not yet defined | Not yet defined | | `APPEND`, `UPDATE`, etc. action types in `POST /v2/op/update` | Not yet defined | Not yet defined | | `dateCreated` and `dateModified` in `options` URI parameter | Not yet defined | Not yet defined | | `GET /v2` operation | Not yet defined | Not yet defined | | `geo:point`, `geo:line`, `geo:box` and `geo:polygon` attribute types | Not yet defined | Not yet defined | +| CLI `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` (and associated env vars) | 3.12.0 | February 29th, 2024 | | `location` metadata to specify entity location | 3.10.1 | June 12th, 2023 | | NGSIv1 API (along with CLI: `-strictNgsiv1Ids` and `-ngsiv1Autocast`) | 3.9.0 (*) | June 2nd, 2023 | | `/ngsi10` and `/ngsi9` URL prefixes | 3.7.0 (*) | May 26th, 2022 | diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index d1cc8a57b2..b8ede361f5 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -139,15 +139,8 @@ static bool isFatherProcess = false; bool fg; char bindAddress[MAX_LEN_IP]; int port; -char dbHost[256]; -char rplSet[64]; char dbName[64]; -char user[256]; char pwd[256]; -char authMech[64]; -char authDb[64]; -bool dbSSL; -bool dbDisableRetryWrites; char dbURI[1024]; char pidPath[256]; bool harakiri; @@ -159,7 +152,6 @@ bool https; bool mtenant; char allowedOrigin[64]; int maxAge; -long dbTimeout; long httpTimeout; long mqttTimeout; int dbPoolSize; @@ -298,18 +290,9 @@ PaArgument paArgs[] = { "-pidpath", pidPath, "PID_PATH", PaString, PaOpt, PIDPATH, PaNL, PaNL, PIDPATH_DESC }, { "-dbURI", dbURI, "MONGO_URI", PaString, PaOpt, _i "", PaNL, PaNL, DBURI_DESC }, - { "-dbhost", dbHost, "MONGO_HOST", PaString, PaOpt, LOCALHOST, PaNL, PaNL, DBHOST_DESC }, - { "-rplSet", rplSet, "MONGO_REPLICA_SET", PaString, PaOpt, _i "", PaNL, PaNL, RPLSET_DESC }, - { "-dbuser", user, "MONGO_USER", PaString, PaOpt, _i "", PaNL, PaNL, DBUSER_DESC }, { "-dbpwd", pwd, "MONGO_PASSWORD", PaString, PaOpt, _i "", PaNL, PaNL, DBPASSWORD_DESC }, - { "-dbAuthMech", authMech, "MONGO_AUTH_MECH", PaString, PaOpt, _i "", PaNL, PaNL, DBAUTHMECH_DESC }, - { "-dbAuthDb", authDb, "MONGO_AUTH_SOURCE", PaString, PaOpt, _i "", PaNL, PaNL, DBAUTHDB_DESC }, - { "-dbSSL", &dbSSL, "MONGO_SSL", PaBool, PaOpt, false, false, true, DBSSL_DESC }, - { "-dbDisableRetryWrites", &dbDisableRetryWrites, "MONGO_DISABLE_RETRY_WRITES", PaBool, PaOpt, false, false, true, DBDISABLERETRYWRITES_DESC }, - { "-db", dbName, "MONGO_DB", PaString, PaOpt, _i "orion", PaNL, PaNL, DB_DESC }, - { "-dbTimeout", &dbTimeout, "MONGO_TIMEOUT", PaULong, PaOpt, 0, 0, UINT_MAX, DB_TMO_DESC }, { "-dbPoolSize", &dbPoolSize, "MONGO_POOL_SIZE", PaInt, PaOpt, 10, 1, 10000, DBPS_DESC }, { "-ipv4", &useOnlyIPv4, "USEIPV4", PaBool, PaOpt, false, false, true, USEIPV4_DESC }, @@ -1133,11 +1116,6 @@ int main(int argC, char* argV[]) LM_X(1, ("dbName too long (max %d characters)", DB_NAME_MAX_LEN)); } - if ((strlen(authMech) > 0) && (strncmp(authMech, "SCRAM-SHA-1", strlen("SCRAM-SHA-1")) != 0) && (strncmp(authMech, "SCRAM-SHA-256", strlen("SCRAM-SHA-256")) != 0)) - { - LM_X(1, ("Fatal Error (-dbAuthMech must be either SCRAM-SHA-1 or SCRAM-SHA-256")); - } - if (useOnlyIPv6 && useOnlyIPv4) { LM_X(1, ("Fatal Error (-ipv4 and -ipv6 can not be activated at the same time. They are incompatible)")); @@ -1220,7 +1198,7 @@ int main(int argC, char* argV[]) alarmMgr.init(relogAlarms); mqttMgr.init(mqttTimeout); orionInit(orionExit, ORION_VERSION, policy, statCounters, statSemWait, statTiming, statNotifQueue, strictIdv1); - mongoInit(dbURI, dbHost, rplSet, dbName, user, pwd, authMech, authDb, dbSSL, dbDisableRetryWrites, mtenant, dbTimeout, writeConcern, dbPoolSize, statSemWait); + mongoInit(dbURI, dbName, pwd, mtenant, writeConcern, dbPoolSize, statSemWait); metricsMgr.init(!disableMetrics, statSemWait); logSummaryInit(&lsPeriod); diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp index 85308f228b..ed1c979ca5 100644 --- a/src/lib/mongoBackend/MongoGlobal.cpp +++ b/src/lib/mongoBackend/MongoGlobal.cpp @@ -105,17 +105,9 @@ bool mongoMultitenant(void) void mongoInit ( const char* dbURI, - const char* dbHost, - const char* rplSet, std::string dbName, - const char* user, const char* pwd, - const char* mechanism, - const char* authDb, - bool dbSSL, - bool dbDisableRetryWrites, bool mtenant, - int64_t timeout, int writeConcern, int dbPoolSize, bool mutexTimeStat @@ -125,17 +117,9 @@ void mongoInit multitenant = mtenant; if (orion::mongoConnectionPoolInit(dbURI, - dbHost, dbName.c_str(), - rplSet, - user, pwd, - mechanism, - authDb, - dbSSL, - dbDisableRetryWrites, mtenant, - timeout, writeConcern, dbPoolSize, mutexTimeStat) != 0) diff --git a/src/lib/mongoBackend/MongoGlobal.h b/src/lib/mongoBackend/MongoGlobal.h index caa6058e8f..7b951c0784 100644 --- a/src/lib/mongoBackend/MongoGlobal.h +++ b/src/lib/mongoBackend/MongoGlobal.h @@ -72,17 +72,9 @@ extern bool mongoMultitenant(void); void mongoInit ( const char* dbURI, - const char* dbHost, - const char* rplSet, std::string dbName, - const char* user, const char* pwd, - const char* mechanism, - const char* authDb, - bool dbSSL, - bool dbDisableRetryWrites, bool mtenant, - int64_t timeout, int writeConcern, int dbPoolSize, bool mutexTimeStat diff --git a/src/lib/mongoDriver/mongoConnectionPool.cpp b/src/lib/mongoDriver/mongoConnectionPool.cpp index 855896753e..bcde607889 100644 --- a/src/lib/mongoDriver/mongoConnectionPool.cpp +++ b/src/lib/mongoDriver/mongoConnectionPool.cpp @@ -312,100 +312,28 @@ static void mongoDriverLogger /* **************************************************************************** * -* mongoConnectionPoolInit - +* composeMongoUri - */ -static std::string composeMongoUri -( - const char* dbURI, - const char* host, - const char* rplSet, - const char* username, - const char* passwd, - const char* mechanism, - const char* authDb, - bool dbSSL, - bool dbDisableRetryWrites, - int64_t timeout -) +static std::string composeMongoUri(const char* dbURI, const char* passwd) { // Compose the mongoUri, taking into account all information std::string uri; - if (strlen(dbURI) != 0) + const char* pwd = strstr(dbURI, "${PWD}"); + if (pwd != NULL) { - if (strlen(username) != 0 || strlen(authDb) != 0 || strlen(rplSet) != 0 || strlen(mechanism) != 0 || dbSSL || dbDisableRetryWrites || timeout > 0) + if (strlen(passwd) == 0) { - LM_X(1, ("Invalid Command Line Options: -dbURI cannot be combined with -dbhost, -rplSet, -dbTimeout, -dbuser, -dbAuthMech, -dbAuthDb, -dbSSL and -dbDisableRetryWrites")); + LM_X(1, ("Invalid Command Line Options: -dbURI is used with a password substitution, but no password (-dbpwd) is supplied")); } - const char* pwd = strstr(dbURI, "${PWD}"); - if (pwd != NULL) - { - if (strlen(passwd) == 0) - { - LM_X(1, ("Invalid Command Line Options: -dbURI is used with a password substitution, but no password (-dbpwd) is supplied")); - } - - // +6 is the length of the "${PWD}" - uri = std::string(dbURI, pwd - dbURI) + passwd + (pwd + 6); - } - else - { - uri = dbURI; - } + // +6 is the length of the "${PWD}" + uri = std::string(dbURI, pwd - dbURI) + passwd + (pwd + 6); } else { - uri = "mongodb://"; - - // Add auth parameter if included - if (strlen(username) != 0 && strlen(passwd) != 0) - { - uri += username + std::string(":") + passwd + "@"; - } - - uri += host + std::string("/"); - - if (strlen(authDb) != 0) - { - uri += authDb; - } - - // First option prefix is '?' symbol - std::string optionPrefix = "?"; - - if (strlen(rplSet) != 0) - { - uri += optionPrefix + "replicaSet=" + rplSet; - optionPrefix = "&"; - } - - if (strlen(mechanism) != 0) - { - uri += optionPrefix + "authMechanism=" + mechanism; - optionPrefix = "&"; - } - - if (dbSSL) - { - uri += optionPrefix + "tls=true&tlsAllowInvalidCertificates=true"; - optionPrefix = "&"; - } - - if (dbDisableRetryWrites) - { - uri += optionPrefix + "retryWrites=false"; - optionPrefix = "&"; - } - - if (timeout > 0) - { - char buf[STRING_SIZE_FOR_LONG]; - i2s(timeout, buf, sizeof(buf)); - uri += optionPrefix + "connectTimeoutMS=" + buf; - optionPrefix = "&"; - } + uri = dbURI; } LM_T(LmtMongo, ("MongoDB connection URI: '%s'", offuscatePassword(uri, passwd).c_str())); @@ -422,17 +350,9 @@ static std::string composeMongoUri int orion::mongoConnectionPoolInit ( const char* dbURI, - const char* host, const char* db, - const char* rplSet, - const char* username, const char* passwd, - const char* mechanism, - const char* authDb, - bool dbSSL, - bool dbDisableRetryWrites, bool mtenant, - int64_t timeout, int writeConcern, int poolSize, bool semTimeStat @@ -451,7 +371,7 @@ int orion::mongoConnectionPoolInit atexit(shutdownClient); // Set mongo Uri to connect - std::string uri = composeMongoUri(dbURI, host, rplSet, username, passwd, mechanism, authDb, dbSSL, dbDisableRetryWrites, timeout); + std::string uri = composeMongoUri(dbURI, passwd); #ifdef UNIT_TEST /* Basically, we are mocking all the DB pool with a single connection. The getMongoConnection() and mongoReleaseConnection() methods diff --git a/src/lib/mongoDriver/mongoConnectionPool.h b/src/lib/mongoDriver/mongoConnectionPool.h index 0dddc183ae..e5991eb55b 100644 --- a/src/lib/mongoDriver/mongoConnectionPool.h +++ b/src/lib/mongoDriver/mongoConnectionPool.h @@ -47,17 +47,9 @@ extern void mongoVersionGet(int* mayor, int* minor); extern int mongoConnectionPoolInit ( const char* dbURI, - const char* host, const char* db, - const char* rplSet, - const char* username, const char* passwd, - const char* mechanism, - const char* authDb, - bool dbSSL, - bool dbDisableRetryWrites, bool mtenant, - int64_t timeout, int writeConcern = 1, int poolSize = 10, bool semTimeStat = false diff --git a/test/unittests/main_UnitTest.cpp b/test/unittests/main_UnitTest.cpp index 3b7718b0c1..8e1b5c1d8f 100644 --- a/test/unittests/main_UnitTest.cpp +++ b/test/unittests/main_UnitTest.cpp @@ -81,15 +81,8 @@ unsigned long logLineMaxSize = 32 * 1024; bool logDeprecate = false; char dbURI[1024]; -char dbHost[256]; -char rplSet[64]; char dbName[64]; -char user[64]; char pwd[64]; -char authMech[64]; -char authDb[64]; -bool dbSSL; -int64_t dbTimeout; int dbPoolSize; int writeConcern; char gtest_filter[1024]; @@ -109,15 +102,8 @@ unsigned long fcMaxInterval = 0; PaArgument paArgs[] = { { "-dbURI", dbURI, "DB_URI", PaString, PaOpt, (int64_t) "", PaNL, PaNL, "" }, - { "-dbhost", dbHost, "DB_HOST", PaString, PaOpt, (int64_t) "localhost", PaNL, PaNL, "" }, - { "-rplSet", rplSet, "RPL_SET", PaString, PaOpt, (int64_t) "", PaNL, PaNL, "" }, - { "-dbuser", user, "DB_USER", PaString, PaOpt, (int64_t) "", PaNL, PaNL, "" }, { "-dbpwd", pwd, "DB_PASSWORD", PaString, PaOpt, (int64_t) "", PaNL, PaNL, "" }, - { "-dbAuthMech", authMech, "DB_AUTH_MECH", PaString, PaOpt, (int64_t) "", PaNL, PaNL, "" }, - { "-dbAuthDb", authDb, "DB_AUTH_DB", PaString, PaOpt, (int64_t) "", PaNL, PaNL, "" }, - { "-dbSSL", &dbSSL, "DB_AUTH_SSL", PaBool, PaOpt, false, false, true, "" }, { "-db", dbName, "DB", PaString, PaOpt, (int64_t) "orion", PaNL, PaNL, "" }, - { "-dbTimeout", &dbTimeout, "DB_TIMEOUT", PaInt64, PaOpt, 0, PaNL, PaNL, "" }, { "-dbPoolSize", &dbPoolSize, "DB_POOL_SIZE", PaInt, PaOpt, 10, 1, 10000, "" }, { "-writeConcern", &writeConcern, "WRITE_CONCERN", PaInt, PaOpt, 1, 0, 1, "" }, { "--gtest_filter=", gtest_filter, "", PaString, PaOpt, (int64_t) "", PaNL, PaNL, "" }, @@ -159,8 +145,8 @@ int main(int argC, char** argV) LM_M(("Init tests")); orionInit(exitFunction, orionUnitTestVersion, SemReadWriteOp, false, false, false, false, false); - // Note that disableRetryTries, multitenancy and mutex time stats are disabled for unit test mongo init - mongoInit(dbURI, dbHost, rplSet, dbName, user, pwd, authMech, authDb, dbSSL, false, false, dbTimeout, writeConcern, dbPoolSize, false); + // Note that multitenancy and mutex time stats are disabled for unit test mongo init + mongoInit(dbURI, dbName, pwd, false, writeConcern, dbPoolSize, false); alarmMgr.init(false); logSummaryInit(&lsPeriod); // setupDatabase(); FIXME #3775: pending on mongo unit test re-enabling From 9558ac9487913e2dde2b43fa0a3bd73b4aa6d83b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 1 Mar 2024 09:50:55 +0100 Subject: [PATCH 190/390] FIX ftest --- .../cases/0000_cli/bool_option_with_value.test | 8 -------- .../cases/0000_cli/command_line_options.test | 8 -------- .../0000_cli/tracelevel_without_logLevel_as_DEBUG.test | 8 -------- test/functionalTest/cases/3658_env_vars/env_vars.test | 8 -------- 4 files changed, 32 deletions(-) diff --git a/test/functionalTest/cases/0000_cli/bool_option_with_value.test b/test/functionalTest/cases/0000_cli/bool_option_with_value.test index 157a33b97a..111fb8dde8 100644 --- a/test/functionalTest/cases/0000_cli/bool_option_with_value.test +++ b/test/functionalTest/cases/0000_cli/bool_option_with_value.test @@ -51,16 +51,8 @@ Usage: contextBroker [option '-U' (extended usage)] [option '-port' ] [option '-pidpath' ] [option '-dbURI' ] - [option '-dbhost' ] - [option '-rplSet' ] - [option '-dbuser' ] [option '-dbpwd' ] - [option '-dbAuthMech' ] - [option '-dbAuthDb' ] - [option '-dbSSL' (enable SSL connection to DB)] - [option '-dbDisableRetryWrites' (set retryWrite parameter to false in DB connections)] [option '-db' ] - [option '-dbTimeout' ] [option '-dbPoolSize' ] [option '-ipv4' (use ip v4 only)] [option '-ipv6' (use ip v6 only)] diff --git a/test/functionalTest/cases/0000_cli/command_line_options.test b/test/functionalTest/cases/0000_cli/command_line_options.test index 20a639175b..f97b786889 100644 --- a/test/functionalTest/cases/0000_cli/command_line_options.test +++ b/test/functionalTest/cases/0000_cli/command_line_options.test @@ -40,16 +40,8 @@ Usage: contextBroker [option '-U' (extended usage)] [option '-port' ] [option '-pidpath' ] [option '-dbURI' ] - [option '-dbhost' ] - [option '-rplSet' ] - [option '-dbuser' ] [option '-dbpwd' ] - [option '-dbAuthMech' ] - [option '-dbAuthDb' ] - [option '-dbSSL' (enable SSL connection to DB)] - [option '-dbDisableRetryWrites' (set retryWrite parameter to false in DB connections)] [option '-db' ] - [option '-dbTimeout' ] [option '-dbPoolSize' ] [option '-ipv4' (use ip v4 only)] [option '-ipv6' (use ip v6 only)] diff --git a/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test b/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test index cf566803ce..8c92e4fffc 100644 --- a/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test +++ b/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test @@ -41,16 +41,8 @@ Usage: contextBroker [option '-U' (extended usage)] [option '-port' ] [option '-pidpath' ] [option '-dbURI' ] - [option '-dbhost' ] - [option '-rplSet' ] - [option '-dbuser' ] [option '-dbpwd' ] - [option '-dbAuthMech' ] - [option '-dbAuthDb' ] - [option '-dbSSL' (enable SSL connection to DB)] - [option '-dbDisableRetryWrites' (set retryWrite parameter to false in DB connections)] [option '-db' ] - [option '-dbTimeout' ] [option '-dbPoolSize' ] [option '-ipv4' (use ip v4 only)] [option '-ipv6' (use ip v6 only)] diff --git a/test/functionalTest/cases/3658_env_vars/env_vars.test b/test/functionalTest/cases/3658_env_vars/env_vars.test index 2ede8f83af..4c8d70bc64 100644 --- a/test/functionalTest/cases/3658_env_vars/env_vars.test +++ b/test/functionalTest/cases/3658_env_vars/env_vars.test @@ -86,16 +86,8 @@ Extended Usage: contextBroker [option '-U' (extended usage)] [option '-port' ] ORION_PORT 1 <= 1026 /1026/ <= 65535 [option '-pidpath' ] ORION_PID_PATH '/tmp/contextBroker.pid' /'/tmp/contextBroker/ [option '-dbURI' ] ORION_MONGO_URI '' /''/ - [option '-dbhost' ] ORION_MONGO_HOST 'localhost' /'localhost'/ - [option '-rplSet' ] ORION_MONGO_REPLICA_SET '' /''/ - [option '-dbuser' ] ORION_MONGO_USER '' /''/ [option '-dbpwd' ] ORION_MONGO_PASSWORD '' /''/ - [option '-dbAuthMech' ] ORION_MONGO_AUTH_SOURCE '' /''/ - [option '-dbSSL' (enable SSL connection to DB)] ORION_MONGO_SSL FALSE /FALSE/ - [option '-dbDisableRetryWrites' (set retryWrite parameter to false in DB connect] ORION_MONGO_DISABLE_RETRY_WRITES FALSE /FALSE/ [option '-db' ] ORION_MONGO_DB 'orion' /'orion'/ - [option '-dbTimeout' ] ORION_MONGO_POOL_SIZE 1 <= 10 /10/ <= 10000 [option '-ipv4' (use ip v4 only)] ORION_USEIPV4 FALSE /FALSE/ [option '-ipv6' (use ip v6 only)] ORION_USEIPV6 FALSE /FALSE/ From 2312fc43c0649f912ae7b3b599e6ae8ec3c1ba31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 1 Mar 2024 16:50:18 +0100 Subject: [PATCH 191/390] FIX default dbURI --- src/app/contextBroker/contextBroker.cpp | 3 +- .../cases/3658_env_vars/env_vars.test | 116 +++++++++--------- 2 files changed, 59 insertions(+), 60 deletions(-) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index b8ede361f5..1bd88da146 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -208,7 +208,6 @@ bool logDeprecate; */ #define PIDPATH _i "/tmp/contextBroker.pid" #define IP_ALL _i "0.0.0.0" -#define LOCALHOST _i "localhost" #define ONE_MONTH_PERIOD (3600 * 24 * 31) #define FG_DESC "don't start as daemon" @@ -289,7 +288,7 @@ PaArgument paArgs[] = { "-port", &port, "PORT", PaInt, PaOpt, 1026, 1, 65535, PORT_DESC }, { "-pidpath", pidPath, "PID_PATH", PaString, PaOpt, PIDPATH, PaNL, PaNL, PIDPATH_DESC }, - { "-dbURI", dbURI, "MONGO_URI", PaString, PaOpt, _i "", PaNL, PaNL, DBURI_DESC }, + { "-dbURI", dbURI, "MONGO_URI", PaString, PaOpt, _i "mongodb://localhost:27017", PaNL, PaNL, DBURI_DESC }, { "-dbpwd", pwd, "MONGO_PASSWORD", PaString, PaOpt, _i "", PaNL, PaNL, DBPASSWORD_DESC }, { "-db", dbName, "MONGO_DB", PaString, PaOpt, _i "orion", PaNL, PaNL, DB_DESC }, diff --git a/test/functionalTest/cases/3658_env_vars/env_vars.test b/test/functionalTest/cases/3658_env_vars/env_vars.test index 4c8d70bc64..60222b9c14 100644 --- a/test/functionalTest/cases/3658_env_vars/env_vars.test +++ b/test/functionalTest/cases/3658_env_vars/env_vars.test @@ -73,64 +73,64 @@ echo --REGEXPECT-- 01. Get the extended usage from the broker withut any alterations - see default state ===================================================================================== -Extended Usage: contextBroker [option '-U' (extended usage)] TRUE /FALSE/ - [option '-u' (usage)] FALSE /FALSE/ - [option '--help' (show help)] FALSE /FALSE/ - [option '--version' (show version)] FALSE /FALSE/ - [option '-logDir' ] ORION_LOG_DIR '/tmp/' - [option '-t' ] ORION_TRACE '' /'(null)'/ - [option '-logLevel' ] ORION_LOG_LEVEL 'WARN' /'WARN'/ - [option '-logAppend' (append to log-file)] ORION_LOG_APPEND FALSE /FALSE/ - [option '-fg' (don't start as daemon)] ORION_FOREGROUND FALSE /FALSE/ - [option '-localIp' ] ORION_LOCALIP '0.0.0.0' /'0.0.0.0'/ - [option '-port' ] ORION_PORT 1 <= 1026 /1026/ <= 65535 - [option '-pidpath' ] ORION_PID_PATH '/tmp/contextBroker.pid' /'/tmp/contextBroker/ - [option '-dbURI' ] ORION_MONGO_URI '' /''/ - [option '-dbpwd' ] ORION_MONGO_PASSWORD '' /''/ - [option '-db' ] ORION_MONGO_DB 'orion' /'orion'/ - [option '-dbPoolSize' ] ORION_MONGO_POOL_SIZE 1 <= 10 /10/ <= 10000 - [option '-ipv4' (use ip v4 only)] ORION_USEIPV4 FALSE /FALSE/ - [option '-ipv6' (use ip v6 only)] ORION_USEIPV6 FALSE /FALSE/ - [option '-https' (use the https 'protocol')] ORION_HTTPS FALSE /FALSE/ - [option '-key' ] ORION_HTTPS_KEYFILE '' /''/ - [option '-cert' ] ORION_HTTPS_CERTFILE '' /''/ - [option '-multiservice' (service multi tenancy mode)] ORION_MULTI_SERVICE FALSE /FALSE/ - [option '-httpTimeout' ] ORION_REQ_TIMEOUT 0 /0/ >= 0 - [option '-reqMutexPolicy' ] ORION_MUTEX_POLICY 'all' /'all'/ - [option '-writeConcern' ] ORION_MONGO_WRITE_CONCERN 0 <= 1 /1/ <= 1 - [option '-corsOrigin' ] ORION_CONN_MEMORY 0 <= 64 /64/ <= 1024 - [option '-maxConnections' ] ORION_MAX_CONN 1020 /1020/ >= 1 - [option '-reqPoolSize' ] ORION_TRQ_POOL_SIZE 0 <= 0 /0/ <= 1024 - [option '-inReqPayloadMaxSize' = 0 - [option '-outReqMsgMaxSize' = 0 - [option '-notificationMode' ] ORION_LOG_LINE_MAX_SIZE 32768 /32768/ >= 100 - [option '-logInfoPayloadMaxSize' = 0 - [option '-disableMetrics' (turn off the 'metrics' feature)] ORION_DISABLE_METRICS FALSE /FALSE/ - [option '-disableNgsiv1' (turn off NGSIv1 request endpoints)] ORION_DISABLE_NGSIV1 FALSE /FALSE/ - [option '-insecureNotif' (allow HTTPS notifications to peers which certificate c] ORION_INSECURE_NOTIF FALSE /FALSE/ - [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSI] ORION_NGSIV1_AUTOCAST FALSE /FALSE/ - [option '-mqttMaxAge' ] ORION_LOG_DIR '/tmp/' + [option '-t' ] ORION_TRACE '' /'(null)'/ + [option '-logLevel' ] ORION_LOG_LEVEL 'WARN' /'WARN'/ + [option '-logAppend' (append to log-file)] ORION_LOG_APPEND FALSE /FALSE/ + [option '-fg' (don't start as daemon)] ORION_FOREGROUND FALSE /FALSE/ + [option '-localIp' ] ORION_LOCALIP '0.0.0.0' /'0.0.0.0'/ + [option '-port' ] ORION_PORT 1 <= 1026 /1026/ <= 65535 + [option '-pidpath' ] ORION_PID_PATH '/tmp/contextBroker.pid' /'/tmp/contextBroker/ + [option '-dbURI' ] ORION_MONGO_URI 'mongodb://localhost:27017' /'mongodb://localhos/ + [option '-dbpwd' ] ORION_MONGO_PASSWORD '' /''/ + [option '-db' ] ORION_MONGO_DB 'orion' /'orion'/ + [option '-dbPoolSize' ] ORION_MONGO_POOL_SIZE 1 <= 10 /10/ <= 10000 + [option '-ipv4' (use ip v4 only)] ORION_USEIPV4 FALSE /FALSE/ + [option '-ipv6' (use ip v6 only)] ORION_USEIPV6 FALSE /FALSE/ + [option '-https' (use the https 'protocol')] ORION_HTTPS FALSE /FALSE/ + [option '-key' ] ORION_HTTPS_KEYFILE '' /''/ + [option '-cert' ] ORION_HTTPS_CERTFILE '' /''/ + [option '-multiservice' (service multi tenancy mode)] ORION_MULTI_SERVICE FALSE /FALSE/ + [option '-httpTimeout' ] ORION_REQ_TIMEOUT 0 /0/ >= 0 + [option '-reqMutexPolicy' ] ORION_MUTEX_POLICY 'all' /'all'/ + [option '-writeConcern' ] ORION_MONGO_WRITE_CONCERN 0 <= 1 /1/ <= 1 + [option '-corsOrigin' ] ORION_CONN_MEMORY 0 <= 64 /64/ <= 1024 + [option '-maxConnections' ] ORION_MAX_CONN 1020 /1020/ >= 1 + [option '-reqPoolSize' ] ORION_TRQ_POOL_SIZE 0 <= 0 /0/ <= 1024 + [option '-inReqPayloadMaxSize' = 0 + [option '-outReqMsgMaxSize' = 0 + [option '-notificationMode' ] ORION_LOG_LINE_MAX_SIZE 32768 /32768/ >= 100 + [option '-logInfoPayloadMaxSize' = 0 + [option '-disableMetrics' (turn off the 'metrics' feature)] ORION_DISABLE_METRICS FALSE /FALSE/ + [option '-disableNgsiv1' (turn off NGSIv1 request endpoints)] ORION_DISABLE_NGSIV1 FALSE /FALSE/ + [option '-insecureNotif' (allow HTTPS notifications to peers which certificate c] ORION_INSECURE_NOTIF FALSE /FALSE/ + [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSI] ORION_NGSIV1_AUTOCAST FALSE /FALSE/ + [option '-mqttMaxAge' Date: Sat, 2 Mar 2024 07:56:58 +0900 Subject: [PATCH 192/390] (JP) REMOVE deprecated db related CLIs (#4527) --- doc/manuals.jp/deprecated.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manuals.jp/deprecated.md b/doc/manuals.jp/deprecated.md index a515871299..535ebbf998 100644 --- a/doc/manuals.jp/deprecated.md +++ b/doc/manuals.jp/deprecated.md @@ -8,7 +8,7 @@ 推奨されなくなった機能のリストと、廃止された機能のバージョンは次のとおりです : -* Orion 3.12.0 での CLI パラメータ (および関連する環境変数): `-dbhost`、`-rplSet`、`-dbTimeout`、`-dbuser`、`-dbAuthMech`、`-dbAuthDb`、`-dbSSL`、および `-dbDisableRetryWrites`。MongoDB URI を構築するために必要な情報が必要な場合は、[このセクション](#mapping-to-mongouri-from-old-cli-parameters) をチェックして、代わりに `dbURI` を使用してください +* Orion 3.12.0 での CLI パラメータ (および関連する環境変数): `-dbhost`、`-rplSet`、`-dbTimeout`、`-dbuser`、`-dbAuthMech`、`-dbAuthDb`、`-dbSSL`、および `-dbDisableRetryWrites`。MongoDB URI を構築するために必要な情報が必要な場合は、[このセクション](#mapping-to-mongouri-from-old-cli-parameters) をチェックして、代わりに `dbURI` を使用してください (Orion 3.13.0 で削除されました) * Orion 3.10.0 での `geo:point`, `geo:line`, `geo:box` および `geo:polygon` 属性タイプ。代わりに `geo:json` を使用してください * Orion 3.8.0 での `GET /v2` 操作。この操作はかなり役に立たず、実際には使用されません。 * Orion 3.1.0 のサブスクリプションでの初期通知 (`skipInitialNotification` オプションと共に)。(Orion 3.2.0 で削除)。初期通知の @@ -103,13 +103,13 @@ Notes: | **削除された機能** | **機能をサポートする Orion ラスト・バージョン** | **バージョンのリリース日** | |--------------------------------------------------------------------------------------|-------------------------------------------------|----------------------------| -| CLI `-dbhost`、`-rplSet`、`-dbTimeout`、`-dbuser`、`-dbAuthMech`、`-dbAuthDb`、`-dbSSL`、および `-dbDisableRetryWrites` (および関連する環境変数) | まだ定義されていません | まだ定義されていません | | `POST /v2/entities` オペレーションの `attributes` フィールド | まだ定義されていません | まだ定義されていません | | `APPEND`, `UPDATE`, など。`POST /v2/op/update` でのアクション・タイプ | まだ定義されていません | まだ定義されていません | | URI パラメータでの `dateCreated` および `dateModified` | まだ定義されていません | まだ定義されていません | | エンティティのロケーションを指定する `location` メタデータ | まだ定義されていません | まだ定義されていません | | `GET /v2` 操作 | まだ定義されていません | まだ定義されていません | | `geo:point`, `geo:line`, `geo:box` および `geo:polygon` 属性タイプ | まだ定義されていません | まだ定義されていません | +| CLI `-dbhost`、`-rplSet`、`-dbTimeout`、`-dbuser`、`-dbAuthMech`、`-dbAuthDb`、`-dbSSL`、および `-dbDisableRetryWrites` (および関連する環境変数) | 3.12.0 | 2024年2月29日 | | エンティティの場所を指定するための `location` メタデータ | 3.10.1 | 2023年6月12日 | | NGSIv1 (関連する CLI パラメータ : `-strictNgsiv1Ids`, `-ngsiv1Autocast`) | 3.9.0 (*) | 2023年6月2日 | | `/ngsi10` および `/ngsi9` URL プレフィックス | 3.7.0 (*) | 2022年5月26日 | From 238667780e62055d18643c4f9b18f3de9c5c11f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 5 Mar 2024 18:56:14 +0100 Subject: [PATCH 193/390] FIX ftest for legacy forced test --- src/lib/expressions/ExprContext.cpp | 4 +- src/lib/ngsiNotify/Notifier.cpp | 29 +- ...n_templates_many_notifications_legacy.test | 449 ++++++++++++++++++ .../covered_custom_notification_legacy.test | 217 +++++++++ 4 files changed, 689 insertions(+), 10 deletions(-) create mode 100644 test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test create mode 100644 test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 43737af844..46e85c5e1e 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -217,7 +217,7 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex return; } // FIXME P3: improve this implemeting a toString() method for ExprContextObject - LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=[object]", key.c_str()); + LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=[object]", key.c_str())); PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.getJexlContext()); } } @@ -400,7 +400,7 @@ void ExprContextList::add(ExprContextObject exprContextObject) return; } // FIXME P3: improve this implemeting a toString() method for ExprContextObject - LM_T(LmtExpr, ("adding to JEXL expression context list (object): [object]"))); + LM_T(LmtExpr, ("adding to JEXL expression context list (object): [object]")); PyList_Append(jexlContext, exprContextObject.getJexlContext()); } diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 8c09270602..a2f0c14071 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -319,18 +319,31 @@ static SenderThreadParams* buildSenderParamsCustom // Used by several macroSubstitute() calls along this function // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode - //ExprContextObject exprContext(exprLang == "legacy"); - ExprContextObject exprContext(true); + //bool legacy = (exprLang == "legacy"); + bool legacy = true; + ExprContextObject exprContext(legacy); + + // It seems that add() semantics are different in legacy and jexl mode. In jexl mode, if the key already exists, it is + // updated. In legacy model, if the key already exists, the operation is ignored (so previous value is preserved). Taking + // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence + // over macros comming from macros of the same name we conditionally add them depending the case exprContext.add("id", en.id); exprContext.add("type", en.type); - exprContext.add("service", tenant); - exprContext.add("servicePath", en.servicePath); - exprContext.add("authToken", xauthToken); + if (!legacy) + { + exprContext.add("service", tenant); + exprContext.add("servicePath", en.servicePath); + exprContext.add("authToken", xauthToken); + } for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode - //en.attributeVector[ix]->addToContext(&exprContext, exprLang == "legacy"); - en.attributeVector[ix]->addToContext(&exprContext, true); + en.attributeVector[ix]->addToContext(&exprContext, legacy); + } + if (legacy) + { + exprContext.add("service", tenant); + exprContext.add("servicePath", en.servicePath); + exprContext.add("authToken", xauthToken); } // diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test new file mode 100644 index 0000000000..d624115962 --- /dev/null +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test @@ -0,0 +1,449 @@ +# Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Notification Templates - many notifications (legacy expressions + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create regular subscription for E1 +# 02. Create same subscription but with 'httpCustom' instead of 'http' +# 03. Create custom subscription for E1 +# 04. Create E1/T1 +# 05. Create E1/T2 +# 06. Create E1/T3 +# 07. Dump accumulator and see 9 notifications +# + +echo "01. Create regular subscription for E1" +echo "======================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1" + } + ], + "condition": { + "attrs": [] + } + }, + "notification": { + "http": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + } + }, + "throttling": 0 +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create same subscription but with 'httpCustom' instead of 'http'" +echo "====================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1" + } + ], + "condition": { + "attrs": [] + } + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + }, + "exprLang": "legacy" + }, + "throttling": 0 +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create custom subscription for E1" +echo "=====================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1" + } + ], + "condition": { + "attrs": [] + } + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "method": "PUT", + "payload": "{ %22A1%22: %22${A1}%22, %22A2%22: %22${A2}%22, %22A3%22: %22${A3}%22 }", + "qs": { "id": "${id}", "type": "${type}", "a1": "${A1}", "a2": "${A2}", "a3": "${A3}" }, + "headers": { "entity-id": "${id}", "entity-type": "${type}", "A1": "${A1}", "A2": "${A2}", "A3": "${A3}" }, + "exprLang": "legacy" + } + }, + "throttling": 0 +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "04. Create E1/T1" +echo "================" +payload='{ + "id": "E1", + "type": "T1", + "A1": true +}' +orionCurl --url /v2/entities?options=keyValues --payload "$payload" +echo +echo + + +echo "05. Create E1/T2" +echo "================" +payload='{ + "id": "E1", + "type": "T2", + "A2": null +}' +orionCurl --url /v2/entities?options=keyValues --payload "$payload" +echo +echo + + +echo "06. Create E1/T3" +echo "================" +payload='{ + "id": "E1", + "type": "T3", + "A3": 3 +}' +orionCurl --url /v2/entities?options=keyValues --payload "$payload" +echo +echo + + +echo "07. Dump accumulator and see 9 notifications" +echo "============================================" +sleep 0.5 # Without this sleep, sometimes the last notification is missing +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create regular subscription for E1 +====================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create same subscription but with 'httpCustom' instead of 'http' +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom subscription for E1 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create E1/T1 +================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T1 +Content-Length: 0 + + + +05. Create E1/T2 +================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T2 +Content-Length: 0 + + + +06. Create E1/T3 +================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T3 +Content-Length: 0 + + + +07. Dump accumulator and see 9 notifications +#SORT_START +============================================ +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 129 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A1": { + "metadata": {}, + "type": "Boolean", + "value": true + }, + "id": "E1", + "type": "T1" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 129 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A1": { + "metadata": {}, + "type": "Boolean", + "value": true + }, + "id": "E1", + "type": "T1" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 +Fiware-Servicepath: / +Entity-Id: E1 +Content-Length: 36 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +A1: true +Entity-Type: T1 +Content-Type: text/plain; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "A1": "true", + "A2": "", + "A3": "" +} +#SORT_END +#SORT_START +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 126 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A2": { + "metadata": {}, + "type": "None", + "value": null + }, + "id": "E1", + "type": "T2" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 126 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A2": { + "metadata": {}, + "type": "None", + "value": null + }, + "id": "E1", + "type": "T2" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +PUT http://127.0.0.1:REGEX(\d+)/notify?a2=null&id=E1&type=T2 +Fiware-Servicepath: / +Entity-Id: E1 +Content-Length: 36 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +A2: null +Entity-Type: T2 +Content-Type: text/plain; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "A1": "", + "A2": "null", + "A3": "" +} +#SORT_END +#SORT_START +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 125 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A3": { + "metadata": {}, + "type": "Number", + "value": 3 + }, + "id": "E1", + "type": "T3" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 125 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "data": [ + { + "A3": { + "metadata": {}, + "type": "Number", + "value": 3 + }, + "id": "E1", + "type": "T3" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 +Fiware-Servicepath: / +A3: 3 +Entity-Id: E1 +Content-Length: 33 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Entity-Type: T3 +Content-Type: text/plain; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) + +{ + "A1": "", + "A2": "", + "A3": "3" +} +======================================= +#SORT_END + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test new file mode 100644 index 0000000000..8321f78322 --- /dev/null +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test @@ -0,0 +1,217 @@ +# Copyright 2022 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Covered custom notifictions (legacy expressions) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create covered custom subscriptions for E1 covering attributes A1 and A2 +# 02. Create not covered custom subscriptions for E2 +# 03. Create E1 with attribute A1=1 +# 04. Dump & reset, see notifications A1=1 and A2=null in custom payload +# 05. Create E2 with attribute A1=1 +# 06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) +# + +echo "01. Create covered custom subscriptions for E1 covering attributes A1 and A2" +echo "============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "covered": true, + "attrs": [ "A1", "A2" ], + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "payload": "{ %22A1-value%22: ${A1}, %22A2-value%22: ${A2} }", + "headers": { + "content-type": "application/json%3B charset%3Dutf-8" + }, + "exprLang": "legacy" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create not covered custom subscriptions for E2" +echo "==================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E2", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "payload": "{ %22A1-value%22: ${A1}, %22A2-value%22: ${A2} }", + "headers": { + "content-type": "application/json%3B charset%3Dutf-8" + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create E1 with attribute A1=1" +echo "=================================" +payload='{ + "id": "E1", + "type": "T", + "A1": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "04. Dump & reset, see notifications A1=1 and A2=null in custom payload" +echo "======================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +echo "05. Create E2 with attribute A1=1" +echo "=================================" +payload='{ + "id": "E2", + "type": "T", + "A1": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON)" +echo "========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create covered custom subscriptions for E1 covering attributes A1 and A2 +============================================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create not covered custom subscriptions for E2 +================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create E1 with attribute A1=1 +================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +04. Dump & reset, see notifications A1=1 and A2=null in custom payload +====================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 35 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ "A1-value": 1, "A2-value": null }======================================= + + +05. Create E2 with attribute A1=1 +================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) +======================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 31 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ "A1-value": 1, "A2-value": }======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 742936f9b54c2ee7a014041928e2fe659d5ef160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Mar 2024 09:42:17 +0100 Subject: [PATCH 194/390] FIX rollback hack to use always legacy expressions --- src/lib/ngsiNotify/Notifier.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index a2f0c14071..6ee599bd5f 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -318,9 +318,7 @@ static SenderThreadParams* buildSenderParamsCustom const std::string exprLang = notification.type == ngsiv2::HttpNotification ? notification.httpInfo.exprLang : notification.mqttInfo.exprLang; // Used by several macroSubstitute() calls along this function - // FIXME PR: force legacy mode to temporarly test all ftest in legacy mode - //bool legacy = (exprLang == "legacy"); - bool legacy = true; + bool legacy = (exprLang == "legacy"); ExprContextObject exprContext(legacy); // It seems that add() semantics are different in legacy and jexl mode. In jexl mode, if the key already exists, it is From ea94c572e53c673810011fd72c64f594004dbe5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 6 Mar 2024 16:15:40 +0100 Subject: [PATCH 195/390] FIX change pyjexl by tcjexl --- ci/deb/build-dep.sh | 4 +- src/lib/expressions/ExprManager.cpp | 18 +-- src/lib/expressions/ExprManager.h | 2 +- .../jexl_transformation.test | 144 ++++++++++++++++++ 4 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test diff --git a/ci/deb/build-dep.sh b/ci/deb/build-dep.sh index 4d35fe450e..28540d6fe6 100755 --- a/ci/deb/build-dep.sh +++ b/ci/deb/build-dep.sh @@ -55,7 +55,7 @@ echo "INSTALL: MongoDB shell" \ && apt-get -y update \ && apt-get -y install mongodb-mongosh -# pyjexl is needed by Context Broker functionality, the others are for ftest suite +# tcjexl is needed by Context Broker functionality, the others are for ftest suite echo "INSTALL: python special dependencies" \ && cd /opt \ && python3 -m venv /opt/ft_env \ @@ -64,7 +64,7 @@ echo "INSTALL: python special dependencies" \ && pip install Werkzeug==2.0.2 \ && pip install paho-mqtt==1.6.1 \ && pip install amqtt==0.11.0b1 \ -&& pip install pyjexl==0.3.0 \ +&& pip install tcjexl==0.1.0 \ && deactivate # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0 diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index bfe6d14b81..d87f37f536 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -40,7 +40,7 @@ */ void ExprManager::init(void) { - pyjexlModule = NULL; + tcjexlModule = NULL; jexlEngine = NULL; if (sem_init(&sem, 0, 1) == -1) @@ -51,14 +51,14 @@ void ExprManager::init(void) Py_Initialize(); LM_T(LmtExpr, ("Python interpreter has been initialized")); - pyjexlModule = PyImport_ImportModule("pyjexl"); - if (pyjexlModule == NULL) + tcjexlModule = PyImport_ImportModule("tcjexl"); + if (tcjexlModule == NULL) { - LM_X(1, ("Fatal Error (error importing pyjexl module: %s)", capturePythonError())); + LM_X(1, ("Fatal Error (error importing tcjexl module: %s)", capturePythonError())); } - LM_T(LmtExpr, ("pyjexl module has been loaded")); + LM_T(LmtExpr, ("tcjexl module has been loaded")); - jexlEngine = PyObject_CallMethod(pyjexlModule, "JEXL", NULL); + jexlEngine = PyObject_CallMethod(tcjexlModule, "JEXL", NULL); if (jexlEngine == NULL) { LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", capturePythonError())); @@ -134,10 +134,10 @@ void ExprManager::release(void) LM_T(LmtExpr, ("jexl engine has been freed")); } - if (pyjexlModule != NULL) + if (tcjexlModule != NULL) { - Py_XDECREF(pyjexlModule); - LM_T(LmtExpr, ("pyjexl module has been freed")); + Py_XDECREF(tcjexlModule); + LM_T(LmtExpr, ("tcjexl module has been freed")); } Py_Finalize(); diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 92525900e0..55c8e90f96 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -39,7 +39,7 @@ class ExprManager { private: - PyObject* pyjexlModule; + PyObject* tcjexlModule; PyObject* jexlEngine; PyObject* jsonModule; sem_t sem; diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test new file mode 100644 index 0000000000..693818d970 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test @@ -0,0 +1,144 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using transformation) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression low: A|lowercase +# 02. Create entity E1 with A=IwantAllInLOWERCase +# 03. Dump accumulator and see notifications (low: iwantallinlowercase) +# + + +echo "01. Create custom sub with custom expression low: A|lowercase" +echo "=============================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "low": { + "value": "${A|lowercase}", + "type": "Calculated" + } + } + }, + "attrs": [ "low" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=IwantAllInLOWERCase" +echo "===============================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": "IwantAllInLOWERCase", + "type": "Text" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Dump accumulator and see notifications (low: iwantallinlowercase)" +echo "=====================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression low: A|lowercase +============================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=IwantAllInLOWERCase +=============================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Dump accumulator and see notifications (low: iwantallinlowercase) +===================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 149 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "low": { + "metadata": {}, + "type": "Calculated", + "value": "iwantallinlowercase" + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From a71c173a3a9092b42d5df85e9bc61c9ce0311a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 7 Mar 2024 12:20:42 +0100 Subject: [PATCH 196/390] ADD ftest for null values and not defined attrs cases --- .../jexl_basic_attrs_in_update.test | 2 +- .../jexl_basic_attrs_not_in_update.test | 2 +- .../jexl_expr_attrs_weird_syntax.test | 2 +- .../jexl_json_navigation.test | 2 +- .../jexl_not_defined_attrs.test | 191 ++++++++++++++++++ .../jexl_null_values.test | 191 ++++++++++++++++++ 6 files changed, 386 insertions(+), 4 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test index 5e031e80e7..601f69d25a 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification (source attributes in udpate) +JEXL basic expression in custom notification (source attributes in udpate) --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test index eaef9821eb..f6c634a077 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification (source attributes not in udpate) +JEXL basic expression in custom notification (source attributes not in udpate) --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test index cf42ab7ebe..ac53119941 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -Legacy expression using attributes with weird syntax +JEXL expression using attributes with weird syntax --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test index 236b872365..05e8a0b061 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification (source attributes not in udpate) +JEXL expression in custom notification (JSON navigation in objects and arrays) --SHELL-INIT-- dbInit CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test new file mode 100644 index 0000000000..4fe7de3997 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test @@ -0,0 +1,191 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (not defined attributes) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression cal: A+1 +# 02. Create entity E1 with B=1 +# 03. Update entity E1 with B=2 +# 04. Dump accumulator and see notifications (cal: null, cal: null) +# + + +echo "01. Create custom sub with custom expression cal: A+1" +echo "=====================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "cal": { + "value": "${A+1}", + "type": "Calculated" + } + } + }, + "attrs": [ "cal" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with B=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with B=2" +echo "=============================" +payload='{ + "B": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (cal: null, cal: null)" +echo "=================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression cal: A+1 +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with B=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with B=2 +============================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (cal: null, cal: null) +================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test new file mode 100644 index 0000000000..35bfbe4567 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test @@ -0,0 +1,191 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (null values) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression cal: A+1 +# 02. Create entity E1 with A=null +# 03. Update entity E1 with A=null again (with forcedUpdate) +# 04. Dump accumulator and see notifications (cal: null, cal: null) +# + + +echo "01. Create custom sub with custom expression cal: A+1" +echo "=====================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "cal": { + "value": "${A+1}", + "type": "Calculated" + } + } + }, + "attrs": [ "cal" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=null" +echo "================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A=null again (with forcedUpdate)" +echo "==========================================================" +payload='{ + "A": { + "value": "null", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs?options=forcedUpdate -X POST --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (cal: null, cal: null)" +echo "=================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression cal: A+1 +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A=null again (with forcedUpdate) +========================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (cal: null, cal: null) +================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From e6f0e6bdcf328fd623d0feed38bd78d71728af77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 7 Mar 2024 12:46:27 +0100 Subject: [PATCH 197/390] ADD ftest for array filtering --- .../jexl_array_filtering.test | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test new file mode 100644 index 0000000000..c20beb102f --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test @@ -0,0 +1,297 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (array filtering) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression sum: a[.b<5] +# 02. Create entity E1 with a=[{b:1},{b:34},{b:4}] +# 03. Update entity E1 with a=[{c:1},{b:-1},{b:2}] +# 04. Update entity E1 with a=[{b:-1},{b:foo}] +# 05. Update entity E1 with a=[{b:-1, c:1},{b:20, d:3}] +# 06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}]) +# + + +echo "01. Create custom sub with custom expression sum: a[.b<5]" +echo "=========================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "cal": { + "value": "${a[.b<5]}", + "type": "TextUnrestricted" + } + } + }, + "attrs": [ "cal" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with a=[{b:1},{b:34},{b:4}]" +echo "================================================" +payload='{ + "id": "E1", + "type": "T", + "a": { + "value": [{"b": 1}, {"b": 34}, {"b": 4}], + "type": "Array" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with a=[{c:1},{b:-1},{b:2}]" +echo "================================================" +payload='{ + "a": { + "value": [{"c": 1}, {"b": -1}, {"b": 2}], + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "04. Update entity E1 with a=[{b:-1},{b:foo}]" +echo "============================================" +payload='{ + "a": { + "value": [{"b": -1}, {"b": "foo"}], + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "05. Update entity E1 with a=[{b:-1, c:1},{b:20, d:3}]" +echo "=====================================================" +payload='{ + "a": { + "value": [{"b": -1, "c": 1}, {"b": 20, "d": 3}], + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}])" +echo "=====================================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: a[.b<5] +========================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with a=[{b:1},{b:34},{b:4}] +================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with a=[{c:1},{b:-1},{b:2}] +================================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Update entity E1 with a=[{b:-1},{b:foo}] +============================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Update entity E1 with a=[{b:-1, c:1},{b:20, d:3}] +===================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}]) +===================================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 151 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "TextUnrestricted", + "value": [ + { + "b": 1 + }, + { + "b": 4 + } + ] + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 138 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "TextUnrestricted", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 138 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "TextUnrestricted", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 150 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "cal": { + "metadata": {}, + "type": "TextUnrestricted", + "value": [ + { + "b": -1, + "c": 1 + } + ] + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 17740eee9886a9ca2f17fb5c88c22385bea73543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 7 Mar 2024 14:25:25 +0100 Subject: [PATCH 198/390] ADD nameOnly to listDatabase command on MongoDB --- CHANGES_NEXT_RELEASE | 1 + src/lib/mongoBackend/MongoGlobal.cpp | 1 + src/lib/mongoDriver/mongoConnectionPool.cpp | 9 ++++----- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e69de29bb2..25236adf1a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -0,0 +1 @@ +- Fix: lighter operation to get databases list from MongoDB (#4517) \ No newline at end of file diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp index 85308f228b..09b920c0d2 100644 --- a/src/lib/mongoBackend/MongoGlobal.cpp +++ b/src/lib/mongoBackend/MongoGlobal.cpp @@ -232,6 +232,7 @@ bool getOrionDatabases(std::vector* dbsP) orion::BSONObjBuilder bob; bob.append("listDatabases", 1); + bob.append("nameOnly", true); if (!orion::runDatabaseCommand("admin", bob.obj(), &result, &err)) { diff --git a/src/lib/mongoDriver/mongoConnectionPool.cpp b/src/lib/mongoDriver/mongoConnectionPool.cpp index 855896753e..3c2c7cff9b 100644 --- a/src/lib/mongoDriver/mongoConnectionPool.cpp +++ b/src/lib/mongoDriver/mongoConnectionPool.cpp @@ -112,22 +112,21 @@ static int pingConnection(mongoc_client_t* conn, const char* db, bool mtenat) // ping will be listDatabases in the admin DB. But if we run in not mtenant // mode listCollections in the default database will suffice - std::string cmd; std::string effectiveDb; + bson_t* ping = bson_new(); if (mtenat) { cmd = "listDatabases"; + BSON_APPEND_INT32(ping, "listDatabases", 1); + BSON_APPEND_BOOL(ping, "nameOnly", true); effectiveDb = "admin"; } else { - cmd = "listCollections"; + BSON_APPEND_INT32(ping, "listCollections", 1); effectiveDb = db; } - - bson_t* ping = bson_new(); - BSON_APPEND_INT32(ping, cmd.c_str(), 1); mongoc_database_t* database = mongoc_client_get_database(conn, effectiveDb.c_str()); bson_error_t error; From 55551083cf971fffab63e81607e0f6cc8a1bd596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 7 Mar 2024 14:59:17 +0100 Subject: [PATCH 199/390] FIX broken build --- src/lib/mongoDriver/mongoConnectionPool.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/mongoDriver/mongoConnectionPool.cpp b/src/lib/mongoDriver/mongoConnectionPool.cpp index 3c2c7cff9b..4359910710 100644 --- a/src/lib/mongoDriver/mongoConnectionPool.cpp +++ b/src/lib/mongoDriver/mongoConnectionPool.cpp @@ -112,18 +112,20 @@ static int pingConnection(mongoc_client_t* conn, const char* db, bool mtenat) // ping will be listDatabases in the admin DB. But if we run in not mtenant // mode listCollections in the default database will suffice + std::string cmdString; std::string effectiveDb; bson_t* ping = bson_new(); if (mtenat) { - cmd = "listDatabases"; + cmdString = "listDatabases"; BSON_APPEND_INT32(ping, "listDatabases", 1); BSON_APPEND_BOOL(ping, "nameOnly", true); effectiveDb = "admin"; } else { + cmdString = "listCollections"; BSON_APPEND_INT32(ping, "listCollections", 1); effectiveDb = db; } @@ -133,7 +135,7 @@ static int pingConnection(mongoc_client_t* conn, const char* db, bool mtenat) int r = 0; if (!mongoc_database_command_with_opts(database, ping, NULL, NULL, NULL, &error)) { - LM_T(LmtMongo, ("ping command (%s at db %s) falied: %s", cmd.c_str(), effectiveDb.c_str(), error.message)); + LM_T(LmtMongo, ("ping command (%s at db %s) falied: %s", cmdString.c_str(), effectiveDb.c_str(), error.message)); // Reference error message: Authentication failed // Reference error message: command listCollections requires authentication if ((strstr(error.message, "Authentication failed") != NULL) || (strstr(error.message, "requires authentication") != NULL)) { From 3a1474ab03096e92ea6c74afc67ea144f38664bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 11 Mar 2024 11:37:44 +0100 Subject: [PATCH 200/390] FIX upgrade tcjexl to 0.2.0 in CI Dockerfile --- ci/deb/build-dep.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deb/build-dep.sh b/ci/deb/build-dep.sh index 28540d6fe6..aac74f5988 100755 --- a/ci/deb/build-dep.sh +++ b/ci/deb/build-dep.sh @@ -64,7 +64,7 @@ echo "INSTALL: python special dependencies" \ && pip install Werkzeug==2.0.2 \ && pip install paho-mqtt==1.6.1 \ && pip install amqtt==0.11.0b1 \ -&& pip install tcjexl==0.1.0 \ +&& pip install tcjexl==0.2.0 \ && deactivate # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0 From 0c6fcd4d10c171c270c8346ee23626772ce69b29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 11 Mar 2024 12:31:58 +0100 Subject: [PATCH 201/390] ADD ftest jexl_transformation_with_arguments --- .../jexl_transformation_with_arguments.test | 250 ++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test new file mode 100644 index 0000000000..85075dc93a --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test @@ -0,0 +1,250 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using transformation with arguments) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression included: S|includes(R) +# 02. Create entity E1 with S=a given string, R=given +# 03. Update entity E1 with S=anotherstring, R=strong +# 04. Update entity E1 with S=morestringarecoming, R=string +# 05. Dump accumulator and see notifications (included: true, included: false, include: true) +# + + +echo "01. Create custom sub with custom expression included: S|includes(R)" +echo "====================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "includes": { + "value": "${S|includes(R)}", + "type": "TextUnrestricted" + } + } + }, + "attrs": [ "includes" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with S=a given string, R=given" +echo "===================================================" +payload='{ + "id": "E1", + "type": "T", + "S": { + "value": "a given string", + "type": "Text" + }, + "R": { + "value": "given", + "type": "Text" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with S=anotherstring, R=strong" +echo "===================================================" +payload='{ + "S": { + "value": "anotherstring", + "type": "Text" + }, + "R": { + "value": "strong", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" +echo +echo + + +echo "04. Update entity E1 with S=morestringarecoming, R=string" +echo "=========================================================" +payload='{ + "S": { + "value": "morestringarecoming", + "type": "Text" + }, + "R": { + "value": "string", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see notifications (included: true, included: false, include: true)" +echo "===========================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression included: S|includes(R) +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with S=a given string, R=given +=================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with S=anotherstring, R=strong +=================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Update entity E1 with S=morestringarecoming, R=string +========================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see notifications (included: true, included: false, include: true) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 143 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "includes": { + "metadata": {}, + "type": "TextUnrestricted", + "value": true + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 144 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "includes": { + "metadata": {}, + "type": "TextUnrestricted", + "value": false + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 143 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "includes": { + "metadata": {}, + "type": "TextUnrestricted", + "value": true + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 2cc242432e489449dde8a4e1de7e94c2e0c75486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 11 Mar 2024 17:31:40 +0100 Subject: [PATCH 202/390] FIX remove forbidden chars restrictions in ngsi subfields --- doc/manuals/orion-api.md | 3 +- src/lib/jsonParseV2/parseContextAttribute.cpp | 5 +- src/lib/jsonParseV2/parseContextAttribute.h | 3 +- src/lib/jsonParseV2/parseSubscription.cpp | 14 +- src/lib/ngsi/ContextAttribute.cpp | 6 +- src/lib/ngsi/ContextAttribute.h | 2 +- .../jexl_array_filtering.test | 18 +- .../jexl_transformation_in_id_and_type.test | 197 ++++++++++++++++++ .../jexl_transformation_with_arguments.test | 14 +- 9 files changed, 227 insertions(+), 35 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 4fbc8d5ab0..50493f53c1 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -398,11 +398,12 @@ the following will be used (note that "%25" is the encoding for "%"). GET /v2/entities/E%253C01%253E ``` -There are some exception cases in which the above restrictions do not apply. In particular, in the following fields: +There are some exception cases in which the above restrictions do not apply. In particular, in the following cases: * URL parameter `q` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `mq` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `georel` and `coords` allow `;` +* Within `ngsi` (i.e. `id`, `type` and attribute values) in [NGSI Payload patching](#ngsi-payload-patching) (to support characters used in the JEXL expression syntax) * Whichever attribute value which uses `TextUnrestricted` as attribute type (see [Special Attribute Types](#special-attribute-types) section) ## Identifiers syntax restrictions diff --git a/src/lib/jsonParseV2/parseContextAttribute.cpp b/src/lib/jsonParseV2/parseContextAttribute.cpp index 751dfb66d3..caf6ac190a 100644 --- a/src/lib/jsonParseV2/parseContextAttribute.cpp +++ b/src/lib/jsonParseV2/parseContextAttribute.cpp @@ -254,7 +254,8 @@ std::string parseContextAttribute ConnectionInfo* ciP, const rapidjson::Value::ConstMemberIterator& iter, ContextAttribute* caP, - bool checkAttrSpecialTypes + bool checkAttrSpecialTypes, + bool relaxForbiddenCheck ) { std::string name = iter->name.GetString(); @@ -383,7 +384,7 @@ std::string parseContextAttribute caP->type = (compoundVector)? defaultType(orion::ValueTypeVector) : defaultType(caP->valueType); } - std::string r = caP->check(ciP->apiVersion, ciP->requestType); + std::string r = caP->check(ciP->apiVersion, ciP->requestType, relaxForbiddenCheck); if (r != "OK") { alarmMgr.badInput(clientIp, "JSON Parse Error", r); diff --git a/src/lib/jsonParseV2/parseContextAttribute.h b/src/lib/jsonParseV2/parseContextAttribute.h index e3210e9403..2ab0ed39e6 100644 --- a/src/lib/jsonParseV2/parseContextAttribute.h +++ b/src/lib/jsonParseV2/parseContextAttribute.h @@ -42,7 +42,8 @@ extern std::string parseContextAttribute ConnectionInfo* ciP, const rapidjson::Value::ConstMemberIterator& iter, ContextAttribute* caP, - bool checkAttrSpecialTypes + bool checkAttrSpecialTypes, + bool relaxForbiddenCheck = false ); diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index cea25d5b52..5fbe2faa38 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -525,11 +525,6 @@ static std::string parseCustomPayload } ngsi->id = iter->value.GetString(); - - if (forbiddenIdChars(V2, ngsi->id.c_str(), "")) - { - return badInput(ciP, ERROR_DESC_BAD_REQUEST_INVALID_CHAR_ENTID); - } } else if (name == "type") { @@ -539,11 +534,6 @@ static std::string parseCustomPayload } ngsi->type = iter->value.GetString(); - - if (forbiddenIdChars(V2, ngsi->type.c_str(), "")) - { - return badInput(ciP, ERROR_DESC_BAD_REQUEST_INVALID_CHAR_ENTTYPE); - } } else // attribute { @@ -551,7 +541,9 @@ static std::string parseCustomPayload ngsi->attributeVector.push_back(caP); - std::string r = parseContextAttribute(ciP, iter, caP, false); + // Note we are using relaxForbiddenCheck true in this case, as JEXL expressions typically use forbidden + // chars and we don't want to fail in that case + std::string r = parseContextAttribute(ciP, iter, caP, false, true); if (r == "max deep reached") { diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 77e86480a9..79ce3f57fc 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1363,7 +1363,7 @@ void ContextAttribute::addToContext(ExprContextObject* exprContextObjectP, bool * * ContextAttribute::check - */ -std::string ContextAttribute::check(ApiVersion apiVersion, RequestType requestType) +std::string ContextAttribute::check(ApiVersion apiVersion, RequestType requestType, bool relaxForbiddenCheck) { size_t len; char errorMsg[128]; @@ -1414,14 +1414,14 @@ std::string ContextAttribute::check(ApiVersion apiVersion, RequestType requestTy return "Invalid characters in attribute type"; } - if ((compoundValueP != NULL) && (compoundValueP->childV.size() != 0) && (type != TEXT_UNRESTRICTED_TYPE)) + if ((!relaxForbiddenCheck) && (compoundValueP != NULL) && (compoundValueP->childV.size() != 0) && (type != TEXT_UNRESTRICTED_TYPE)) { return compoundValueP->check(""); } if (valueType == orion::ValueTypeString) { - if ((type != TEXT_UNRESTRICTED_TYPE) && (forbiddenChars(stringValue.c_str()))) + if ((!relaxForbiddenCheck) && (type != TEXT_UNRESTRICTED_TYPE) && (forbiddenChars(stringValue.c_str()))) { alarmMgr.badInput(clientIp, "found a forbidden character in the value of an attribute", stringValue); return "Invalid characters in attribute value"; diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index caa36cbfe5..f242bf101d 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -134,7 +134,7 @@ typedef struct ContextAttribute /* Helper method to be use in some places wher '%s' is needed */ std::string getValue(void) const; - std::string check(ApiVersion apiVersion, RequestType requestType); + std::string check(ApiVersion apiVersion, RequestType requestType, bool relaxForbiddenCheck = false); ContextAttribute* clone(); bool compoundItemExists(const std::string& compoundPath, orion::CompoundValueNode** compoundItemPP = NULL); diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test index c20beb102f..0a3bb36e7d 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test @@ -57,7 +57,7 @@ payload='{ "ngsi": { "cal": { "value": "${a[.b<5]}", - "type": "TextUnrestricted" + "type": "Calculated" } } }, @@ -179,7 +179,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) ===================================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 151 +Content-Length: 145 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -192,7 +192,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": [ { "b": 1 @@ -211,7 +211,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 138 +Content-Length: 132 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -224,7 +224,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": null }, "id": "E1", @@ -236,7 +236,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 138 +Content-Length: 132 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -249,7 +249,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": null }, "id": "E1", @@ -261,7 +261,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 150 +Content-Length: 144 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -274,7 +274,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "cal": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Calculated", "value": [ { "b": -1, diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test new file mode 100644 index 0000000000..fc128a4e4d --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test @@ -0,0 +1,197 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using transformation in id and type) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression id=S|substring(0,2), type=S|substring(3,5) +# 02. Create entity E with S=E1:T1, A:1 +# 03. Update entity E with S=E2:T2, A:2 +# 04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2) +# + + +echo "01. Create custom sub with custom expression id=S|substring(0,2), type=S|substring(3,5)" +echo "=======================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "id": "${S|substring(0,2)}", + "type": "${S|substring(3,5)}" + } + }, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E with S=E1:T1, A:1" +echo "=====================================" +payload='{ + "id": "E", + "type": "T", + "S": { + "value": "E1:T1", + "type": "Text" + }, + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E with S=E2:T2, A:2" +echo "=====================================" +payload='{ + "S": { + "value": "E2:T2", + "type": "Text" + }, + "A": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2)" +echo "=================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression id=S|substring(0,2), type=S|substring(3,5) +======================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E with S=E1:T1, A:1 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. Update entity E with S=E2:T2, A:2 +===================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2) +================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 124 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "id": "E1", + "type": "T1" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 124 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "id": "E2", + "type": "T2" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test index 85075dc93a..5616763fde 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test @@ -56,7 +56,7 @@ payload='{ "ngsi": { "includes": { "value": "${S|includes(R)}", - "type": "TextUnrestricted" + "type": "Boolean" } } }, @@ -169,7 +169,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) =========================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 143 +Content-Length: 134 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -183,7 +183,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "id": "E1", "includes": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Boolean", "value": true }, "type": "T" @@ -194,7 +194,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 144 +Content-Length: 135 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -208,7 +208,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "id": "E1", "includes": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Boolean", "value": false }, "type": "T" @@ -219,7 +219,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 143 +Content-Length: 134 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -233,7 +233,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "id": "E1", "includes": { "metadata": {}, - "type": "TextUnrestricted", + "type": "Boolean", "value": true }, "type": "T" From c1adca02ebd6244c0a10604340bd4c4bf7f06a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 20 Mar 2024 11:25:04 +0100 Subject: [PATCH 203/390] FIX compile using C++14 --- CHANGES_NEXT_RELEASE | 3 ++- CMakeLists.txt | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 25236adf1a..c82e9210c5 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ -- Fix: lighter operation to get databases list from MongoDB (#4517) \ No newline at end of file +- Fix: lighter operation to get databases list from MongoDB (#4517) +- Hardening: compile code using C++14 standard diff --git a/CMakeLists.txt b/CMakeLists.txt index 42a046ec8d..c97c20263b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,10 @@ PROJECT(contextBroker) # find_package() for mongo driver doesn't work with cmake 2 cmake_minimum_required(VERSION 3.0) +# set C++14 standard +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + # # DEBUG or RELEASE build ? # From 84cba2e3e0466a81ac5a04ca5c8d7ad9cb119266 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 15:42:07 +0000 Subject: [PATCH 204/390] Bump pymongo from 2.8 to 4.6.3 in /test/acceptance/lettuce Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 2.8 to 4.6.3. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/2.8...4.6.3) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test/acceptance/lettuce/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/lettuce/requirements.txt b/test/acceptance/lettuce/requirements.txt index 06178d0a7e..62c12bd010 100644 --- a/test/acceptance/lettuce/requirements.txt +++ b/test/acceptance/lettuce/requirements.txt @@ -1,5 +1,5 @@ git+https://visitor:visitor@pdihub.hi.inet/fiware/fiware-iotqaUtils@v.3.1.2 -pymongo==2.8 +pymongo==4.6.3 flask==2.3.2 # changed from 0.12.3 to 2.3.2 due to vulnerability check in github although we haven't checked if it works... lettuce==0.2.20 requests==2.31.0 # changed from 2.20.0 (test/acceptance is no longer in use) From 5c01748490b9268cb88175e779c271def11ef73f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 15:42:54 +0000 Subject: [PATCH 205/390] Bump pymongo from 3.0.3 to 4.6.3 in /test/acceptance/behave Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 3.0.3 to 4.6.3. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/3.0.3...4.6.3) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test/acceptance/behave/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/behave/requirements.txt b/test/acceptance/behave/requirements.txt index 587ff63d25..c5ffa91382 100644 --- a/test/acceptance/behave/requirements.txt +++ b/test/acceptance/behave/requirements.txt @@ -6,6 +6,6 @@ PyHamcrest==1.8.3 requests==2.31.0 xmltodict==0.9.2 fabric==1.10.2 -pymongo==3.0.3 +pymongo==4.6.3 termcolor==1.1.0 git+https://github.com/telefonicaid/iotqatools.git@master From ce59d90aa0baaf1054f2494b6de5ad6e20034a17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 16:36:07 +0000 Subject: [PATCH 206/390] Bump pymongo from 4.6.1 to 4.6.3 in /scripts/entities_consistency Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 4.6.1 to 4.6.3. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.6.1...4.6.3) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- scripts/entities_consistency/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/entities_consistency/requirements.txt b/scripts/entities_consistency/requirements.txt index 99c8a008ba..b62fd5c160 100644 --- a/scripts/entities_consistency/requirements.txt +++ b/scripts/entities_consistency/requirements.txt @@ -1,2 +1,2 @@ -pymongo==4.6.1 +pymongo==4.6.3 deepdiff==6.7.1 From cca479b867254e14e81ea4b2c87d7723bdc172f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 8 Apr 2024 20:28:08 +0200 Subject: [PATCH 207/390] Update test/acceptance/lettuce/requirements.txt --- test/acceptance/lettuce/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/lettuce/requirements.txt b/test/acceptance/lettuce/requirements.txt index 62c12bd010..1914ecede5 100644 --- a/test/acceptance/lettuce/requirements.txt +++ b/test/acceptance/lettuce/requirements.txt @@ -1,5 +1,5 @@ git+https://visitor:visitor@pdihub.hi.inet/fiware/fiware-iotqaUtils@v.3.1.2 -pymongo==4.6.3 +pymongo==4.6.3 # changed from 2.8 due to vulnerability check in github although we haven't checked if it works... flask==2.3.2 # changed from 0.12.3 to 2.3.2 due to vulnerability check in github although we haven't checked if it works... lettuce==0.2.20 requests==2.31.0 # changed from 2.20.0 (test/acceptance is no longer in use) From 9f99bbb122f9b47f9705889a8a4cc264b322a690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 8 Apr 2024 20:29:36 +0200 Subject: [PATCH 208/390] Update test/acceptance/behave/requirements.txt --- test/acceptance/behave/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/behave/requirements.txt b/test/acceptance/behave/requirements.txt index c5ffa91382..d824bf71d0 100644 --- a/test/acceptance/behave/requirements.txt +++ b/test/acceptance/behave/requirements.txt @@ -6,6 +6,6 @@ PyHamcrest==1.8.3 requests==2.31.0 xmltodict==0.9.2 fabric==1.10.2 -pymongo==4.6.3 +pymongo==4.6.3 # changed since 3.0.3 due to vulnerability check termcolor==1.1.0 git+https://github.com/telefonicaid/iotqatools.git@master From cca7d63318dee393d4e8874ada0f7790b63b28c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 9 Apr 2024 15:57:04 +0200 Subject: [PATCH 209/390] FIX rename entities_consistency.py script to more general oriondb_consistency.py --- .../README.md | 61 ++++--- .../oriondb_consistency.py} | 168 +++++++++--------- .../requirements.txt | 0 .../test_oriondb_consistency.py} | 6 +- .../validation_data.js | 0 5 files changed, 119 insertions(+), 116 deletions(-) rename scripts/{entities_consistency => oriondb_consistency}/README.md (76%) rename scripts/{entities_consistency/entities_consistency.py => oriondb_consistency/oriondb_consistency.py} (91%) rename scripts/{entities_consistency => oriondb_consistency}/requirements.txt (100%) rename scripts/{entities_consistency/test_entities_consistency.py => oriondb_consistency/test_oriondb_consistency.py} (93%) rename scripts/{entities_consistency => oriondb_consistency}/validation_data.js (100%) diff --git a/scripts/entities_consistency/README.md b/scripts/oriondb_consistency/README.md similarity index 76% rename from scripts/entities_consistency/README.md rename to scripts/oriondb_consistency/README.md index 30474120a6..9816f06b29 100644 --- a/scripts/entities_consistency/README.md +++ b/scripts/oriondb_consistency/README.md @@ -1,7 +1,7 @@ -The entities consistency script analyze the contents of the entities database in orion DBs and +The database consistency script analyze the contents of the Orion database (entities, subscriptions, etc.) and check several consistency rules, reporting violations found. -Ref: [entity document database model]([../../doc/manuals/admin/database_model.md#entities-collection]) +Ref: [Orion database model](../../doc/manuals/admin/database_model.md) ## Requirements @@ -10,15 +10,18 @@ Usage of virtual env is recommended. ## Usage -Run `entities_consistency.py -h` for arguments details. +Run `oriondb_consistency.py -h` for arguments details. ## Rules -* Rules 1x: DB inconsistencies (use to be severe problems) -* Rules 2x: Syntax restrictions -* Rules 9x: Usage of legacy features +* Rules for entities + * Rules E1x: DB inconsistencies (use to be severe problems) + * Rules E2x: Syntax restrictions + * Rules E9x: Usage of legacy features +* Rules for subscriptions + * Rules S9x: Usage of legacy features -### Rule 10: `_id` field consistency +### Rule E10: `_id` field consistency Each entity in DB has an `_id` field with three subfields: @@ -26,7 +29,7 @@ Each entity in DB has an `_id` field with three subfields: * `type` * `servicePath` -### Rule 11: mandatory fields in entity +### Rule E11: mandatory fields in entity The following fields are mandatory: @@ -36,7 +39,7 @@ The following fields are mandatory: It is not an exhaustive check of every field in the database model, but some entities created/updated with old Orion versions may be missing them. -### Rule 12: mandatory fields in attribute +### Rule E12: mandatory fields in attribute The following subfields are mandatory for every attribute: @@ -46,15 +49,15 @@ The following subfields are mandatory for every attribute: It is not an exhaustive check of every field in the database model, but some entities created/updated with old Orion versions may be missing them. -### Rule 13: `attrNames` field consistency +### Rule E13: `attrNames` field consistency For each item in `attrNames` array there is a corresponding key in `attrs` object and the other way around. -### Rule 14: `mdNames` field consistency +### Rule E14: `mdNames` field consistency For every attribute, for each item in `mdNames` array there is a corresponding key in `md` object and the other way around. -### Rule 15: not swapped subkeys in `_id` +### Rule E15: not swapped subkeys in `_id` In MongoDB JSON objects are stored taking order into account, so DB allows to have a document with `_id` equal to `{"id": "E", "type": "T", "servicePath": "/"}` and at the same time have another document with `_id` @@ -62,7 +65,7 @@ equal to `{"type": "T", "id": "E", "servicePath": "/"}` without violating `_id` This rule checks that this is not happening in the entities collection. -### Rule 16: `location` field consistency +### Rule E16: `location` field consistency Check that location in consistent. In particular: @@ -73,41 +76,41 @@ Check that location in consistent. In particular: This rule is for location inconsistencies. For usage of deprecated geo types, there are additional rules in the 9x group. -### Rule 17: `lastCorrelator` existence +### Rule E17: `lastCorrelator` existence Check if `lastCorrelator` is included. This field was introduced in [Orion 1.8.0](https://github.com/telefonicaid/fiware-orion/releases/tag/1.8.0) (released in September 11th, 2017). -### Rule 20: entity id syntax +### Rule E20: entity id syntax Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. -### Rule 21: entity type syntax +### Rule E21: entity type syntax Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. -### Rule 22: entity servicePath syntax +### Rule E22: entity servicePath syntax Check [servicePath documentation](https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/orion-api.md#entity-service-path) -### Rule 23: attribute name syntax +### Rule E23: attribute name syntax Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. -### Rule 24: attribute type syntax +### Rule E24: attribute type syntax Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. -### Rule 25: metadata name syntax +### Rule E25: metadata name syntax Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. -### Rule 26: metadata type syntax +### Rule E26: metadata type syntax Check [identifiers syntax restrictions](../../doc/manuals/orion-api.md#identifiers-syntax-restrictions) for this case. -### Rule 90: detect usage of `geo:x` attribute type where `x` different from `json` +### Rule E90: detect usage of `geo:x` attribute type where `x` different from `json` Check usage of deprecated geo-location types, i.e: @@ -123,13 +126,13 @@ Suggested action is to: Note this rule doesn't check location consistency for this case (e.g. more than one geo-location attribute in the same entity). That's done by another rule in the 1x group. -### Rule 91: detect usage of more than one legacy `location` metadata +### Rule E91: detect usage of more than one legacy `location` metadata Check usage of `location` in more than one attribute of the same entity. Note this rule doesn't check location consistency for this case (that's done by another rule in the 1x group). -### Rule 92: detect legacy `location` metadata should be `WGS84` or `WSG84` +### Rule E92: detect legacy `location` metadata should be `WGS84` or `WSG84` The value of the `location` metadata should be `WGS84` or `WSG84`. @@ -138,7 +141,7 @@ Additional consideration: * Entities attributes may have `location` metadata with values different from `WGS84` or `WSG84` if created using NGSIv2. That would be a false positive in this rule validation * This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). -### Rule 93: detect usage of redundant legacy `location` +### Rule E93: detect usage of redundant legacy `location` Checks usage of redundant `location` metadata, i.e. when at the same time a `geo:` type is used in the same attribute. @@ -150,7 +153,7 @@ Additional, considerations: * This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. * This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). -### Rule 94: detect usage of not redundant legacy `location` +### Rule E94: detect usage of not redundant legacy `location` Checks usage of not redundant `location` metadata, i.e. when at the same time the type of the attribute is nog `geo:`. same attribute. @@ -168,9 +171,9 @@ Additional, considerations: ## Testing -You can test the `entities_consistency.py` script this qy: +You can test the `oriondb_consistency.py` script this qy: 1. Populate `orion-validation` DB with testing document. To do so, copy-paste the content of the `validation_data.js` in `mongosh` -2. Run `test_entities_consistency.py` test and check the log output +2. Run `test_oriondb_consistency.py` test and check the log output -You can also run `test_entities_consistenct.py` under coverage to check every rule is covering all the possible validation cases. +You can also run `test_orion_consistenct.py` under coverage to check every rule is covering all the possible validation cases. diff --git a/scripts/entities_consistency/entities_consistency.py b/scripts/oriondb_consistency/oriondb_consistency.py similarity index 91% rename from scripts/entities_consistency/entities_consistency.py rename to scripts/oriondb_consistency/oriondb_consistency.py index 4da9541710..65f0c41393 100644 --- a/scripts/entities_consistency/entities_consistency.py +++ b/scripts/oriondb_consistency/oriondb_consistency.py @@ -206,9 +206,9 @@ def check_id(id): # Rules functions -def rule10(entity): +def ruleE10(entity): """ - Rule 10: `_id` field consistency + Rule E10: `_id` field consistency See README.md for an explanation of the rule """ @@ -224,9 +224,9 @@ def rule10(entity): return None -def rule11(entity): +def ruleE11(entity): """ - Rule 11: mandatory fields in entity + Rule E11: mandatory fields in entity See README.md for an explanation of the rule """ @@ -242,9 +242,9 @@ def rule11(entity): return None -def rule12(entity): +def ruleE12(entity): """ - Rule 12: mandatory fields in attribute + Rule E12: mandatory fields in attribute See README.md for an explanation of the rule """ @@ -265,9 +265,9 @@ def rule12(entity): return None -def rule13(entity): +def ruleE13(entity): """ - Rule 13: `attrNames` field consistency + Rule E13: `attrNames` field consistency See README.md for an explanation of the rule """ @@ -300,9 +300,9 @@ def rule13(entity): return None -def rule14(entity): +def ruleE14(entity): """ - Rule 14: `mdNames` field consistency + Rule E14: `mdNames` field consistency See README.md for an explanation of the rule """ @@ -339,9 +339,9 @@ def rule14(entity): return None -def rule15(entities_collection): +def ruleE15(entities_collection): """ - Rule 15: not swapped subkeys in `_id` + Rule E15: not swapped subkeys in `_id` See README.md for an explanation of the rule @@ -374,9 +374,9 @@ def rule15(entities_collection): return None -def rule16(entity): +def ruleE16(entity): """ - Rule 16: `location` field consistency + Rule E16: `location` field consistency See README.md for an explanation of the rule """ @@ -422,9 +422,9 @@ def rule16(entity): return f"location field detected but no geo attribute is present (maybe metadata location is used?)" -def rule17(entity): +def ruleE17(entity): """ - Rule 17: `lastCorrelator` existence + Rule E17: `lastCorrelator` existence See README.md for an explanation of the rule """ @@ -434,9 +434,9 @@ def rule17(entity): return None -def rule20(entity): +def ruleE20(entity): """ - Rule 20: entity id syntax + Rule E20: entity id syntax See README.md for an explanation of the rule """ @@ -450,9 +450,9 @@ def rule20(entity): return None -def rule21(entity): +def ruleE21(entity): """ - Rule 21: entity type syntax + Rule E21: entity type syntax See README.md for an explanation of the rule """ @@ -466,9 +466,9 @@ def rule21(entity): return None -def rule22(entity): +def ruleE22(entity): """ - Rule 22: entity servicePath syntax + Rule E22: entity servicePath syntax See README.md for an explanation of the rule @@ -504,9 +504,9 @@ def rule22(entity): return f"unallowed characters in '{sp_levels[i]}' in servicePath level #{i}" -def rule23(entity): +def ruleE23(entity): """ - Rule 23: attribute name syntax + Rule E23: attribute name syntax See README.md for an explanation of the rule """ @@ -532,9 +532,9 @@ def rule23(entity): return None -def rule24(entity): +def ruleE24(entity): """ - Rule 24: attribute type syntax + Rule E24: attribute type syntax See README.md for an explanation of the rule """ @@ -554,9 +554,9 @@ def rule24(entity): return None -def rule25(entity): +def ruleE25(entity): """ - Rule 25: metadata name syntax + Rule E25: metadata name syntax See README.md for an explanation of the rule """ @@ -583,9 +583,9 @@ def rule25(entity): return None -def rule26(entity): +def ruleE26(entity): """ - Rule 26: metadata type syntax + Rule E26: metadata type syntax See README.md for an explanation of the rule """ @@ -607,9 +607,9 @@ def rule26(entity): return None -def rule90(entity): +def ruleE90(entity): """ - Rule 90: detect usage of `geo:x` attribute type where `x` different from `json` + Rule E90: detect usage of `geo:x` attribute type where `x` different from `json` See README.md for an explanation of the rule """ @@ -628,9 +628,9 @@ def rule90(entity): return f"usage of deprecated geo type in attributes: {', '.join(s)}" -def rule91(entity): +def ruleE91(entity): """ - Rule 91: detect usage of more than one legacy `location` metadata + Rule E91: detect usage of more than one legacy `location` metadata See README.md for an explanation of the rule """ @@ -646,9 +646,9 @@ def rule91(entity): return None -def rule92(entity): +def ruleE92(entity): """ - Rule 92: detect legacy `location` metadata should be `WGS84` or `WSG84` + Rule E92: detect legacy `location` metadata should be `WGS84` or `WSG84` See README.md for an explanation of the rule """ @@ -667,9 +667,9 @@ def rule92(entity): return None -def rule93(entity): +def ruleE93(entity): """ - Rule 93: detect usage of redundant legacy `location` + Rule E93: detect usage of redundant legacy `location` See README.md for an explanation of the rule """ @@ -682,9 +682,9 @@ def rule93(entity): return None -def rule94(entity): +def ruleE94(entity): """ - Rule 94: detect usage of not redundant legacy `location` + Rule E94: detect usage of not redundant legacy `location` See README.md for an explanation of the rule """ @@ -700,106 +700,106 @@ def rule94(entity): rules = [ # Rules 1x { - 'label': 'Rule10', + 'label': 'RuleE10', 'global': False, - 'func': rule10 + 'func': ruleE10 }, { - 'label': 'Rule11', + 'label': 'RuleE11', 'global': False, - 'func': rule11 + 'func': ruleE11 }, { - 'label': 'Rule12', + 'label': 'RuleE12', 'global': False, - 'func': rule12 + 'func': ruleE12 }, { - 'label': 'Rule13', + 'label': 'RuleE13', 'global': False, - 'func': rule13 + 'func': ruleE13 }, { - 'label': 'Rule14', + 'label': 'RuleE14', 'global': False, - 'func': rule14 + 'func': ruleE14 }, { - 'label': 'Rule15', + 'label': 'RuleE15', 'global': True, - 'func': rule15 + 'func': ruleE15 }, { - 'label': 'Rule16', + 'label': 'RuleE16', 'global': False, - 'func': rule16 + 'func': ruleE16 }, { - 'label': 'Rule17', + 'label': 'RuleE17', 'global': False, - 'func': rule17 + 'func': ruleE17 }, # Rules 2x { - 'label': 'Rule20', + 'label': 'RuleE20', 'global': False, - 'func': rule20 + 'func': ruleE20 }, { - 'label': 'Rule21', + 'label': 'RuleE21', 'global': False, - 'func': rule21 + 'func': ruleE21 }, { - 'label': 'Rule22', + 'label': 'RuleE22', 'global': False, - 'func': rule22 + 'func': ruleE22 }, { - 'label': 'Rule23', + 'label': 'RuleE23', 'global': False, - 'func': rule23 + 'func': ruleE23 }, { - 'label': 'Rule24', + 'label': 'RuleE24', 'global': False, - 'func': rule24 + 'func': ruleE24 }, { - 'label': 'Rule25', + 'label': 'RuleE25', 'global': False, - 'func': rule25 + 'func': ruleE25 }, { - 'label': 'Rule26', + 'label': 'RuleE26', 'global': False, - 'func': rule26 + 'func': ruleE26 }, # Rules 9x { - 'label': 'Rule90', + 'label': 'RuleE90', 'global': False, - 'func': rule90 + 'func': ruleE90 }, { - 'label': 'Rule91', + 'label': 'RuleE91', 'global': False, - 'func': rule91 + 'func': ruleE91 }, { - 'label': 'Rule92', + 'label': 'RuleE92', 'global': False, - 'func': rule92 + 'func': ruleE92 }, { - 'label': 'Rule93', + 'label': 'RuleE93', 'global': False, - 'func': rule93 + 'func': ruleE93 }, { - 'label': 'Rule94', + 'label': 'RuleE94', 'global': False, - 'func': rule94 + 'func': ruleE94 } ] @@ -876,8 +876,8 @@ def process_db(logger, db_name, db_conn, include_entity_date, query, rules_exp): if __name__ == '__main__': parser = argparse.ArgumentParser( - prog='entities_consistency', - description='Check consistency in Orion entities collection in DB') + prog='oriondb_consistency', + description='Check consistency in Orion DB') parser.add_argument('--mongoUri', dest='mongo_uri', default='mongodb://localhost:27017', help='MongoDB URI. Default is mongodb://localhost:27017') @@ -885,9 +885,9 @@ def process_db(logger, db_name, db_conn, include_entity_date, query, rules_exp): help='DB name to check. If omitted all DBs starting with "orion" will be checked.') parser.add_argument('--include-entities-date', dest='include_entities_date', default=False, action='store_true', help='include entity modification time in log traces') - parser.add_argument('--query', dest='query', default='{}', + parser.add_argument('--entities-query', dest='query', default='{}', help='query to filter entities to check, in JSON MongoDB query language. By default, ' - 'all entities in the collection will be checked.') + 'all entities in the collection will be checked. Applies to Rule Exx rules.') parser.add_argument('--rules-exp', dest='rules_exp', help='Specifies the rules to apply, as a regular expression. By default all rules are applied.') parser.add_argument('--logLevel', dest='log_level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'], default='INFO', diff --git a/scripts/entities_consistency/requirements.txt b/scripts/oriondb_consistency/requirements.txt similarity index 100% rename from scripts/entities_consistency/requirements.txt rename to scripts/oriondb_consistency/requirements.txt diff --git a/scripts/entities_consistency/test_entities_consistency.py b/scripts/oriondb_consistency/test_oriondb_consistency.py similarity index 93% rename from scripts/entities_consistency/test_entities_consistency.py rename to scripts/oriondb_consistency/test_oriondb_consistency.py index f0ab04e399..07ff631bc0 100644 --- a/scripts/entities_consistency/test_entities_consistency.py +++ b/scripts/oriondb_consistency/test_oriondb_consistency.py @@ -30,13 +30,13 @@ # the result as log output. # # You can run this test under coverage, so you can check the coverage of the different rules in -# entities_consistency.py +# oriondb_consistency.py import unittest from pymongo import MongoClient import logging -from entities_consistency import process_db +from oriondb_consistency import process_db class TestEntitiesConsistency(unittest.TestCase): def test_process_db(self): @@ -51,5 +51,5 @@ def test_process_db(self): logger = logging.getLogger() # connect to MongoDB and process validation DB - mongo_client = MongoClient('mongodb://localhost:27017') + mongo_client = MongoClient('mongodb://localhost:37017') process_db(logger, 'orion-validation', mongo_client, False, {}, None) diff --git a/scripts/entities_consistency/validation_data.js b/scripts/oriondb_consistency/validation_data.js similarity index 100% rename from scripts/entities_consistency/validation_data.js rename to scripts/oriondb_consistency/validation_data.js From 7039520372391fbd62500f90bc5aa5282111cfeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 9 Apr 2024 17:23:47 +0200 Subject: [PATCH 210/390] ADD Rule S90 --- scripts/oriondb_consistency/README.md | 8 + .../oriondb_consistency.py | 165 ++++++++++++------ .../test_oriondb_consistency.py | 6 +- .../oriondb_consistency/validation_data.js | 53 ++++++ 4 files changed, 177 insertions(+), 55 deletions(-) diff --git a/scripts/oriondb_consistency/README.md b/scripts/oriondb_consistency/README.md index 9816f06b29..52499ecf4f 100644 --- a/scripts/oriondb_consistency/README.md +++ b/scripts/oriondb_consistency/README.md @@ -169,6 +169,14 @@ Additional, considerations: * This rule assumes only one `location` is in the entity (i.e. Rule 91 is not violated). If that doesn't occur, only the first occurrence is taken into account. * This rule doesn't check location consistency for this case (that's done by another rule in the 1x group). +### Rule S90: detect usage of legacy notification format + +Check usage of legacy notification format in subscriptions (i.e. [`notification.attrsFormat`](../../doc/manuals/orion-api.md#subscriptionnotification) set to `legacy`). + +Suggestion action is to: + +* Change `notification.attrsFormat` to one of the supported formats (e.g. `normalized`). + ## Testing You can test the `oriondb_consistency.py` script this qy: diff --git a/scripts/oriondb_consistency/oriondb_consistency.py b/scripts/oriondb_consistency/oriondb_consistency.py index 65f0c41393..6756e65c07 100644 --- a/scripts/oriondb_consistency/oriondb_consistency.py +++ b/scripts/oriondb_consistency/oriondb_consistency.py @@ -696,115 +696,165 @@ def ruleE94(entity): return None +def ruleS90(csub): + """ + Rule S90: Check usage of legacy notification format in subscriptions + + See README.md for an explanation of the rule + """ + if csub['format'] == 'JSON': + return f"notification legacy format in use (endpoint: {csub['reference']})" + + return None -rules = [ - # Rules 1x +rules_inventory = [ + # Rules E1x { 'label': 'RuleE10', + 'collection': 'entities', 'global': False, 'func': ruleE10 }, { 'label': 'RuleE11', + 'collection': 'entities', 'global': False, 'func': ruleE11 }, { 'label': 'RuleE12', + 'collection': 'entities', 'global': False, 'func': ruleE12 }, { 'label': 'RuleE13', + 'collection': 'entities', 'global': False, 'func': ruleE13 }, { 'label': 'RuleE14', + 'collection': 'entities', 'global': False, 'func': ruleE14 }, { 'label': 'RuleE15', + 'collection': 'entities', 'global': True, 'func': ruleE15 }, { 'label': 'RuleE16', + 'collection': 'entities', 'global': False, 'func': ruleE16 }, { 'label': 'RuleE17', + 'collection': 'entities', 'global': False, 'func': ruleE17 }, - # Rules 2x + # Rules E2x { 'label': 'RuleE20', + 'collection': 'entities', 'global': False, 'func': ruleE20 }, { 'label': 'RuleE21', + 'collection': 'entities', 'global': False, 'func': ruleE21 }, { 'label': 'RuleE22', + 'collection': 'entities', 'global': False, 'func': ruleE22 }, { 'label': 'RuleE23', + 'collection': 'entities', 'global': False, 'func': ruleE23 }, { 'label': 'RuleE24', + 'collection': 'entities', 'global': False, 'func': ruleE24 }, { 'label': 'RuleE25', + 'collection': 'entities', 'global': False, 'func': ruleE25 }, { 'label': 'RuleE26', + 'collection': 'entities', 'global': False, 'func': ruleE26 }, - # Rules 9x + # Rules E9x { 'label': 'RuleE90', + 'collection': 'entities', 'global': False, 'func': ruleE90 }, { 'label': 'RuleE91', + 'collection': 'entities', 'global': False, 'func': ruleE91 }, { 'label': 'RuleE92', + 'collection': 'entities', 'global': False, 'func': ruleE92 }, { 'label': 'RuleE93', + 'collection': 'entities', 'global': False, 'func': ruleE93 }, { 'label': 'RuleE94', + 'collection': 'entities', 'global': False, 'func': ruleE94 + }, + # Rules S9x + { + 'label': 'RuleS90', + 'collection': 'csubs', + 'global': False, + 'func': ruleS90 } ] +def get_id(doc, col, include_entity_date): + """ + Depending the collection and some arguments, the id is got in a way or another + """ + if col == 'entities': + id_string = json.dumps(doc['_id']) + if 'modDate' in doc: + id_string = f"({datetime.fromtimestamp(doc['modDate']).strftime('%Y-%m-%dT%H:%M:%SZ')}) {id_string}" + else: + id_string = f"()) {id_string}" + return f"entity {id_string}" + else: # col == 'csubs' + return f"subscription {doc['_id']}" -def process_db(logger, db_name, db_conn, include_entity_date, query, rules_exp): +def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp): """ Process an individual DB @@ -812,64 +862,65 @@ def process_db(logger, db_name, db_conn, include_entity_date, query, rules_exp): :param db_name: the name of the DB to process :param db_conn: connection to MongoDB :param include_entity_date: if True, include entity modification date in log traces - :param query: query to filter entities to be processed + :param queries: dict with per-colletion queries to filter entities to be processed (the key in the dictionary is + the collection to apply the query) :param rules_exp: regular expression to filter rules to apply :return: fails """ logger.info(f'processing {db_name}') - n = 0 - failed_entities = 0 + n = { + 'entities': 0, + 'csubs': 0 + } + failed_docs = { + 'entities': 0, + 'csubs': 0 + } fails = 0 - # check collection existence - if 'entities' not in db_conn[db_name].list_collection_names(): - logger.warning(f'collections entities not found in {db_name} database, nothing to do') - return + for col in ['entities', 'csubs']: + if col not in db_conn[db_name].list_collection_names(): + logger.warning(f'collection {col} not found in {db_name} database') - # apply global rules - for rule in rules: - if rules_exp is not None and not re.search(rules_exp, rule['label']): - continue + # filter out rules + rules = [] + for rule in rules_inventory: + if rules_exp is None or re.search(rules_exp, rule['label']): + rules.append(rule) + # first process global rules + for rule in rules: if rule['global']: - s = rule['func'](db_conn[db_name]['entities']) + col = rule['collection'] + s = rule['func'](db_conn[db_name][col]) if s is not None: - logger.warning(f'DB {db_name} {rule["label"]} violation in entities collection: {s}') + logger.warning(f'DB {db_name} {rule["label"]} violation in {col} collection: {s}') fails += 1 - # apply per-entity rules - for entity in db_conn[db_name]['entities'].find(query): - n += 1 - entity_fail = False - - id_string = json.dumps(entity['_id']) - if include_entity_date: - if 'modDate' in entity: - id_string = f"({datetime.fromtimestamp(entity['modDate']).strftime('%Y-%m-%dT%H:%M:%SZ')}) {id_string}" - else: - id_string = f"()) {id_string}" - - logger.debug(f'* processing entity {id_string}') - for rule in rules: - if rules_exp is not None and not re.search(rules_exp, rule['label']): - continue - - if not rule['global']: - s = rule['func'](entity) - if s is not None: - logger.warning(f'DB {db_name} {rule["label"]} violation for entity {id_string}: {s}') - entity_fail = True - fails += 1 - - if entity_fail: - failed_entities += 1 - - if n > 0: - logger.info( - f'processed {db_name}: {failed_entities}/{n} ({round(failed_entities / n * 100, 2)}%) failed entities with {fails} rule violations') - else: - logger.warning(f'{db_name} has 0 entities (maybe it should be cleaned up?)') + # second process not global rules, per collection + for col in ['entities', 'csubs']: + for doc in db_conn[db_name][col].find(queries[col]): + n[col] += 1 + doc_fail = False + id_string = get_id(doc, col, include_entity_date) + logger.debug(f'* processing {id_string}') + for rule in rules: + if not rule['global'] and rule['collection'] == col: + s = rule['func'](doc) + if s is not None: + logger.warning(f'DB {db_name} {rule["label"]} violation for {id_string}: {s}') + doc_fail = True + fails += 1 + + if doc_fail: + failed_docs[col] += 1 + + + for col in ['entities', 'csubs']: + if n[col] > 0: + logger.info( + f'processed {db_name} in collection {col}: {failed_docs[col]}/{n[col]} ({round(failed_docs[col] / n[col] * 100, 2)}%) failed docs') return fails @@ -885,9 +936,12 @@ def process_db(logger, db_name, db_conn, include_entity_date, query, rules_exp): help='DB name to check. If omitted all DBs starting with "orion" will be checked.') parser.add_argument('--include-entities-date', dest='include_entities_date', default=False, action='store_true', help='include entity modification time in log traces') - parser.add_argument('--entities-query', dest='query', default='{}', + parser.add_argument('--query-entities', dest='query_entities', default='{}', help='query to filter entities to check, in JSON MongoDB query language. By default, ' 'all entities in the collection will be checked. Applies to Rule Exx rules.') + parser.add_argument('--query-csubs', dest='query_csubs', default='{}', + help='query to filter csubs to check, in JSON MongoDB query language. By default, ' + 'all subscriptions in the collection will be checked. Applies to Rule Sxx rules.') parser.add_argument('--rules-exp', dest='rules_exp', help='Specifies the rules to apply, as a regular expression. By default all rules are applied.') parser.add_argument('--logLevel', dest='log_level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'], default='INFO', @@ -909,12 +963,15 @@ def process_db(logger, db_name, db_conn, include_entity_date, query, rules_exp): db_names = mongo_client.list_database_names() # to remove starting and trailing ' char, in case it is used - query = json.loads(args.query.replace("'", "")) + queries = { + 'entities': json.loads(args.query_entities.replace("'", "")), + 'csubs': json.loads(args.query_csubs.replace("'", "")), + } fails = 0 if args.db is not None: if args.db in db_names: - fails += process_db(logger, args.db, mongo_client, args.include_entities_date, query, args.rules_exp) + fails += process_db(logger, args.db, mongo_client, args.include_entities_date, queries, args.rules_exp) else: logger.fatal(f'database {args.db} does not exist') sys.exit(1) @@ -922,6 +979,6 @@ def process_db(logger, db_name, db_conn, include_entity_date, query, rules_exp): # Process all Orion databases for db_name in db_names: if db_name.startswith('orion-'): - fails += process_db(logger, db_name, mongo_client, args.include_entities_date, query, args.rules_exp) + fails += process_db(logger, db_name, mongo_client, args.include_entities_date, queries, args.rules_exp) logger.info(f'total rule violations: {fails}') diff --git a/scripts/oriondb_consistency/test_oriondb_consistency.py b/scripts/oriondb_consistency/test_oriondb_consistency.py index 07ff631bc0..3ccf08237e 100644 --- a/scripts/oriondb_consistency/test_oriondb_consistency.py +++ b/scripts/oriondb_consistency/test_oriondb_consistency.py @@ -52,4 +52,8 @@ def test_process_db(self): # connect to MongoDB and process validation DB mongo_client = MongoClient('mongodb://localhost:37017') - process_db(logger, 'orion-validation', mongo_client, False, {}, None) + queries = { + 'entities': {}, + 'csubs': {} + } + process_db(logger, 'orion-validation', mongo_client, False, queries, None) diff --git a/scripts/oriondb_consistency/validation_data.js b/scripts/oriondb_consistency/validation_data.js index a602e41ff0..0d1e929d44 100644 --- a/scripts/oriondb_consistency/validation_data.js +++ b/scripts/oriondb_consistency/validation_data.js @@ -4451,3 +4451,56 @@ db.getSiblingDB("orion-validation").entities.insertMany([ "lastCorrelator": "88e1bc10-b9dd-11ee-8755-080027cd35f1" } ]) + +db.getSiblingDB("orion-validation").csubs.insertMany([ + { + "expiration": { + "$numberLong": "9223372036854775807" + }, + "reference": "http://notify-receptor:5055/notify", + "custom": true, + "timeout": { + "$numberLong": "0" + }, + "headers": { + "fiware-servicepath": "/SS1" + }, + "throttling": { + "$numberLong": "0" + }, + "maxFailsLimit": { + "$numberLong": "-1" + }, + "servicePath": "/SS2", + "status": "active", + "statusLastChange": 1682640509.6958137, + "entities": [ + { + "id": ".*", + "isPattern": "true", + "type": "T", + "isTypePattern": false + } + ], + "attrs": [ + "temperature", + ], + "metadata": [], + "blacklist": false, + "onlyChanged": false, + "covered": false, + "description": "Foobar", + "conditions": [ + "TimeInstant" + ], + "expression": { + "q": "", + "mq": "", + "geometry": "", + "coords": "", + "georel": "" + }, + "altTypes": [], + "format": "JSON" + } +]) From a88563fb0c72b7355b0a440799fdb92d57a3a47f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 9 Apr 2024 18:12:25 +0200 Subject: [PATCH 211/390] FIX autofix framework added --- scripts/oriondb_consistency/README.md | 14 ++ .../oriondb_consistency.py | 178 ++++++++++++------ .../test_oriondb_consistency.py | 2 +- 3 files changed, 134 insertions(+), 60 deletions(-) diff --git a/scripts/oriondb_consistency/README.md b/scripts/oriondb_consistency/README.md index 52499ecf4f..bbc141edfd 100644 --- a/scripts/oriondb_consistency/README.md +++ b/scripts/oriondb_consistency/README.md @@ -177,6 +177,20 @@ Suggestion action is to: * Change `notification.attrsFormat` to one of the supported formats (e.g. `normalized`). +Autofix action: + +* `notification.attrsFormat` is set to `normalized`. + +## Autofix mode + +If `--autofix` is spefified in the command line arguments, the script implements automatic fixing of documents +violating the rules. + +**WARNING: the usage of `--autofix` will do DB modification**. Use it with care. It is strongly recommended to +do a backup of your data before using it. + +It only works for some rules. Check specific rules documntation for details (look for "Autofix action"). + ## Testing You can test the `oriondb_consistency.py` script this qy: diff --git a/scripts/oriondb_consistency/oriondb_consistency.py b/scripts/oriondb_consistency/oriondb_consistency.py index 6756e65c07..7dd289d2a8 100644 --- a/scripts/oriondb_consistency/oriondb_consistency.py +++ b/scripts/oriondb_consistency/oriondb_consistency.py @@ -31,6 +31,7 @@ import json import sys import re +import copy # Helper functions @@ -219,9 +220,11 @@ def ruleE10(entity): missing_fields.append(field) if len(missing_fields) > 0: - return f"missing subfields in _id: {', '.join(missing_fields)}" + r = f"missing subfields in _id: {', '.join(missing_fields)}" else: - return None + r = None + + return r, None def ruleE11(entity): @@ -237,9 +240,11 @@ def ruleE11(entity): missing_fields.append(field) if len(missing_fields) > 0: - return f"missing fields: {', '.join(missing_fields)}" + r = f"missing fields: {', '.join(missing_fields)}" else: - return None + r = None + + return r, None def ruleE12(entity): @@ -260,9 +265,11 @@ def ruleE12(entity): s.append(f"in attribute '{attr}' missing fields: {', '.join(missing_fields)}") if len(s) > 0: - return ', '.join(s) + r = ', '.join(s) else: - return None + r = None + + return r, None def ruleE13(entity): @@ -295,10 +302,11 @@ def ruleE13(entity): s.append(f"attributes in attrs object not found in attrNames: {','.join(attrs_not_in_attrnames)}") if len(s) > 0: - return ', '.join(s) + r = ', '.join(s) else: - return None + r = None + return r, None def ruleE14(entity): """ @@ -334,9 +342,11 @@ def ruleE14(entity): f"in attribute '{item}' metadata in md object not found in mdNames: {', '.join(md_not_in_mdnames)}") if len(s) > 0: - return ', '.join(s) + r = ', '.join(s) else: - return None + r = None + + return r, None def ruleE15(entities_collection): @@ -369,10 +379,11 @@ def ruleE15(entities_collection): f"_id uniqueness violation for entity id='{id}' type='{type}' servicePath='{service_path}' found {count} times") if len(s) > 0: - return ', '.join(s) + r = ', '.join(s) else: - return None + r = None + return r, None def ruleE16(entity): """ @@ -380,6 +391,8 @@ def ruleE16(entity): See README.md for an explanation of the rule """ + # FIXME: this function should be re-factored. It has many return points. It's ugly... + # check that as much as one attribute is using geo type geo_attrs = [] for attr in entity['attrs']: @@ -388,7 +401,7 @@ def ruleE16(entity): geo_attrs.append(attr) if len(geo_attrs) > 1: - return f"more than one attribute with geo type: {', '.join(geo_attrs)}" + return f"more than one attribute with geo type: {', '.join(geo_attrs)}", None if len(geo_attrs) == 1: # If geo attr found, then check that there is consistent location field @@ -396,13 +409,13 @@ def ruleE16(entity): geo_type = entity['attrs'][geo_attr]['type'] if 'location' not in entity: - return f"geo location '{geo_attr}' ({geo_type}) not null but location field not found in entity" + return f"geo location '{geo_attr}' ({geo_type}) not null but location field not found in entity", None if entity['location']['attrName'] != geo_attr: - return f"location.attrName ({entity['location']['attrName']}) differs from '{geo_attr}'" + return f"location.attrName ({entity['location']['attrName']}) differs from '{geo_attr}'", None geo_json = to_geo_json(entity['attrs'][geo_attr]) if type(geo_json) == str: - return geo_json + return geo_json, None # https://www.testcult.com/deep-comparison-of-json-in-python/ diff = DeepDiff(geo_json, entity['location']['coords'], ignore_order=True) @@ -412,14 +425,16 @@ def ruleE16(entity): geo_json = convert_strings_to_numbers(geo_json) if not DeepDiff(geo_json, entity['location']['coords'], ignore_order=True): return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) is consistent, but value " \ - f"should use numbers for coordinates instead of strings" + f"should use numbers for coordinates instead of strings", None else: # Other causes - return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) value: {diff}" + return f"location.coords and GeoJSON derived from '{geo_attr}' ({geo_type}) value: {diff}", None else: # len(geo_attrs) == 0 # If no geo attr found, check there isn't a location field if 'location' in entity: - return f"location field detected but no geo attribute is present (maybe metadata location is used?)" + return f"location field detected but no geo attribute is present (maybe metadata location is used?)", None + + return None, None def ruleE17(entity): @@ -429,9 +444,11 @@ def ruleE17(entity): See README.md for an explanation of the rule """ if 'lastCorrelator' not in entity: - return f"missing lastCorrelator" + r = f"missing lastCorrelator" else: - return None + r = None + + return r, None def ruleE20(entity): @@ -445,9 +462,9 @@ def ruleE20(entity): if 'id' in entity['_id']: r = check_id(entity['_id']['id']) if r is not None: - return f"entity id ({entity['_id']['id']}) syntax violation: {r}" + return f"entity id ({entity['_id']['id']}) syntax violation: {r}", None - return None + return None, None def ruleE21(entity): @@ -461,9 +478,9 @@ def ruleE21(entity): if 'type' in entity['_id']: r = check_id(entity['_id']['type']) if r is not None: - return f"entity type ({entity['_id']['type']}) syntax violation: {r}" + return f"entity type ({entity['_id']['type']}) syntax violation: {r}", None - return None + return None, None def ruleE22(entity): @@ -477,32 +494,33 @@ def ruleE22(entity): # servicePath existence is checked by another rule if 'servicePath' not in entity['_id']: - return None + return None, None sp = entity['_id']['servicePath'] # Scope must start with / (only "absolute" scopes are allowed) if not sp.startswith('/'): - return f"servicePath '{sp}' does not starts with '/'" + return f"servicePath '{sp}' does not starts with '/'", None # This special case can be problematic (as split will detect always a level and the minimum length rule # will break. So early return if sp == '/': - return None + return None, None # 10 maximum scope levels in a path sp_levels = sp[1:].split('/') if len(sp_levels) > 10: - return f"servicePath has {len(sp_levels)} tokens but the limit is 10" + return f"servicePath has {len(sp_levels)} tokens but the limit is 10", None # 50 maximum characters in each level (1 char is minimum), only alphanumeric and underscore allowed for i in range(len(sp_levels)): if len(sp_levels[i]) == 0: - return f'servicePath level #{i} length is 0 but minimum is 1' + return f'servicePath level #{i} length is 0 but minimum is 1', None if len(sp_levels[i]) > 50: - return f'servicePath level #{i} length is {len(sp_levels[i])} but maximum is 50' + return f'servicePath level #{i} length is {len(sp_levels[i])} but maximum is 50', None if re.search('[^a-zA-Z0-9_]', sp_levels[i]): - return f"unallowed characters in '{sp_levels[i]}' in servicePath level #{i}" + return f"unallowed characters in '{sp_levels[i]}' in servicePath level #{i}", None + return None, None def ruleE23(entity): """ @@ -527,9 +545,11 @@ def ruleE23(entity): s.append(f"attribute name ({attr}) syntax violation: {r}") if len(s) > 0: - return ', '.join(s) + r = ', '.join(s) else: - return None + r = None + + return r, None def ruleE24(entity): @@ -549,9 +569,11 @@ def ruleE24(entity): s.append(f"in attribute '{attr}' type ({type}) syntax violation: {r}") if len(s) > 0: - return ', '.join(s) + r = ', '.join(s) else: - return None + r = None + + return r, None def ruleE25(entity): @@ -578,9 +600,11 @@ def ruleE25(entity): s.append(f"in attribute '{attr}' metadata name ({md}) syntax violation: {r}") if len(s) > 0: - return ', '.join(s) + r = ', '.join(s) else: - return None + r = None + + return r, None def ruleE26(entity): @@ -602,9 +626,11 @@ def ruleE26(entity): s.append(f"in attribute '{attr}' metadata '{md}' type ({type}) syntax violation: {r}") if len(s) > 0: - return ', '.join(s) + r = ', '.join(s) else: - return None + r = None + + return r, None def ruleE90(entity): @@ -625,7 +651,11 @@ def ruleE90(entity): # if len(s) > 0: - return f"usage of deprecated geo type in attributes: {', '.join(s)}" + r = f"usage of deprecated geo type in attributes: {', '.join(s)}" + else: + r = None + + return r, None def ruleE91(entity): @@ -641,9 +671,11 @@ def ruleE91(entity): attrs.append(attr) if len(attrs) > 1: - return f"location metadata found {len(attrs)} times in attributes: {', '.join(attrs)} (maximum should be just 1)" + r = f"location metadata found {len(attrs)} times in attributes: {', '.join(attrs)} (maximum should be just 1)" else: - return None + r = None + + return r, None def ruleE92(entity): @@ -662,9 +694,11 @@ def ruleE92(entity): f"in attribute '{attr}' location metadata value is {location_value} (should be WGS84 or WSG84)") if len(s) > 0: - return ', '.join(s) + r = ', '.join(s) else: - return None + r = None + + return r, None def ruleE93(entity): @@ -677,9 +711,9 @@ def ruleE93(entity): if 'md' in entity['attrs'][attr]: for md in entity['attrs'][attr]['md']: if md == 'location' and is_geo_type(entity['attrs'][attr]['type']): - return f"in attribute '{attr}' redundant location metadata found (attribute is already using {entity['attrs'][attr]['type']} type)" + return f"in attribute '{attr}' redundant location metadata found (attribute is already using {entity['attrs'][attr]['type']} type)", None - return None + return None, None def ruleE94(entity): @@ -692,9 +726,9 @@ def ruleE94(entity): if 'md' in entity['attrs'][attr]: for md in entity['attrs'][attr]['md']: if md == 'location' and not is_geo_type(entity['attrs'][attr]['type']): - return f"in attribute '{attr}' location metadata found (attribute type is {entity['attrs'][attr]['type']})" + return f"in attribute '{attr}' location metadata found (attribute type is {entity['attrs'][attr]['type']})", None - return None + return None, None def ruleS90(csub): """ @@ -703,9 +737,14 @@ def ruleS90(csub): See README.md for an explanation of the rule """ if csub['format'] == 'JSON': - return f"notification legacy format in use (endpoint: {csub['reference']})" + r = f"notification legacy format in use (endpoint: {csub['reference']})" + fixed_csub = copy.deepcopy(csub) + fixed_csub['format'] = 'normalized' + else: + r = None + fixed_csub = None - return None + return r, fixed_csub rules_inventory = [ # Rules E1x @@ -846,15 +885,16 @@ def get_id(doc, col, include_entity_date): """ if col == 'entities': id_string = json.dumps(doc['_id']) - if 'modDate' in doc: - id_string = f"({datetime.fromtimestamp(doc['modDate']).strftime('%Y-%m-%dT%H:%M:%SZ')}) {id_string}" - else: - id_string = f"()) {id_string}" + if include_entity_date: + if 'modDate' in doc: + id_string = f"({datetime.fromtimestamp(doc['modDate']).strftime('%Y-%m-%dT%H:%M:%SZ')}) {id_string}" + else: + id_string = f"()) {id_string}" return f"entity {id_string}" else: # col == 'csubs' return f"subscription {doc['_id']}" -def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp): +def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp, autofix): """ Process an individual DB @@ -865,6 +905,7 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp :param queries: dict with per-colletion queries to filter entities to be processed (the key in the dictionary is the collection to apply the query) :param rules_exp: regular expression to filter rules to apply + :param autofix: True if autofix is activated :return: fails """ @@ -890,6 +931,7 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp rules.append(rule) # first process global rules + # NOTE: global rules doesn't accept autofix parameter for rule in rules: if rule['global']: col = rule['collection'] @@ -899,6 +941,10 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp fails += 1 # second process not global rules, per collection + autofix = { + 'entities': [], + 'csubs': [] + } for col in ['entities', 'csubs']: for doc in db_conn[db_name][col].find(queries[col]): n[col] += 1 @@ -907,11 +953,13 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp logger.debug(f'* processing {id_string}') for rule in rules: if not rule['global'] and rule['collection'] == col: - s = rule['func'](doc) + (s, fixed_doc) = rule['func'](doc) if s is not None: logger.warning(f'DB {db_name} {rule["label"]} violation for {id_string}: {s}') doc_fail = True fails += 1 + if fixed_doc is not None: + autofix[col].append(fixed_doc) if doc_fail: failed_docs[col] += 1 @@ -922,6 +970,8 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp logger.info( f'processed {db_name} in collection {col}: {failed_docs[col]}/{n[col]} ({round(failed_docs[col] / n[col] * 100, 2)}%) failed docs') + # TODO: implement save logic for autofix dict + return fails @@ -944,10 +994,20 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp 'all subscriptions in the collection will be checked. Applies to Rule Sxx rules.') parser.add_argument('--rules-exp', dest='rules_exp', help='Specifies the rules to apply, as a regular expression. By default all rules are applied.') + parser.add_argument('--autofix', dest='atutofix', action='store_true', + help='Applies some automatic fixes. Not for all rules. Check documentation. WARNING: this ' + 'operation may modify OrionDBs, use with care') parser.add_argument('--logLevel', dest='log_level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'], default='INFO', help='log level. Default is INFO') args = parser.parse_args() + if args.autofix: + print("WARNING!!!! Parameter --autofix has been activated, so this script may modify documents in Orion DBs (check documentation") + print("for details). These modifications cannot be undone. If you are sure you want to continue type 'yes' and press Enter") + confirm = input() + if (confirm != 'yes'): + sys.exit() + # sets the logging configuration logging.basicConfig( level=logging.getLevelName(args.log_level), @@ -971,7 +1031,7 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp fails = 0 if args.db is not None: if args.db in db_names: - fails += process_db(logger, args.db, mongo_client, args.include_entities_date, queries, args.rules_exp) + fails += process_db(logger, args.db, mongo_client, args.include_entities_date, queries, args.rules_exp, args.autofix) else: logger.fatal(f'database {args.db} does not exist') sys.exit(1) @@ -979,6 +1039,6 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp # Process all Orion databases for db_name in db_names: if db_name.startswith('orion-'): - fails += process_db(logger, db_name, mongo_client, args.include_entities_date, queries, args.rules_exp) + fails += process_db(logger, db_name, mongo_client, args.include_entities_date, queries, args.rules_exp, args.autofix) logger.info(f'total rule violations: {fails}') diff --git a/scripts/oriondb_consistency/test_oriondb_consistency.py b/scripts/oriondb_consistency/test_oriondb_consistency.py index 3ccf08237e..6181a9042e 100644 --- a/scripts/oriondb_consistency/test_oriondb_consistency.py +++ b/scripts/oriondb_consistency/test_oriondb_consistency.py @@ -56,4 +56,4 @@ def test_process_db(self): 'entities': {}, 'csubs': {} } - process_db(logger, 'orion-validation', mongo_client, False, queries, None) + process_db(logger, 'orion-validation', mongo_client, False, queries, None, False) From 21ec49f9ddb82d35eabee8c5861135c65de90a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 10 Apr 2024 13:23:41 +0200 Subject: [PATCH 212/390] FIX autofix updates in db --- .../oriondb_consistency.py | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/scripts/oriondb_consistency/oriondb_consistency.py b/scripts/oriondb_consistency/oriondb_consistency.py index 7dd289d2a8..b9ab0efb85 100644 --- a/scripts/oriondb_consistency/oriondb_consistency.py +++ b/scripts/oriondb_consistency/oriondb_consistency.py @@ -23,7 +23,7 @@ __author__ = 'fermin' -from pymongo import MongoClient +from pymongo import MongoClient, ReplaceOne from deepdiff import DeepDiff from datetime import datetime import argparse @@ -746,6 +746,11 @@ def ruleS90(csub): return r, fixed_csub +collections_inventory = [ + 'entities', + 'csubs' +] + rules_inventory = [ # Rules E1x { @@ -910,17 +915,16 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp """ logger.info(f'processing {db_name}') - n = { - 'entities': 0, - 'csubs': 0 - } - failed_docs = { - 'entities': 0, - 'csubs': 0 - } - fails = 0 - - for col in ['entities', 'csubs']: + n = {} + n_failed = {} + modified_docs = {} + for col in collections_inventory: + n[col] = 0 + n_failed[col] = 0 + modified_docs[col] = [] + n_fails = 0 + + for col in collections_inventory: if col not in db_conn[db_name].list_collection_names(): logger.warning(f'collection {col} not found in {db_name} database') @@ -930,22 +934,19 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp if rules_exp is None or re.search(rules_exp, rule['label']): rules.append(rule) - # first process global rules - # NOTE: global rules doesn't accept autofix parameter + # first: process global rules for rule in rules: if rule['global']: col = rule['collection'] - s = rule['func'](db_conn[db_name][col]) + # FIXME: fixed_doc doesn't make sense for global rules (althoug it could be implemented in a more + # general way, returning an array, thinking in a future possible extension) + (s, fixed_doc) = rule['func'](db_conn[db_name][col]) if s is not None: logger.warning(f'DB {db_name} {rule["label"]} violation in {col} collection: {s}') - fails += 1 + n_fails += 1 - # second process not global rules, per collection - autofix = { - 'entities': [], - 'csubs': [] - } - for col in ['entities', 'csubs']: + # second: process not global rules, per collection + for col in collections_inventory: for doc in db_conn[db_name][col].find(queries[col]): n[col] += 1 doc_fail = False @@ -957,22 +958,29 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp if s is not None: logger.warning(f'DB {db_name} {rule["label"]} violation for {id_string}: {s}') doc_fail = True - fails += 1 + n_fails += 1 if fixed_doc is not None: - autofix[col].append(fixed_doc) + modified_docs[col].append(fixed_doc) if doc_fail: - failed_docs[col] += 1 - + n_failed[col] += 1 - for col in ['entities', 'csubs']: + for col in collections_inventory: if n[col] > 0: logger.info( - f'processed {db_name} in collection {col}: {failed_docs[col]}/{n[col]} ({round(failed_docs[col] / n[col] * 100, 2)}%) failed docs') + f'processed {db_name} in collection {col}: {n_failed[col]}/{n[col]} ({round(n_failed[col] / n[col] * 100, 2)}%) failed docs') - # TODO: implement save logic for autofix dict + for col in collections_inventory: + bulk = [] + logger.debug(f'about to update in {col} collection: {modified_docs[col]}') + for doc in modified_docs[col]: + bulk.append(ReplaceOne({'_id': doc['_id']}, doc)) + logger.info(f'{len(bulk)} documents in {col} collection could be fixed') + if autofix and len(bulk) > 0: + logger.info(f'updating {len(bulk)} documents in {col} collection...') + db_conn[db_name][col].bulk_write(bulk) - return fails + return n_fails if __name__ == '__main__': @@ -994,7 +1002,7 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp 'all subscriptions in the collection will be checked. Applies to Rule Sxx rules.') parser.add_argument('--rules-exp', dest='rules_exp', help='Specifies the rules to apply, as a regular expression. By default all rules are applied.') - parser.add_argument('--autofix', dest='atutofix', action='store_true', + parser.add_argument('--autofix', dest='autofix', action='store_true', help='Applies some automatic fixes. Not for all rules. Check documentation. WARNING: this ' 'operation may modify OrionDBs, use with care') parser.add_argument('--logLevel', dest='log_level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'], default='INFO', From b7ad3ac333cbbdf5ddb657fb247c41d065ea0766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 10 Apr 2024 13:37:52 +0200 Subject: [PATCH 213/390] FIX minor fix --- scripts/oriondb_consistency/oriondb_consistency.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/oriondb_consistency/oriondb_consistency.py b/scripts/oriondb_consistency/oriondb_consistency.py index b9ab0efb85..eefb2c306d 100644 --- a/scripts/oriondb_consistency/oriondb_consistency.py +++ b/scripts/oriondb_consistency/oriondb_consistency.py @@ -975,10 +975,11 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp logger.debug(f'about to update in {col} collection: {modified_docs[col]}') for doc in modified_docs[col]: bulk.append(ReplaceOne({'_id': doc['_id']}, doc)) - logger.info(f'{len(bulk)} documents in {col} collection could be fixed') - if autofix and len(bulk) > 0: - logger.info(f'updating {len(bulk)} documents in {col} collection...') - db_conn[db_name][col].bulk_write(bulk) + if len(bulk) > 0: + logger.info(f'{len(bulk)} documents in {col} collection could be fixed') + if autofix: + logger.info(f'updating {len(bulk)} documents in {col} collection...') + db_conn[db_name][col].bulk_write(bulk) return n_fails From 4dbc1bf22d6c835fd2eb4de33fd552eb3ed19502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 10 Apr 2024 13:47:02 +0200 Subject: [PATCH 214/390] FIX minor fix --- scripts/oriondb_consistency/oriondb_consistency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/oriondb_consistency/oriondb_consistency.py b/scripts/oriondb_consistency/oriondb_consistency.py index eefb2c306d..167fafa2be 100644 --- a/scripts/oriondb_consistency/oriondb_consistency.py +++ b/scripts/oriondb_consistency/oriondb_consistency.py @@ -976,9 +976,9 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp for doc in modified_docs[col]: bulk.append(ReplaceOne({'_id': doc['_id']}, doc)) if len(bulk) > 0: - logger.info(f'{len(bulk)} documents in {col} collection could be fixed') + logger.warn(f'{len(bulk)} documents in {col} collection could be fixed') if autofix: - logger.info(f'updating {len(bulk)} documents in {col} collection...') + logger.warn(f'updating {len(bulk)} documents in {col} collection...') db_conn[db_name][col].bulk_write(bulk) return n_fails From 95a95a1b162bef13ac6412d290f590655776dff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 10 Apr 2024 13:47:54 +0200 Subject: [PATCH 215/390] FIX minor fix --- scripts/oriondb_consistency/oriondb_consistency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/oriondb_consistency/oriondb_consistency.py b/scripts/oriondb_consistency/oriondb_consistency.py index 167fafa2be..db1cc84637 100644 --- a/scripts/oriondb_consistency/oriondb_consistency.py +++ b/scripts/oriondb_consistency/oriondb_consistency.py @@ -976,9 +976,9 @@ def process_db(logger, db_name, db_conn, include_entity_date, queries, rules_exp for doc in modified_docs[col]: bulk.append(ReplaceOne({'_id': doc['_id']}, doc)) if len(bulk) > 0: - logger.warn(f'{len(bulk)} documents in {col} collection could be fixed') + logger.warning(f'{len(bulk)} documents in {col} collection could be fixed') if autofix: - logger.warn(f'updating {len(bulk)} documents in {col} collection...') + logger.warning(f'updating {len(bulk)} documents in {col} collection...') db_conn[db_name][col].bulk_write(bulk) return n_fails From b94324cdd80ac8da3174e3765887f1aa02a934c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 10 Apr 2024 16:34:47 +0200 Subject: [PATCH 216/390] FIX improvements --- scripts/oriondb_consistency/oriondb_consistency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/oriondb_consistency/oriondb_consistency.py b/scripts/oriondb_consistency/oriondb_consistency.py index db1cc84637..cc09290d7f 100644 --- a/scripts/oriondb_consistency/oriondb_consistency.py +++ b/scripts/oriondb_consistency/oriondb_consistency.py @@ -737,7 +737,7 @@ def ruleS90(csub): See README.md for an explanation of the rule """ if csub['format'] == 'JSON': - r = f"notification legacy format in use (endpoint: {csub['reference']})" + r = f"notification legacy format in use (endpoint: {csub['reference']}, servicePath: {csub['servicePath']})" fixed_csub = copy.deepcopy(csub) fixed_csub['format'] = 'normalized' else: From 3968dfa14480082c16386446f545ee14f2d465db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 12 Apr 2024 14:36:05 +0200 Subject: [PATCH 217/390] FIX remove Python jexl implementation --- CMakeLists.txt | 4 +- ci/deb/build-dep.sh | 2 - ci/deb/makefile | 3 - docker/Dockerfile | 1 - docker/Dockerfile.alpine | 1 - src/lib/common/JsonHelper.cpp | 11 ++ src/lib/common/JsonHelper.h | 1 + src/lib/expressions/CMakeLists.txt | 2 - src/lib/expressions/ExprContext.cpp | 181 ++++------------------- src/lib/expressions/ExprContext.h | 27 ++-- src/lib/expressions/ExprManager.cpp | 79 +++------- src/lib/expressions/ExprManager.h | 11 +- src/lib/expressions/ExprResult.cpp | 3 +- src/lib/expressions/ExprResult.h | 3 +- src/lib/expressions/exprCommon.cpp | 60 -------- src/lib/expressions/exprCommon.h | 41 ----- src/lib/ngsiNotify/Notifier.cpp | 1 + src/lib/serviceRoutines/versionTreat.cpp | 12 +- test/unittests/CMakeLists.txt | 1 - 19 files changed, 89 insertions(+), 355 deletions(-) delete mode 100644 src/lib/expressions/exprCommon.cpp delete mode 100644 src/lib/expressions/exprCommon.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 03bfaab443..b871e47ad8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,6 +234,7 @@ find_package (mongoc-1.0 1.24.3 EXACT) SET (COMMON_STATIC_LIBS microhttpd.a mosquitto.a + cjexl.a mongo::mongoc_static ) @@ -246,7 +247,6 @@ SET (DYNAMIC_LIBS uuid crypto sasl2 - python3.11 ) # @@ -265,8 +265,6 @@ endif (UNIT_TEST) # Common include # include_directories("/usr/include") -include_directories("/usr/include/python3.11/") - # Needed for the new C driver include_directories("/usr/local/include/libmongoc-1.0") diff --git a/ci/deb/build-dep.sh b/ci/deb/build-dep.sh index aac74f5988..ea8531172a 100755 --- a/ci/deb/build-dep.sh +++ b/ci/deb/build-dep.sh @@ -55,7 +55,6 @@ echo "INSTALL: MongoDB shell" \ && apt-get -y update \ && apt-get -y install mongodb-mongosh -# tcjexl is needed by Context Broker functionality, the others are for ftest suite echo "INSTALL: python special dependencies" \ && cd /opt \ && python3 -m venv /opt/ft_env \ @@ -64,7 +63,6 @@ echo "INSTALL: python special dependencies" \ && pip install Werkzeug==2.0.2 \ && pip install paho-mqtt==1.6.1 \ && pip install amqtt==0.11.0b1 \ -&& pip install tcjexl==0.2.0 \ && deactivate # Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0 diff --git a/ci/deb/makefile b/ci/deb/makefile index 7646a27dbd..be524b592f 100644 --- a/ci/deb/makefile +++ b/ci/deb/makefile @@ -69,9 +69,6 @@ install_unit: install unit: @echo '------------------------------------- make unit ----------------------------------------' - # ft_env needed for pyjexl module. Very important to "chain" the two commands with '; \' - # otherwise it seems unitTest doesn't get the venv and it uses default Python in the system - . /opt/ft_env/bin/activate ; \ BUILD_UNIT/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:/tmp/builder/logs/unit.xml build_functional: prepare diff --git a/docker/Dockerfile b/docker/Dockerfile index a54f4844f4..b5b80157d4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -88,7 +88,6 @@ RUN \ cd cmake-build && \ # Different from ci/deb/build-dep.sh and build from source documentation, we add here also # the MONGOC_TEST_USE_CRYPT_SHARED=FALSE. It needs Python and in this tiny image we don't have it - # FIXME PR: now that Python is going to be include for JEXL functionalty, maybe this should be changed... cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DMONGOC_TEST_USE_CRYPT_SHARED=FALSE .. && \ make && \ make install && \ diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index b871d42a5d..524335f677 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -93,7 +93,6 @@ RUN \ cd cmake-build && \ # Different from ci/deb/build-dep.sh and build from source documentation, we add here also # the MONGOC_TEST_USE_CRYPT_SHARED=FALSE. It needs Python and in this tiny image we don't have it - # FIXME PR: now that Python is going to be include for JEXL functionalty, maybe this should be changed... cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DMONGOC_TEST_USE_CRYPT_SHARED=FALSE .. && \ make && \ make install && \ diff --git a/src/lib/common/JsonHelper.cpp b/src/lib/common/JsonHelper.cpp index 812e9ee1f1..3897f011cf 100644 --- a/src/lib/common/JsonHelper.cpp +++ b/src/lib/common/JsonHelper.cpp @@ -406,6 +406,17 @@ void JsonVectorHelper::addBool(bool b) +/* **************************************************************************** +* +* JsonObjectHelper::addNull - +*/ +void JsonVectorHelper::addNull(void) +{ + addRaw("null"); +} + + + /* **************************************************************************** * * JsonVectorHelper::str - diff --git a/src/lib/common/JsonHelper.h b/src/lib/common/JsonHelper.h index 52e633c14d..47a52c0ff0 100644 --- a/src/lib/common/JsonHelper.h +++ b/src/lib/common/JsonHelper.h @@ -64,6 +64,7 @@ class JsonVectorHelper void addNumber(double value); void addDate(double timestamp); void addBool(bool b); + void addNull(void); std::string str(); diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index bfdd9da83c..4f938e3634 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -21,7 +21,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES - exprCommon.cpp ExprManager.cpp ExprContext.cpp ExprResult.cpp @@ -29,7 +28,6 @@ SET (SOURCES ) SET (HEADERS - exprCommon.h ExprManager.h ExprContext.h ExprResult.h diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 46e85c5e1e..4b59840475 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -28,8 +28,6 @@ #include "common/string.h" #include "logMsg/logMsg.h" #include "expressions/ExprContext.h" -#include "expressions/exprCommon.h" - /* **************************************************************************** @@ -39,18 +37,6 @@ ExprContextObject::ExprContextObject(bool _legacy) { legacy = _legacy; - - if (!legacy) - { - jexlContext = PyDict_New(); - - if (jexlContext == NULL) - { - // Note that this ruins the object. We log this situation just one time here, but we include a NULL check - // in every other method to be safer - LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); - } - } } @@ -59,16 +45,16 @@ ExprContextObject::ExprContextObject(bool _legacy) * * ExprContextObject::getJexlContext - */ -PyObject* ExprContextObject::getJexlContext(void) +std::string ExprContextObject::getJexlContext(void) { - return jexlContext; + return jh.str(); } /* **************************************************************************** * -* ExprContextObject::getJexlContext - +* ExprContextObject::getMap - */ std::map* ExprContextObject::getMap(void) { @@ -96,19 +82,8 @@ void ExprContextObject::add(const std::string &key, const std::string &_value, b } else { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context object (string): %s=%s", key.c_str(), _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - if (value == NULL) - { - LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); - return; - } - PyDict_SetItemString(jexlContext, key.c_str(), value); - Py_DECREF(value); + jh.addString(key, _value); } } @@ -127,19 +102,8 @@ void ExprContextObject::add(const std::string &key, double _value) } else { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context object (double): %s=%f", key.c_str(), _value)); - PyObject* value = Py_BuildValue("d", _value); - if (value == NULL) - { - LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); - return; - } - PyDict_SetItemString(jexlContext, key.c_str(), value); - Py_DECREF(value); + jh.addNumber(key, _value); } } @@ -158,19 +122,8 @@ void ExprContextObject::add(const std::string &key, bool _value) } else { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); - if (_value) - { - PyDict_SetItemString(jexlContext, key.c_str(), Py_True); - } - else - { - PyDict_SetItemString(jexlContext, key.c_str(), Py_False); - } + jh.addBool(key, _value); } } @@ -189,12 +142,8 @@ void ExprContextObject::add(const std::string &key) } else { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context object (none): %s", key.c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), Py_None); + jh.addNull(key); } } @@ -212,13 +161,9 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex } else { - if (jexlContext == NULL) - { - return; - } - // FIXME P3: improve this implemeting a toString() method for ExprContextObject - LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=[object]", key.c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), exprContextObject.getJexlContext()); + std::string s = exprContextObject.getJexlContext(); + LM_T(LmtExpr, ("adding to JEXL expression context object (object): %s=%s", key.c_str(), s.c_str())); + jh.addRaw(key, s); } } @@ -236,13 +181,9 @@ void ExprContextObject::add(const std::string &key, ExprContextList exprContextL } else { - if (jexlContext == NULL) - { - return; - } - // FIXME P3: improve this implemeting a toString() method for ExprContextList - LM_T(LmtExpr, ("adding to JEXL expression context object (list): %s=[list]", key.c_str())); - PyDict_SetItemString(jexlContext, key.c_str(), exprContextList.get()); + std::string s = exprContextList.get(); + LM_T(LmtExpr, ("adding to JEXL expression context object (list): %s=%s", key.c_str(), s.c_str())); + jh.addRaw(key, s); } } @@ -265,12 +206,7 @@ bool ExprContextObject::isLegacy(void) */ void ExprContextObject::release(void) { - if (jexlContext == NULL) - { - return; - } - // FIXME PR: this is not correct. Recursively release of the dict object - //Py_XDECREF(jexlContext); + // FIXME PR: this method is probably no longer needed } @@ -281,14 +217,7 @@ void ExprContextObject::release(void) */ ExprContextList::ExprContextList() { - jexlContext = PyList_New(0); - - if (jexlContext == NULL) - { - // Note that this ruins the object. We log this situation just one time here, but we include a NULL check - // in every other method to be safer - LM_E(("Runtime Error (fail creating new dict: %s)", capturePythonError())); - } + // FIXME PR: this method is probably no longer needed } @@ -297,9 +226,9 @@ ExprContextList::ExprContextList() * * ExprContextList::get - */ -PyObject* ExprContextList::get(void) +std::string ExprContextList::get(void) { - return jexlContext; + return jh.str(); } @@ -310,19 +239,9 @@ PyObject* ExprContextList::get(void) */ void ExprContextList::add(const std::string &_value) { - if (jexlContext == NULL) - { - return; - } - LM_T(LmtExpr, ("adding to JEXL expression context list (string): %s=", _value.c_str())); - PyObject* value = Py_BuildValue("s", _value.c_str()); - if (value == NULL) - { - LM_E(("Runtime Error (fail creating string value: %s)", capturePythonError())); - return; - } - PyList_Append(jexlContext, value); - Py_DECREF(value); + + LM_T(LmtExpr, ("adding to JEXL expression context list (string): %s", _value.c_str())); + jh.addString(_value); } @@ -333,19 +252,8 @@ void ExprContextList::add(const std::string &_value) */ void ExprContextList::add(double _value) { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context list (double): %f", _value)); - PyObject* value = Py_BuildValue("d", _value); - if (value == NULL) - { - LM_E(("Runtime Error (fail creating double value: %s)", capturePythonError())); - return; - } - PyList_Append(jexlContext, value); - Py_DECREF(value); + jh.addNumber(_value); } @@ -356,19 +264,8 @@ void ExprContextList::add(double _value) */ void ExprContextList::add(bool _value) { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context list (bool): %s", _value ? "true" : "false")); - if (_value) - { - PyList_Append(jexlContext, Py_True); - } - else - { - PyList_Append(jexlContext, Py_False); - } + jh.addBool(_value); } @@ -379,12 +276,8 @@ void ExprContextList::add(bool _value) */ void ExprContextList::add(void) { - if (jexlContext == NULL) - { - return; - } LM_T(LmtExpr, ("adding to JEXL expression context list (none)")); - PyList_Append(jexlContext, Py_None); + jh.addNull(); } @@ -395,13 +288,9 @@ void ExprContextList::add(void) */ void ExprContextList::add(ExprContextObject exprContextObject) { - if (jexlContext == NULL) - { - return; - } - // FIXME P3: improve this implemeting a toString() method for ExprContextObject - LM_T(LmtExpr, ("adding to JEXL expression context list (object): [object]")); - PyList_Append(jexlContext, exprContextObject.getJexlContext()); + std::string s = exprContextObject.getJexlContext(); + LM_T(LmtExpr, ("adding to JEXL expression context list (object): %s", s.c_str())); + jh.addRaw(s); } @@ -412,13 +301,10 @@ void ExprContextList::add(ExprContextObject exprContextObject) */ void ExprContextList::add(ExprContextList exprContextList) { - if (jexlContext == NULL) - { - return; - } - // FIXME P3: improve this implemeting a toString() method for ExprContextList - LM_T(LmtExpr, ("adding to JEXL expression context list (list): [list]")); - PyList_Append(jexlContext, exprContextList.get()); + + std::string s = exprContextList.get(); + LM_T(LmtExpr, ("adding to JEXL expression context list (list): %s", s.c_str())); + jh.addRaw(s); } @@ -429,10 +315,5 @@ void ExprContextList::add(ExprContextList exprContextList) */ void ExprContextList::release(void) { - if (jexlContext == NULL) - { - return; - } - // FIXME PR: this is not correct. Recursively release of the list object - //Py_XDECREF(jexlContext); + // FIXME PR: this method is probably no longer needed } \ No newline at end of file diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index a3068b3138..41a9acd980 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -26,10 +26,11 @@ * Author: Fermin Galan */ -#include #include #include +#include "common/JsonHelper.h" + class ExprContextList; // forward declaration /* **************************************************************************** @@ -40,13 +41,13 @@ class ExprContextObject { private: bool legacy; - PyObject* jexlContext; // used in regular (i.e. not legacy) mode - std::map repl; // used in legacy mode + JsonObjectHelper jh; // used in regular (i.e. not legacy) mode + std::map repl; // used in legacy mode public: ExprContextObject(bool legacy = false); - PyObject* getJexlContext(void); + std::string getJexlContext(void); std::map* getMap(void); void add(const std::string& key, const std::string& value, bool raw = false); @@ -64,20 +65,20 @@ class ExprContextObject class ExprContextList { private: - PyObject* jexlContext; + JsonVectorHelper jh; public: ExprContextList(); - PyObject* get(void); - void add(const std::string& value); - void add(double value); - void add(bool value); - void add(void); - void add(ExprContextObject exprContextObject); - void add(ExprContextList exprContextList); + std::string get(void); + void add(const std::string& value); + void add(double value); + void add(bool value); + void add(void); + void add(ExprContextObject exprContextObject); + void add(ExprContextList exprContextList); - void release(void); + void release(void); }; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index d87f37f536..a9e16fa34b 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -23,15 +23,24 @@ * Author: Fermin Galan */ -#include - #include "expressions/ExprManager.h" #include "expressions/ExprResult.h" -#include "expressions/exprCommon.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" +// Interface to use libcjexl.a +extern "C" { + void* new_engine(); +} + +extern "C" { + void free_engine(void* ptr); +} + +extern "C" { + const char* eval(void* ptr, const char* script_ptr, const char* context_ptr); +} /* **************************************************************************** @@ -40,30 +49,13 @@ */ void ExprManager::init(void) { - tcjexlModule = NULL; - jexlEngine = NULL; + // FIXME PR: this is probably not needed + //if (sem_init(&sem, 0, 1) == -1) + //{ + // LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); + //} - if (sem_init(&sem, 0, 1) == -1) - { - LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); - } - - Py_Initialize(); - LM_T(LmtExpr, ("Python interpreter has been initialized")); - - tcjexlModule = PyImport_ImportModule("tcjexl"); - if (tcjexlModule == NULL) - { - LM_X(1, ("Fatal Error (error importing tcjexl module: %s)", capturePythonError())); - } - LM_T(LmtExpr, ("tcjexl module has been loaded")); - - jexlEngine = PyObject_CallMethod(tcjexlModule, "JEXL", NULL); - if (jexlEngine == NULL) - { - LM_X(1, ("Fatal Error (error creating jexlEngine: %s)", capturePythonError())); - } - LM_T(LmtExpr, ("jexl engine has been created")); + jexlEngine = new_engine(); } @@ -95,26 +87,8 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st { // JEXL based evaluation LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - - PyObject* expression = Py_BuildValue("s", _expression.c_str()); - if (expression == NULL) - { - LM_W(("error building JEXL expression: %s", capturePythonError())); - return r; - } - - PyObject* result = PyObject_CallMethod(jexlEngine, "evaluate", "OO", expression, exprContextObjectP->getJexlContext()); - Py_XDECREF(expression); - if (result == NULL) - { - LM_W(("error evaluating JEXL expression: %s", capturePythonError())); - return r; - } - + const char* result = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); r.fill(result); - - // FIXME PR: does this Py_XDECREF() recursively in the case of dicts or lists? - Py_XDECREF(result); } return r; @@ -128,18 +102,5 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st */ void ExprManager::release(void) { - if (jexlEngine != NULL) - { - Py_XDECREF(jexlEngine); - LM_T(LmtExpr, ("jexl engine has been freed")); - } - - if (tcjexlModule != NULL) - { - Py_XDECREF(tcjexlModule); - LM_T(LmtExpr, ("tcjexl module has been freed")); - } - - Py_Finalize(); - LM_T(LmtExpr, ("Python interpreter has been finalized")); + free_engine(jexlEngine); } \ No newline at end of file diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 55c8e90f96..821f7e9ad9 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -26,8 +26,8 @@ * Author: Fermin Galan */ -#include -#include +// FIXME PR: probably this is no longer needed +//#include #include "expressions/ExprContext.h" #include "expressions/ExprResult.h" @@ -39,10 +39,9 @@ class ExprManager { private: - PyObject* tcjexlModule; - PyObject* jexlEngine; - PyObject* jsonModule; - sem_t sem; + // FIXME PR: probably this is no longer needed + //sem_t sem; + void* jexlEngine; public: void init(void); diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index a57cbc9ab4..8c3dd8d86d 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -24,7 +24,6 @@ */ #include "expressions/ExprResult.h" -#include "expressions/exprCommon.h" #include "common/string.h" #include "common/JsonHelper.h" @@ -256,7 +255,7 @@ void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* * fill - * */ -void ExprResult::fill(PyObject* result) +void ExprResult::fill(std::string result) { // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index df1ec0f597..12dd928497 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -30,7 +30,6 @@ #include "parse/CompoundValueNode.h" -#include #include /* **************************************************************************** @@ -50,7 +49,7 @@ class ExprResult // Use only when valueType is object or vector orion::CompoundValueNode* compoundValueP; - void fill(PyObject* result); + void fill(std::string result); std::string toString(void); void release(void); diff --git a/src/lib/expressions/exprCommon.cpp b/src/lib/expressions/exprCommon.cpp deleted file mode 100644 index 1372d73eaa..0000000000 --- a/src/lib/expressions/exprCommon.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -#include "expressions/exprCommon.h" - -/* **************************************************************************** -* -* capturePythonError - -*/ -const char* capturePythonError() -{ - if (PyErr_Occurred()) - { - PyObject* ptype; - PyObject* pvalue; - PyObject* ptraceback; - - // Fetch the exception type, value, and traceback - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - - if (pvalue != NULL) - { - PyObject* strObj = PyObject_Str(pvalue); - const char* errorMessage = PyUnicode_AsUTF8(strObj); - - // Release the Python objects - Py_XDECREF(strObj); - Py_XDECREF(ptype); - Py_XDECREF(pvalue); - Py_XDECREF(ptraceback); - - return errorMessage; - } - } - - PyErr_Clear(); - return ""; -} \ No newline at end of file diff --git a/src/lib/expressions/exprCommon.h b/src/lib/expressions/exprCommon.h deleted file mode 100644 index 7b9667d1b4..0000000000 --- a/src/lib/expressions/exprCommon.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ -#define SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ - -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -#include - - - -/* **************************************************************************** -* -* capturePythonError - -*/ -extern const char* capturePythonError(); - - - -#endif // SRC_LIB_EXPRESSIONS_EXPRCOMMON_H_ diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 6ee599bd5f..0c6fe587ff 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -321,6 +321,7 @@ static SenderThreadParams* buildSenderParamsCustom bool legacy = (exprLang == "legacy"); ExprContextObject exprContext(legacy); + // FIXME PR: no-legacy context is now based in JsonHelper, which may mess with key repetition. Check this // It seems that add() semantics are different in legacy and jexl mode. In jexl mode, if the key already exists, it is // updated. In legacy model, if the key already exists, the operation is ignored (so previous value is preserved). Taking // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index 5ea06218d9..30a42ff25a 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -46,7 +46,6 @@ #include #include #include -#include /* **************************************************************************** * @@ -71,7 +70,7 @@ std::string libVersions(void) std::string mhd = " \"libmicrohttpd\": "; std::string ssl = " \"openssl\": "; std::string rjson = " \"rapidjson\": "; - std::string python = " \"libpython\": "; + std::string cjexl = " \"libcjexl\": "; std::string mongo = " \"mongoc\": "; std::string bson = " \"bson\": "; @@ -86,17 +85,12 @@ std::string libVersions(void) char mosqVersion[16]; snprintf(mosqVersion, sizeof(mosqVersion), "%d.%d.%d", mosqMayor, mosqMinor, mosqRevision); - char pyVersion[16]; - snprintf(pyVersion, sizeof(pyVersion), "%d.%d.%d", - (PY_VERSION_HEX >> 24) & 0xFF, - (PY_VERSION_HEX >> 16) & 0xFF, - (PY_VERSION_HEX >> 8) & 0xFF); - total += boost + "\"" + BOOST_LIB_VERSION "\"" + ",\n"; total += curl + "\"" + curlVersion + "\"" + ",\n"; total += mosq + "\"" + mosqVersion + "\"" + ",\n"; total += mhd + "\"" + MHD_get_version() + "\"" + ",\n"; - total += python + "\"" + pyVersion + "\"" + ",\n"; + // FIXME PR + //total += cjexl + "\"" + cjexlVersion + "\"" + ",\n"; #ifdef OLD_SSL_VERSION_FORMAT // Needed by openssl 1.1.1n in Debian 11 and before total += ssl + "\"" + SHLIB_VERSION_NUMBER "\"" + ",\n"; diff --git a/test/unittests/CMakeLists.txt b/test/unittests/CMakeLists.txt index a929c69b39..af8fbaf709 100644 --- a/test/unittests/CMakeLists.txt +++ b/test/unittests/CMakeLists.txt @@ -217,7 +217,6 @@ include_directories("${PROJECT_SOURCE_DIR}/test") # Needed for the new C driver # FIXME: why do we need this declared here if they are also declared # in the main CMakeFiles.txt? -include_directories("/usr/include/python3.11/") include_directories("/usr/local/include/libmongoc-1.0") include_directories("/usr/local/include/libbson-1.0") From 6504fe66905cb497ae7d8660d6b12af2a125a787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Apr 2024 13:42:13 +0200 Subject: [PATCH 218/390] FIX solution without ExprResult class (have problemas with rendering decimals) --- src/lib/common/macroSubstitute.cpp | 20 +- src/lib/expressions/CMakeLists.txt | 2 - src/lib/expressions/ExprManager.cpp | 13 +- src/lib/expressions/ExprManager.h | 3 +- src/lib/expressions/ExprResult.cpp | 383 ---------------------------- src/lib/expressions/ExprResult.h | 60 ----- 6 files changed, 15 insertions(+), 466 deletions(-) delete mode 100644 src/lib/expressions/ExprResult.cpp delete mode 100644 src/lib/expressions/ExprResult.h diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 23b631a710..d9ba2bc54a 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -53,8 +53,8 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - ExprResult r = exprMgr.evaluate(exprContextObjectP, macroName); - if (r.valueType == orion::ValueTypeNull) + std::string r = exprMgr.evaluate(exprContextObjectP, macroName); + if (r == "null") { return notFoundDefault; } @@ -63,11 +63,11 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - return removeQuotes(r.toString()); + return removeQuotes(r); } else { - return r.toString(); + return r; } } } @@ -100,31 +100,31 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e */ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, const std::string key, const std::string& notFoundDefault, bool raw) { - ExprResult r = exprMgr.evaluate(exprContextObjectP, key); + std::string r = exprMgr.evaluate(exprContextObjectP, key); - if (r.valueType == orion::ValueTypeNull) + if (r == "null") { return notFoundDefault; } else { - std::string s = r.toString(); + std::string r; // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - s = removeQuotes(s); + r = removeQuotes(r); } if (raw) { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), // so double quotes have to be be removed - return removeQuotes(s); + return removeQuotes(r); } else { - return s; + return r; } } } diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index 4f938e3634..f38c3d8d6e 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -23,14 +23,12 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES ExprManager.cpp ExprContext.cpp - ExprResult.cpp exprMgr.cpp ) SET (HEADERS ExprManager.h ExprContext.h - ExprResult.h exprMgr.h ) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index a9e16fa34b..2d5dffa29e 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -24,7 +24,6 @@ */ #include "expressions/ExprManager.h" -#include "expressions/ExprResult.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" @@ -64,11 +63,9 @@ void ExprManager::init(void) * * ExprManager::evaluate - */ -ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) +std::string ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) { - ExprResult r; - r.valueType = orion::ValueTypeNull; - + std::string r; if (exprContextObjectP->isLegacy()) { // std::map based evaluation. Only pure replacement is supported @@ -79,16 +76,14 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st std::map::iterator iter = replacementsP->find(_expression); if (iter != replacementsP->end()) { - r.valueType = orion::ValueTypeString; - r.stringValue = iter->second; + r = iter->second; } } else { // JEXL based evaluation LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - const char* result = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); - r.fill(result); + r = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); } return r; diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 821f7e9ad9..c73a4ebc40 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -30,7 +30,6 @@ //#include #include "expressions/ExprContext.h" -#include "expressions/ExprResult.h" /* **************************************************************************** * @@ -45,7 +44,7 @@ class ExprManager public: void init(void); - ExprResult evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); + std::string evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); void release(void); }; diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp deleted file mode 100644 index 8c3dd8d86d..0000000000 --- a/src/lib/expressions/ExprResult.cpp +++ /dev/null @@ -1,383 +0,0 @@ -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -#include "expressions/ExprResult.h" - -#include "common/string.h" -#include "common/JsonHelper.h" -#include "logMsg/logMsg.h" - - - -/* **************************************************************************** -* -* getPyObjectType - -*/ -static orion::ValueType getPyObjectType(PyObject* obj) -{ - // PyBool_Check() has to be done before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be - // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 - if (PyBool_Check(obj)) - { - return orion::ValueTypeBoolean; - } - else if (PyLong_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyFloat_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyDict_Check(obj)) - { - return orion::ValueTypeObject; - } - else if (PyList_Check(obj)) - { - return orion::ValueTypeVector; - } - else if (obj == Py_None) - { - return orion::ValueTypeNull; - } - else - { - // For other types we use string (this is also a failsafe for types not being strings) - return orion::ValueTypeString; - } -} - - -static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); // forward declaration - - -/* **************************************************************************** -* -* processListItem - -* -*/ -void processListItem(orion::CompoundValueNode* parentP, PyObject* value) -{ - orion::CompoundValueNode* nodeP; - - const char* str; - double d; - bool b; - PyObject *keyAux, *valueAux; - Py_ssize_t pos, size; - - switch (getPyObjectType(value)) - { - case orion::ValueTypeString: - str = PyUnicode_AsUTF8(value); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - } - else - { - LM_T(LmtExpr, ("processListITem (string): %s", str)); - nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); - parentP->add(nodeP); - } - break; - - case orion::ValueTypeNumber: - d = PyFloat_AsDouble(value); - LM_T(LmtExpr, ("processList (double): %f", d)); - nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); - parentP->add(nodeP); - break; - - case orion::ValueTypeBoolean: - b = PyObject_IsTrue(value); - LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); - nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); - parentP->add(nodeP); - break; - - case orion::ValueTypeNull: - nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeNull); - parentP->add(nodeP); - break; - - case orion::ValueTypeVector: - nodeP = new orion::CompoundValueNode(orion::ValueTypeVector); - size = PyList_Size(value); - for (Py_ssize_t ix = 0; ix < size; ++ix) - { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(nodeP, PyList_GetItem(value, ix)); - } - parentP->add(nodeP); - break; - - case orion::ValueTypeObject: - nodeP = new orion::CompoundValueNode(orion::ValueTypeObject); - pos = 0; - while (PyDict_Next(value, &pos, &keyAux, &valueAux)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(nodeP, keyAux, valueAux); - } - parentP->add(nodeP); - break; - - case orion::ValueTypeNotGiven: - LM_E(("Runtime Error (value type not given))")); - break; - - default: - LM_E(("Runtime Error (value type unknown))")); - } -} - - - -/* **************************************************************************** -* -* processDictItem - -* -*/ -void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) -{ - const char * keyStr = PyUnicode_AsUTF8(key); - if (keyStr == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - return; - } - - orion::CompoundValueNode* nodeP; - const char* str; - double d; - bool b; - PyObject *keyAux, *valueAux; - Py_ssize_t pos, size; - - switch (getPyObjectType(value)) - { - case orion::ValueTypeString: - str = PyUnicode_AsUTF8(value); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - } - else - { - LM_T(LmtExpr, ("processListITem (string): %s", str)); - nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); - parentP->add(nodeP); - } - break; - - case orion::ValueTypeNumber: - d = PyFloat_AsDouble(value); - LM_T(LmtExpr, ("processList (double): %f", d)); - nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); - parentP->add(nodeP); - break; - - case orion::ValueTypeBoolean: - b = PyObject_IsTrue(value); - LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); - nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); - parentP->add(nodeP); - break; - - case orion::ValueTypeNull: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeNull); - parentP->add(nodeP); - break; - - case orion::ValueTypeVector: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeVector); - size = PyList_Size(value); - for (Py_ssize_t ix = 0; ix < size; ++ix) - { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(nodeP, PyList_GetItem(value, ix)); - } - parentP->add(nodeP); - break; - - case orion::ValueTypeObject: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeObject); - pos = 0; - while (PyDict_Next(value, &pos, &keyAux, &valueAux)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(nodeP, keyAux, valueAux); - } - parentP->add(nodeP); - break; - - case orion::ValueTypeNotGiven: - LM_E(("Runtime Error (value type not given))")); - break; - - default: - LM_E(("Runtime Error (value type unknown))")); - } -} - - - -/* **************************************************************************** -* -* fill - -* -*/ -void ExprResult::fill(std::string result) -{ - // If nothing changes, the returned value would be null (failsafe) - valueType = orion::ValueTypeNull; - - // Special case: expresion evalutes to None - if (result == Py_None) - { - LM_T(LmtExpr, ("ExprResult is null")); - valueType = orion::ValueTypeNull; - return; - } - - // Other not null types - valueType = getPyObjectType(result); - if (valueType == orion::ValueTypeNumber) - { - numberValue = PyFloat_AsDouble(result); - LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); - } - else if (valueType == orion::ValueTypeBoolean) - { - boolValue = PyObject_IsTrue(result); - LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); - } - else if (valueType == orion::ValueTypeObject) - { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(result, &pos, &key, &value)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(compoundValueP, key, value); - } - } - else if (valueType == orion::ValueTypeVector) - { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); - Py_ssize_t size = PyList_Size(result); - for (Py_ssize_t ix = 0; ix < size; ++ix) - { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(compoundValueP, PyList_GetItem(result, ix)); - } - } - else if (valueType == orion::ValueTypeString) - { - const char* str = PyUnicode_AsUTF8(result); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else - { - LM_T(LmtExpr, ("ExprResult (string): %s", str)); - stringValue = std::string(str); - } - } -} - - - -/* **************************************************************************** -* -* toString - -* -* Pretty similar to ContextAttribute::toJsonValue() -* -*/ -std::string ExprResult::toString(void) -{ - if (valueType == orion::ValueTypeNumber) - { - return double2string(numberValue); - } - else if (valueType == orion::ValueTypeBoolean) - { - return boolValue ? "true" : "false"; - } - else if (valueType == orion::ValueTypeString) - { - // FIXME PR: does this break the no legacy - //return "\"" + toJsonString(stringValue) + "\""; - return "\"" + stringValue + "\""; - } - else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) - { - if (compoundValueP != NULL) - { - return compoundValueP->toJson(); - } - else - { - LM_E(("Runtime Error (result is vector/object but compountValue is NULL)")); - return ""; - } - } - else if (valueType == orion::ValueTypeNull) - { - return "null"; - } - else - { - LM_E(("Runtime Error (not allowed type in ExprResult: %s)", valueTypeName(valueType))); - return ""; - } -} - - - -/* **************************************************************************** -* -* ExprResult::release - -*/ -void ExprResult::release(void) -{ - if (compoundValueP != NULL) - { - delete compoundValueP; - compoundValueP = NULL; - } -} \ No newline at end of file diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h deleted file mode 100644 index 12dd928497..0000000000 --- a/src/lib/expressions/ExprResult.h +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ -#define SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ - -/* -* -* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: Fermin Galan -*/ - -#include "orionTypes/OrionValueType.h" - -#include "parse/CompoundValueNode.h" - -#include - -/* **************************************************************************** -* -* ExprResult - -*/ -class ExprResult -{ -public: - // Similar to the fields used in ContextAttribute.h - - orion::ValueType valueType; // Type of value - std::string stringValue; // "value" as a String - double numberValue; // "value" as a Number - bool boolValue; // "value" as a Boolean - - // Use only when valueType is object or vector - orion::CompoundValueNode* compoundValueP; - - void fill(std::string result); - - std::string toString(void); - void release(void); -}; - - - -#endif // SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ From ed875f52937c1635c163d27129124c85db23e233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Apr 2024 13:42:58 +0200 Subject: [PATCH 219/390] Revert "FIX solution without ExprResult class (have problemas with rendering decimals)" This reverts commit 6ec9f58d85f71d2c96aa01e1bc4d5cceb5e97c37. --- src/lib/common/macroSubstitute.cpp | 20 +- src/lib/expressions/CMakeLists.txt | 2 + src/lib/expressions/ExprManager.cpp | 13 +- src/lib/expressions/ExprManager.h | 3 +- src/lib/expressions/ExprResult.cpp | 383 ++++++++++++++++++++++++++++ src/lib/expressions/ExprResult.h | 60 +++++ 6 files changed, 466 insertions(+), 15 deletions(-) create mode 100644 src/lib/expressions/ExprResult.cpp create mode 100644 src/lib/expressions/ExprResult.h diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index d9ba2bc54a..23b631a710 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -53,8 +53,8 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // len("${") + len("}") = 3 std::string macroName = stringValue.substr(2, stringValue.size() - 3); - std::string r = exprMgr.evaluate(exprContextObjectP, macroName); - if (r == "null") + ExprResult r = exprMgr.evaluate(exprContextObjectP, macroName); + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } @@ -63,11 +63,11 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - return removeQuotes(r); + return removeQuotes(r.toString()); } else { - return r; + return r.toString(); } } } @@ -100,31 +100,31 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e */ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, const std::string key, const std::string& notFoundDefault, bool raw) { - std::string r = exprMgr.evaluate(exprContextObjectP, key); + ExprResult r = exprMgr.evaluate(exprContextObjectP, key); - if (r == "null") + if (r.valueType == orion::ValueTypeNull) { return notFoundDefault; } else { - std::string r; + std::string s = r.toString(); // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - r = removeQuotes(r); + s = removeQuotes(s); } if (raw) { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), // so double quotes have to be be removed - return removeQuotes(r); + return removeQuotes(s); } else { - return r; + return s; } } } diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index f38c3d8d6e..4f938e3634 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -23,12 +23,14 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6) SET (SOURCES ExprManager.cpp ExprContext.cpp + ExprResult.cpp exprMgr.cpp ) SET (HEADERS ExprManager.h ExprContext.h + ExprResult.h exprMgr.h ) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 2d5dffa29e..a9e16fa34b 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -24,6 +24,7 @@ */ #include "expressions/ExprManager.h" +#include "expressions/ExprResult.h" #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" @@ -63,9 +64,11 @@ void ExprManager::init(void) * * ExprManager::evaluate - */ -std::string ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) +ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const std::string& _expression) { - std::string r; + ExprResult r; + r.valueType = orion::ValueTypeNull; + if (exprContextObjectP->isLegacy()) { // std::map based evaluation. Only pure replacement is supported @@ -76,14 +79,16 @@ std::string ExprManager::evaluate(ExprContextObject* exprContextObjectP, const s std::map::iterator iter = replacementsP->find(_expression); if (iter != replacementsP->end()) { - r = iter->second; + r.valueType = orion::ValueTypeString; + r.stringValue = iter->second; } } else { // JEXL based evaluation LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - r = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); + const char* result = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); + r.fill(result); } return r; diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index c73a4ebc40..821f7e9ad9 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -30,6 +30,7 @@ //#include #include "expressions/ExprContext.h" +#include "expressions/ExprResult.h" /* **************************************************************************** * @@ -44,7 +45,7 @@ class ExprManager public: void init(void); - std::string evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); + ExprResult evaluate(ExprContextObject* exprContextObjectP, const std::string& expression); void release(void); }; diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp new file mode 100644 index 0000000000..8c3dd8d86d --- /dev/null +++ b/src/lib/expressions/ExprResult.cpp @@ -0,0 +1,383 @@ +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "expressions/ExprResult.h" + +#include "common/string.h" +#include "common/JsonHelper.h" +#include "logMsg/logMsg.h" + + + +/* **************************************************************************** +* +* getPyObjectType - +*/ +static orion::ValueType getPyObjectType(PyObject* obj) +{ + // PyBool_Check() has to be done before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be + // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 + if (PyBool_Check(obj)) + { + return orion::ValueTypeBoolean; + } + else if (PyLong_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyFloat_Check(obj)) + { + return orion::ValueTypeNumber; + } + else if (PyDict_Check(obj)) + { + return orion::ValueTypeObject; + } + else if (PyList_Check(obj)) + { + return orion::ValueTypeVector; + } + else if (obj == Py_None) + { + return orion::ValueTypeNull; + } + else + { + // For other types we use string (this is also a failsafe for types not being strings) + return orion::ValueTypeString; + } +} + + +static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); // forward declaration + + +/* **************************************************************************** +* +* processListItem - +* +*/ +void processListItem(orion::CompoundValueNode* parentP, PyObject* value) +{ + orion::CompoundValueNode* nodeP; + + const char* str; + double d; + bool b; + PyObject *keyAux, *valueAux; + Py_ssize_t pos, size; + + switch (getPyObjectType(value)) + { + case orion::ValueTypeString: + str = PyUnicode_AsUTF8(value); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + } + else + { + LM_T(LmtExpr, ("processListITem (string): %s", str)); + nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); + parentP->add(nodeP); + } + break; + + case orion::ValueTypeNumber: + d = PyFloat_AsDouble(value); + LM_T(LmtExpr, ("processList (double): %f", d)); + nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); + parentP->add(nodeP); + break; + + case orion::ValueTypeBoolean: + b = PyObject_IsTrue(value); + LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); + nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); + parentP->add(nodeP); + break; + + case orion::ValueTypeNull: + nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeNull); + parentP->add(nodeP); + break; + + case orion::ValueTypeVector: + nodeP = new orion::CompoundValueNode(orion::ValueTypeVector); + size = PyList_Size(value); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(nodeP, PyList_GetItem(value, ix)); + } + parentP->add(nodeP); + break; + + case orion::ValueTypeObject: + nodeP = new orion::CompoundValueNode(orion::ValueTypeObject); + pos = 0; + while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() + processDictItem(nodeP, keyAux, valueAux); + } + parentP->add(nodeP); + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given))")); + break; + + default: + LM_E(("Runtime Error (value type unknown))")); + } +} + + + +/* **************************************************************************** +* +* processDictItem - +* +*/ +void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +{ + const char * keyStr = PyUnicode_AsUTF8(key); + if (keyStr == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + return; + } + + orion::CompoundValueNode* nodeP; + const char* str; + double d; + bool b; + PyObject *keyAux, *valueAux; + Py_ssize_t pos, size; + + switch (getPyObjectType(value)) + { + case orion::ValueTypeString: + str = PyUnicode_AsUTF8(value); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + } + else + { + LM_T(LmtExpr, ("processListITem (string): %s", str)); + nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); + parentP->add(nodeP); + } + break; + + case orion::ValueTypeNumber: + d = PyFloat_AsDouble(value); + LM_T(LmtExpr, ("processList (double): %f", d)); + nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); + parentP->add(nodeP); + break; + + case orion::ValueTypeBoolean: + b = PyObject_IsTrue(value); + LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); + nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); + parentP->add(nodeP); + break; + + case orion::ValueTypeNull: + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeNull); + parentP->add(nodeP); + break; + + case orion::ValueTypeVector: + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeVector); + size = PyList_Size(value); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(nodeP, PyList_GetItem(value, ix)); + } + parentP->add(nodeP); + break; + + case orion::ValueTypeObject: + nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeObject); + pos = 0; + while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() + processDictItem(nodeP, keyAux, valueAux); + } + parentP->add(nodeP); + break; + + case orion::ValueTypeNotGiven: + LM_E(("Runtime Error (value type not given))")); + break; + + default: + LM_E(("Runtime Error (value type unknown))")); + } +} + + + +/* **************************************************************************** +* +* fill - +* +*/ +void ExprResult::fill(std::string result) +{ + // If nothing changes, the returned value would be null (failsafe) + valueType = orion::ValueTypeNull; + + // Special case: expresion evalutes to None + if (result == Py_None) + { + LM_T(LmtExpr, ("ExprResult is null")); + valueType = orion::ValueTypeNull; + return; + } + + // Other not null types + valueType = getPyObjectType(result); + if (valueType == orion::ValueTypeNumber) + { + numberValue = PyFloat_AsDouble(result); + LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); + } + else if (valueType == orion::ValueTypeBoolean) + { + boolValue = PyObject_IsTrue(result); + LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); + } + else if (valueType == orion::ValueTypeObject) + { + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(result, &pos, &key, &value)) + { + // No need to free memory of each dict item here, the whole result object is freed + // in ExprManager::evaluate() + processDictItem(compoundValueP, key, value); + } + } + else if (valueType == orion::ValueTypeVector) + { + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); + Py_ssize_t size = PyList_Size(result); + for (Py_ssize_t ix = 0; ix < size; ++ix) + { + // No need to free memory of each list item here, the whole result object is freed + // in ExprManager::evaluate() + processListItem(compoundValueP, PyList_GetItem(result, ix)); + } + } + else if (valueType == orion::ValueTypeString) + { + const char* str = PyUnicode_AsUTF8(result); + if (str == NULL) + { + LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); + valueType = orion::ValueTypeNull; + } + else + { + LM_T(LmtExpr, ("ExprResult (string): %s", str)); + stringValue = std::string(str); + } + } +} + + + +/* **************************************************************************** +* +* toString - +* +* Pretty similar to ContextAttribute::toJsonValue() +* +*/ +std::string ExprResult::toString(void) +{ + if (valueType == orion::ValueTypeNumber) + { + return double2string(numberValue); + } + else if (valueType == orion::ValueTypeBoolean) + { + return boolValue ? "true" : "false"; + } + else if (valueType == orion::ValueTypeString) + { + // FIXME PR: does this break the no legacy + //return "\"" + toJsonString(stringValue) + "\""; + return "\"" + stringValue + "\""; + } + else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) + { + if (compoundValueP != NULL) + { + return compoundValueP->toJson(); + } + else + { + LM_E(("Runtime Error (result is vector/object but compountValue is NULL)")); + return ""; + } + } + else if (valueType == orion::ValueTypeNull) + { + return "null"; + } + else + { + LM_E(("Runtime Error (not allowed type in ExprResult: %s)", valueTypeName(valueType))); + return ""; + } +} + + + +/* **************************************************************************** +* +* ExprResult::release - +*/ +void ExprResult::release(void) +{ + if (compoundValueP != NULL) + { + delete compoundValueP; + compoundValueP = NULL; + } +} \ No newline at end of file diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h new file mode 100644 index 0000000000..12dd928497 --- /dev/null +++ b/src/lib/expressions/ExprResult.h @@ -0,0 +1,60 @@ +#ifndef SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ +#define SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermin Galan +*/ + +#include "orionTypes/OrionValueType.h" + +#include "parse/CompoundValueNode.h" + +#include + +/* **************************************************************************** +* +* ExprResult - +*/ +class ExprResult +{ +public: + // Similar to the fields used in ContextAttribute.h + + orion::ValueType valueType; // Type of value + std::string stringValue; // "value" as a String + double numberValue; // "value" as a Number + bool boolValue; // "value" as a Boolean + + // Use only when valueType is object or vector + orion::CompoundValueNode* compoundValueP; + + void fill(std::string result); + + std::string toString(void); + void release(void); +}; + + + +#endif // SRC_LIB_EXPRESSIONS_EXPRRESULT_H_ From 9cc132a643efe567f3db7a096e89b14f362d292e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Apr 2024 16:54:21 +0200 Subject: [PATCH 220/390] FIX new cjexl function names and cjexl_version() usage --- src/lib/expressions/ExprManager.cpp | 12 ++++++------ src/lib/expressions/ExprResult.cpp | 11 +++++++++-- src/lib/serviceRoutines/versionTreat.cpp | 8 ++++++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index a9e16fa34b..195e3c480a 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -31,15 +31,15 @@ // Interface to use libcjexl.a extern "C" { - void* new_engine(); + void* cjexl_new_engine(); } extern "C" { - void free_engine(void* ptr); + void cjexl_free_engine(void* ptr); } extern "C" { - const char* eval(void* ptr, const char* script_ptr, const char* context_ptr); + const char* cjexl_eval(void* ptr, const char* script_ptr, const char* context_ptr); } @@ -55,7 +55,7 @@ void ExprManager::init(void) // LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); //} - jexlEngine = new_engine(); + jexlEngine = cjexl_new_engine(); } @@ -87,7 +87,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st { // JEXL based evaluation LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - const char* result = eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); + const char* result = cjexl_eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); r.fill(result); } @@ -102,5 +102,5 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st */ void ExprManager::release(void) { - free_engine(jexlEngine); + cjexl_free_engine(jexlEngine); } \ No newline at end of file diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 8c3dd8d86d..92e40fb3f3 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -23,6 +23,11 @@ * Author: Fermin Galan */ +// The ExprResult class is used as return value in ExprManager::evaluate(). We could return std::string +// in that function and simplify (so avoiding the ExprResult class). But in that case float rounding is +// problematic (e.g. 1.999999 instead of 2), as they don't take advanage of the ad hoc logic implemented +// in ContextAttribute rendering + #include "expressions/ExprResult.h" #include "common/string.h" @@ -31,6 +36,7 @@ +#if 0 /* **************************************************************************** * * getPyObjectType - @@ -247,7 +253,7 @@ void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* LM_E(("Runtime Error (value type unknown))")); } } - +#endif /* **************************************************************************** @@ -257,6 +263,7 @@ void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* */ void ExprResult::fill(std::string result) { + /* // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; @@ -316,7 +323,7 @@ void ExprResult::fill(std::string result) LM_T(LmtExpr, ("ExprResult (string): %s", str)); stringValue = std::string(str); } - } + }*/ } diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index 30a42ff25a..eec99efedb 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -47,6 +47,11 @@ #include #include +// Interface to use libcjexl.a +extern "C" { + const char* cjexl_version(); +} + /* **************************************************************************** * * version - @@ -89,8 +94,7 @@ std::string libVersions(void) total += curl + "\"" + curlVersion + "\"" + ",\n"; total += mosq + "\"" + mosqVersion + "\"" + ",\n"; total += mhd + "\"" + MHD_get_version() + "\"" + ",\n"; - // FIXME PR - //total += cjexl + "\"" + cjexlVersion + "\"" + ",\n"; + total += cjexl + "\"" + cjexl_version() + "\"" + ",\n"; #ifdef OLD_SSL_VERSION_FORMAT // Needed by openssl 1.1.1n in Debian 11 and before total += ssl + "\"" + SHLIB_VERSION_NUMBER "\"" + ",\n"; From 8097a8d2dbb33a8dfb6c13a7d6905a6031f0a901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 15 Apr 2024 18:05:46 +0200 Subject: [PATCH 221/390] FIX ExprResult implementation --- src/lib/common/JsonHelper.cpp | 18 +- src/lib/common/JsonHelper.h | 2 + src/lib/expressions/ExprManager.cpp | 9 +- src/lib/expressions/ExprResult.cpp | 332 +++++++----------- .../jexl_array_filtering.test | 29 +- 5 files changed, 165 insertions(+), 225 deletions(-) diff --git a/src/lib/common/JsonHelper.cpp b/src/lib/common/JsonHelper.cpp index 3897f011cf..3bd6fd5d2a 100644 --- a/src/lib/common/JsonHelper.cpp +++ b/src/lib/common/JsonHelper.cpp @@ -138,7 +138,7 @@ std::string objectToJson(std::map& list) * * JsonObjectHelper - */ -JsonObjectHelper::JsonObjectHelper(): empty(true) +JsonObjectHelper::JsonObjectHelper(): empty(true), closed(false) { ss += '{'; } @@ -282,7 +282,12 @@ void JsonObjectHelper::addNull(const std::string& key) */ std::string JsonObjectHelper::str() { - ss += '}'; + // This check allows to call str() several times (needed when this is used in ExprContext) + if (!closed) + { + ss += '}'; + closed = true; + } return ss; } @@ -292,7 +297,7 @@ std::string JsonObjectHelper::str() * * JsonVectorHelper - */ -JsonVectorHelper::JsonVectorHelper(): empty(true) +JsonVectorHelper::JsonVectorHelper(): empty(true), closed(false) { ss += '['; } @@ -423,6 +428,11 @@ void JsonVectorHelper::addNull(void) */ std::string JsonVectorHelper::str() { - ss += ']'; + // This check allows to call str() several times (needed when this is used in ExprContext) + if (!closed) + { + ss += ']'; + closed = true; + } return ss; } diff --git a/src/lib/common/JsonHelper.h b/src/lib/common/JsonHelper.h index 47a52c0ff0..bcb8aaeaf2 100644 --- a/src/lib/common/JsonHelper.h +++ b/src/lib/common/JsonHelper.h @@ -50,6 +50,7 @@ class JsonObjectHelper private: std::string ss; bool empty; + bool closed; }; @@ -72,6 +73,7 @@ class JsonVectorHelper private: std::string ss; bool empty; + bool closed; }; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 195e3c480a..86e24cbae0 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -72,7 +72,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st if (exprContextObjectP->isLegacy()) { // std::map based evaluation. Only pure replacement is supported - LM_T(LmtExpr, ("evaluating legacy expresion: <%s>", _expression.c_str())); + LM_T(LmtExpr, ("evaluating legacy expression: <%s>", _expression.c_str())); std::map* replacementsP = exprContextObjectP->getMap(); @@ -81,13 +81,16 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st { r.valueType = orion::ValueTypeString; r.stringValue = iter->second; + LM_T(LmtExpr, ("legacy evaluation result: <%s>", r.stringValue.c_str())); } } else { // JEXL based evaluation - LM_T(LmtExpr, ("evaluating JEXL expresion: <%s>", _expression.c_str())); - const char* result = cjexl_eval(jexlEngine, _expression.c_str(), exprContextObjectP->getJexlContext().c_str()); + std::string context = exprContextObjectP->getJexlContext(); + LM_T(LmtExpr, ("evaluating JEXL expression <%s> with context <%s>", _expression.c_str(), context.c_str())); + const char* result = cjexl_eval(jexlEngine, _expression.c_str(), context.c_str()); + LM_T(LmtExpr, ("JEXL evaluation result: <%s>", result)); r.fill(result); } diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 92e40fb3f3..939c3ebfc8 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -28,56 +28,21 @@ // problematic (e.g. 1.999999 instead of 2), as they don't take advanage of the ad hoc logic implemented // in ContextAttribute rendering +#include "rapidjson/document.h" + #include "expressions/ExprResult.h" #include "common/string.h" #include "common/JsonHelper.h" #include "logMsg/logMsg.h" +#include "jsonParseV2/jsonParseTypeNames.h" +#include "jsonParseV2/utilsParse.h" -#if 0 -/* **************************************************************************** -* -* getPyObjectType - -*/ -static orion::ValueType getPyObjectType(PyObject* obj) -{ - // PyBool_Check() has to be done before than PyLong_Check(). Alternatively, PyLong_CheckExact() could be - // used (not tested). See: https://stackoverflow.com/q/77990353/1485926 - if (PyBool_Check(obj)) - { - return orion::ValueTypeBoolean; - } - else if (PyLong_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyFloat_Check(obj)) - { - return orion::ValueTypeNumber; - } - else if (PyDict_Check(obj)) - { - return orion::ValueTypeObject; - } - else if (PyList_Check(obj)) - { - return orion::ValueTypeVector; - } - else if (obj == Py_None) - { - return orion::ValueTypeNull; - } - else - { - // For other types we use string (this is also a failsafe for types not being strings) - return orion::ValueTypeString; - } -} +static void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstMemberIterator& iter); // forward declaration -static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value); // forward declaration /* **************************************************************************** @@ -85,81 +50,58 @@ static void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, Py * processListItem - * */ -void processListItem(orion::CompoundValueNode* parentP, PyObject* value) +void processListItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstValueIterator& iter) { orion::CompoundValueNode* nodeP; - const char* str; - double d; - bool b; - PyObject *keyAux, *valueAux; - Py_ssize_t pos, size; + std::string type = jsonParseTypeNames[iter->GetType()]; - switch (getPyObjectType(value)) + if (type == "String") { - case orion::ValueTypeString: - str = PyUnicode_AsUTF8(value); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - } - else - { - LM_T(LmtExpr, ("processListITem (string): %s", str)); - nodeP = new orion::CompoundValueNode("", str, orion::ValueTypeString); - parentP->add(nodeP); - } - break; - - case orion::ValueTypeNumber: - d = PyFloat_AsDouble(value); - LM_T(LmtExpr, ("processList (double): %f", d)); - nodeP = new orion::CompoundValueNode("", d, orion::ValueTypeNumber); + nodeP = new orion::CompoundValueNode("", iter->GetString(), orion::ValueTypeString); parentP->add(nodeP); - break; - - case orion::ValueTypeBoolean: - b = PyObject_IsTrue(value); - LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); - nodeP = new orion::CompoundValueNode("", b, orion::ValueTypeBoolean); + } + else if (type == "Number") + { + nodeP = new orion::CompoundValueNode("", iter->GetDouble(), orion::ValueTypeNumber); parentP->add(nodeP); - break; - - case orion::ValueTypeNull: + } + else if (type == "True") + { + nodeP = new orion::CompoundValueNode("", true, orion::ValueTypeBoolean); + parentP->add(nodeP); + } + else if (type == "False") + { + nodeP = new orion::CompoundValueNode("", false, orion::ValueTypeBoolean); + parentP->add(nodeP); + } + else if (type == "Null") + { nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeNull); parentP->add(nodeP); - break; - - case orion::ValueTypeVector: - nodeP = new orion::CompoundValueNode(orion::ValueTypeVector); - size = PyList_Size(value); - for (Py_ssize_t ix = 0; ix < size; ++ix) + } + else if (type == "Array") + { + nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeVector); + for (rapidjson::Value::ConstValueIterator iter2 = iter->Begin(); iter2 != iter->End(); ++iter2) { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(nodeP, PyList_GetItem(value, ix)); + processListItem(nodeP, iter2); } parentP->add(nodeP); - break; - - case orion::ValueTypeObject: - nodeP = new orion::CompoundValueNode(orion::ValueTypeObject); - pos = 0; - while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + } + else if (type == "Object") + { + nodeP = new orion::CompoundValueNode("", "", orion::ValueTypeObject); + for (rapidjson::Value::ConstMemberIterator iter2 = iter->MemberBegin(); iter2 != iter->MemberEnd(); ++iter2) { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(nodeP, keyAux, valueAux); + processDictItem(nodeP, iter2); } parentP->add(nodeP); - break; - - case orion::ValueTypeNotGiven: - LM_E(("Runtime Error (value type not given))")); - break; - - default: - LM_E(("Runtime Error (value type unknown))")); + } + else + { + LM_E(("Runtime Error (unknown type: %s)", type.c_str())); } } @@ -170,90 +112,62 @@ void processListItem(orion::CompoundValueNode* parentP, PyObject* value) * processDictItem - * */ -void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* value) +void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstMemberIterator& iter) { - const char * keyStr = PyUnicode_AsUTF8(key); - if (keyStr == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - return; - } - orion::CompoundValueNode* nodeP; - const char* str; - double d; - bool b; - PyObject *keyAux, *valueAux; - Py_ssize_t pos, size; - switch (getPyObjectType(value)) - { - case orion::ValueTypeString: - str = PyUnicode_AsUTF8(value); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - } - else - { - LM_T(LmtExpr, ("processListITem (string): %s", str)); - nodeP = new orion::CompoundValueNode(keyStr, str, orion::ValueTypeString); - parentP->add(nodeP); - } - break; + std::string name = iter->name.GetString(); + std::string type = jsonParseTypeNames[iter->value.GetType()]; - case orion::ValueTypeNumber: - d = PyFloat_AsDouble(value); - LM_T(LmtExpr, ("processList (double): %f", d)); - nodeP = new orion::CompoundValueNode(keyStr, d, orion::ValueTypeNumber); + if (type == "String") + { + nodeP = new orion::CompoundValueNode(name, iter->value.GetString(), orion::ValueTypeString); parentP->add(nodeP); - break; - - case orion::ValueTypeBoolean: - b = PyObject_IsTrue(value); - LM_T(LmtExpr, ("ExprResult (bool): %s", b ? "true": "false")); - nodeP = new orion::CompoundValueNode(keyStr, b, orion::ValueTypeBoolean); + } + else if (type == "Number") + { + nodeP = new orion::CompoundValueNode(name, iter->value.GetDouble(), orion::ValueTypeNumber); parentP->add(nodeP); - break; - - case orion::ValueTypeNull: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeNull); + } + else if (type == "True") + { + nodeP = new orion::CompoundValueNode(name, true, orion::ValueTypeBoolean); parentP->add(nodeP); - break; - - case orion::ValueTypeVector: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeVector); - size = PyList_Size(value); - for (Py_ssize_t ix = 0; ix < size; ++ix) + } + else if (type == "False") + { + nodeP = new orion::CompoundValueNode(name, false, orion::ValueTypeBoolean); + parentP->add(nodeP); + } + else if (type == "Null") + { + nodeP = new orion::CompoundValueNode(name, "", orion::ValueTypeNull); + parentP->add(nodeP); + } + else if (type == "Array") + { + nodeP = new orion::CompoundValueNode(name, "", orion::ValueTypeVector); + for (rapidjson::Value::ConstValueIterator iter2 = iter->value.Begin(); iter2 != iter->value.End(); ++iter2) { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(nodeP, PyList_GetItem(value, ix)); + processListItem(nodeP, iter2); } parentP->add(nodeP); - break; - - case orion::ValueTypeObject: - nodeP = new orion::CompoundValueNode(keyStr, "", orion::ValueTypeObject); - pos = 0; - while (PyDict_Next(value, &pos, &keyAux, &valueAux)) + } + else if (type == "Object") + { + nodeP = new orion::CompoundValueNode(name, "", orion::ValueTypeObject); + for (rapidjson::Value::ConstMemberIterator iter2 = iter->value.MemberBegin(); iter2 != iter->value.MemberEnd(); ++iter2) { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(nodeP, keyAux, valueAux); + processDictItem(nodeP, iter2); } parentP->add(nodeP); - break; - - case orion::ValueTypeNotGiven: - LM_E(("Runtime Error (value type not given))")); - break; - - default: - LM_E(("Runtime Error (value type unknown))")); + } + else + { + LM_E(("Runtime Error (unknown type: %s)", type.c_str())); } } -#endif + /* **************************************************************************** @@ -263,67 +177,67 @@ void processDictItem(orion::CompoundValueNode* parentP, PyObject* key, PyObject* */ void ExprResult::fill(std::string result) { - /* // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; - // Special case: expresion evalutes to None - if (result == Py_None) + rapidjson::Document document; + + document.Parse(result.c_str()); + + if (document.HasParseError()) { - LM_T(LmtExpr, ("ExprResult is null")); - valueType = orion::ValueTypeNull; + LM_E(("Runtime Error (parsing ExprResult: %s)", parseErrorString(document.GetParseError()).c_str())); return; } - // Other not null types - valueType = getPyObjectType(result); - if (valueType == orion::ValueTypeNumber) + std::string type = jsonParseTypeNames[document.GetType()]; + + if (type == "String") { - numberValue = PyFloat_AsDouble(result); - LM_T(LmtExpr, ("ExprResult (double): %f", numberValue)); + stringValue = document.GetString(); + valueType = orion::ValueTypeString; } - else if (valueType == orion::ValueTypeBoolean) + else if (type == "Number") { - boolValue = PyObject_IsTrue(result); - LM_T(LmtExpr, ("ExprResult (bool): %s", boolValue ? "true": "false")); + numberValue = document.GetDouble(); + valueType = orion::ValueTypeNumber; } - else if (valueType == orion::ValueTypeObject) + else if (type == "True") { - compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(result, &pos, &key, &value)) - { - // No need to free memory of each dict item here, the whole result object is freed - // in ExprManager::evaluate() - processDictItem(compoundValueP, key, value); - } + boolValue = true; + valueType = orion::ValueTypeBoolean; } - else if (valueType == orion::ValueTypeVector) + else if (type == "False") { + boolValue = false; + valueType = orion::ValueTypeBoolean; + } + else if (type == "Null") + { + valueType = orion::ValueTypeNull; + } + else if (type == "Array") + { + valueType = orion::ValueTypeVector; compoundValueP = new orion::CompoundValueNode(orion::ValueTypeVector); - Py_ssize_t size = PyList_Size(result); - for (Py_ssize_t ix = 0; ix < size; ++ix) + for (rapidjson::Value::ConstValueIterator iter = document.Begin(); iter != document.End(); ++iter) { - // No need to free memory of each list item here, the whole result object is freed - // in ExprManager::evaluate() - processListItem(compoundValueP, PyList_GetItem(result, ix)); + processListItem(compoundValueP, iter); } } - else if (valueType == orion::ValueTypeString) + else if (type == "Object") { - const char* str = PyUnicode_AsUTF8(result); - if (str == NULL) - { - LM_E(("Runtime Error (error obtaning str representation: %s)", capturePythonError())); - valueType = orion::ValueTypeNull; - } - else + valueType = orion::ValueTypeObject; + compoundValueP = new orion::CompoundValueNode(orion::ValueTypeObject); + for (rapidjson::Value::ConstMemberIterator iter = document.MemberBegin(); iter != document.MemberEnd(); ++iter) { - LM_T(LmtExpr, ("ExprResult (string): %s", str)); - stringValue = std::string(str); + processDictItem(compoundValueP, iter); } - }*/ + } + else + { + LM_E(("Runtime Error (unknown type: %s)", type.c_str())); + } } diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test index 0a3bb36e7d..d6428e2074 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test @@ -36,7 +36,7 @@ accumulatorStart --pretty-print # 03. Update entity E1 with a=[{c:1},{b:-1},{b:2}] # 04. Update entity E1 with a=[{b:-1},{b:foo}] # 05. Update entity E1 with a=[{b:-1, c:1},{b:20, d:3}] -# 06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}]) +# 06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=[{b:-1},{b:2}], cal=[{b:-1}], cal=[{b:-1, c:1}]) # @@ -123,8 +123,8 @@ echo echo -echo "06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}])" -echo "=====================================================================================================" +echo "06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=[{b:-1},{b:2}], cal=[{b:-1}], cal=[{b:-1, c:1}])" +echo "===================================================================================================================" accumulatorDump echo echo @@ -175,8 +175,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=null, cal=null, cal=[{b:-1, c:1}]) -===================================================================================================== +06. Dump accumulator and see notifications (cal=[{b:1},{b:4}], cal=[{b:-1},{b:2}], cal=[{b:-1}], cal=[{b:-1, c:1}]) +=================================================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 145 @@ -211,7 +211,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 132 +Content-Length: 146 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -225,7 +225,14 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "cal": { "metadata": {}, "type": "Calculated", - "value": null + "value": [ + { + "b": -1 + }, + { + "b": 2 + } + ] }, "id": "E1", "type": "T" @@ -236,7 +243,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 132 +Content-Length: 138 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -250,7 +257,11 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "cal": { "metadata": {}, "type": "Calculated", - "value": null + "value": [ + { + "b": -1 + } + ] }, "id": "E1", "type": "T" From c5411b32a7f8498519595a7cf5cbad2c5110b07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 18 Apr 2024 17:17:53 +0200 Subject: [PATCH 222/390] FIX cleanup --- src/lib/expressions/ExprContext.cpp | 33 ----------------------------- src/lib/expressions/ExprContext.h | 6 ------ src/lib/expressions/ExprManager.cpp | 6 ------ src/lib/expressions/ExprManager.h | 5 ----- src/lib/expressions/ExprResult.cpp | 2 +- src/lib/ngsiNotify/Notifier.cpp | 8 ------- 6 files changed, 1 insertion(+), 59 deletions(-) diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 4b59840475..6f8421ca1e 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -200,28 +200,6 @@ bool ExprContextObject::isLegacy(void) -/* **************************************************************************** -* -* ExprContextObject::release - -*/ -void ExprContextObject::release(void) -{ - // FIXME PR: this method is probably no longer needed -} - - - -/* **************************************************************************** -* -* ExprContextList::ExprContextList - -*/ -ExprContextList::ExprContextList() -{ - // FIXME PR: this method is probably no longer needed -} - - - /* **************************************************************************** * * ExprContextList::get - @@ -306,14 +284,3 @@ void ExprContextList::add(ExprContextList exprContextList) LM_T(LmtExpr, ("adding to JEXL expression context list (list): %s", s.c_str())); jh.addRaw(s); } - - - -/* **************************************************************************** -* -* ExprContextList::relesase - -*/ -void ExprContextList::release(void) -{ - // FIXME PR: this method is probably no longer needed -} \ No newline at end of file diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index 41a9acd980..aa92e3094a 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -58,8 +58,6 @@ class ExprContextObject void add(const std::string& key, ExprContextList exprContextList); bool isLegacy(void); - - void release(void); }; class ExprContextList @@ -68,8 +66,6 @@ class ExprContextList JsonVectorHelper jh; public: - ExprContextList(); - std::string get(void); void add(const std::string& value); void add(double value); @@ -77,8 +73,6 @@ class ExprContextList void add(void); void add(ExprContextObject exprContextObject); void add(ExprContextList exprContextList); - - void release(void); }; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 86e24cbae0..028b8808f7 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -49,12 +49,6 @@ extern "C" { */ void ExprManager::init(void) { - // FIXME PR: this is probably not needed - //if (sem_init(&sem, 0, 1) == -1) - //{ - // LM_X(1, ("Fatal Error (error initializing 'jexl mgr' semaphore: %s)", strerror(errno))); - //} - jexlEngine = cjexl_new_engine(); } diff --git a/src/lib/expressions/ExprManager.h b/src/lib/expressions/ExprManager.h index 821f7e9ad9..5d0892c078 100644 --- a/src/lib/expressions/ExprManager.h +++ b/src/lib/expressions/ExprManager.h @@ -26,9 +26,6 @@ * Author: Fermin Galan */ -// FIXME PR: probably this is no longer needed -//#include - #include "expressions/ExprContext.h" #include "expressions/ExprResult.h" @@ -39,8 +36,6 @@ class ExprManager { private: - // FIXME PR: probably this is no longer needed - //sem_t sem; void* jexlEngine; public: diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 939c3ebfc8..6bb5bf4e87 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -191,7 +191,7 @@ void ExprResult::fill(std::string result) } std::string type = jsonParseTypeNames[document.GetType()]; - + if (type == "String") { stringValue = document.GetString(); diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 0c6fe587ff..b13d113b5a 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -372,7 +372,6 @@ static SenderThreadParams* buildSenderParamsCustom if (macroSubstitute(&url, notifUrl, &exprContext, "", true) == false) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } @@ -389,7 +388,6 @@ static SenderThreadParams* buildSenderParamsCustom if (!setPayload(includePayload, notifPayload, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, &mimeType, &renderFormat)) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } } @@ -406,7 +404,6 @@ static SenderThreadParams* buildSenderParamsCustom if (!setNgsiPayload(ngsi, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } mimeType = "application/json"; @@ -426,7 +423,6 @@ static SenderThreadParams* buildSenderParamsCustom if ((macroSubstitute(&key, it->first, &exprContext, "", true) == false) || (macroSubstitute(&value, it->second, &exprContext, "", true) == false)) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } @@ -453,7 +449,6 @@ static SenderThreadParams* buildSenderParamsCustom if ((macroSubstitute(&key, it->first, &exprContext, "", true) == false) || (macroSubstitute(&value, it->second, &exprContext, "", true) == false)) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } @@ -485,7 +480,6 @@ static SenderThreadParams* buildSenderParamsCustom if (!parseUrl(url, host, port, uriPath, protocol)) { LM_E(("Runtime Error (not sending notification: malformed URL: '%s')", url.c_str())); - exprContext.release(); return NULL; } @@ -520,7 +514,6 @@ static SenderThreadParams* buildSenderParamsCustom if (macroSubstitute(&topic, notification.mqttInfo.topic, &exprContext, "", true) == false) { // Warning already logged in macroSubstitute() - exprContext.release(); return NULL; } } @@ -556,7 +549,6 @@ static SenderThreadParams* buildSenderParamsCustom snprintf(suffix, sizeof(suffix), "%u", correlatorCounter); paramsP->fiwareCorrelator = fiwareCorrelator + "; cbnotif=" + suffix; - exprContext.release(); return paramsP; } From 2a20139fb3a8b2d640d9c037f825e56ee498ca25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 19 Apr 2024 16:24:58 +0200 Subject: [PATCH 223/390] FIX align version tests --- .../cases/0000_version_operation/version_via_rest.test | 2 +- test/functionalTest/cases/0501_cors/version_request.test | 6 +++--- .../cases/1916_fiware_correlator/fiware_correlator.test | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/functionalTest/cases/0000_version_operation/version_via_rest.test b/test/functionalTest/cases/0000_version_operation/version_via_rest.test index e4fd1517f7..1ff1b3f428 100644 --- a/test/functionalTest/cases/0000_version_operation/version_via_rest.test +++ b/test/functionalTest/cases/0000_version_operation/version_via_rest.test @@ -64,7 +64,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/functionalTest/cases/0501_cors/version_request.test b/test/functionalTest/cases/0501_cors/version_request.test index 57b963b40a..7c56566ec8 100644 --- a/test/functionalTest/cases/0501_cors/version_request.test +++ b/test/functionalTest/cases/0501_cors/version_request.test @@ -94,7 +94,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -131,7 +131,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -166,7 +166,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", diff --git a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test index 01cd198b53..b7ec3feefb 100644 --- a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test +++ b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test @@ -73,7 +73,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", @@ -108,7 +108,7 @@ Content-Length: REGEX(\d+) "libcurl": "REGEX(.*)", "libmosquitto": "REGEX(.*)", "libmicrohttpd": "REGEX(.*)", - "libpython": "REGEX(.*)", + "libcjexl": "REGEX(.*)", "openssl": "REGEX(.*)", "rapidjson": "REGEX(.*)", "mongoc": "REGEX(.*)", From fb976cdcf590da1517d217ccc197fe678f9ae223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 22 Apr 2024 12:26:33 +0200 Subject: [PATCH 224/390] FIX unit tests --- test/unittests/serviceRoutines/versionTreat_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unittests/serviceRoutines/versionTreat_test.cpp b/test/unittests/serviceRoutines/versionTreat_test.cpp index fd8a4e2e4b..1ebdff883c 100644 --- a/test/unittests/serviceRoutines/versionTreat_test.cpp +++ b/test/unittests/serviceRoutines/versionTreat_test.cpp @@ -95,7 +95,7 @@ TEST(versionTreat, ok) EXPECT_TRUE(strstr(out.c_str(), "libcurl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "libmicrohttpd") != NULL); EXPECT_TRUE(strstr(out.c_str(), "libmosquitto") != NULL); - EXPECT_TRUE(strstr(out.c_str(), "libpython") != NULL); + EXPECT_TRUE(strstr(out.c_str(), "libcjexl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "openssl") != NULL); EXPECT_TRUE(strstr(out.c_str(), "rapidjson") != NULL); EXPECT_TRUE(strstr(out.c_str(), "mongoc") != NULL); From 3bc9432126a4663cab11e612a4f997596e375391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 22 Apr 2024 16:39:00 +0200 Subject: [PATCH 225/390] FIX sort keys in JSON used in test --- ...ttp_ngsi_adding_full_replacement_attribute_compound.test | 2 +- ..._http_ngsi_adding_full_replacement_attribute_simple.test | 2 +- ..._ngsi_adding_partial_replacement_attribute_compound.test | 6 +++--- ...tp_ngsi_adding_partial_replacement_attribute_simple.test | 4 ++-- ...custom_notification_http_ngsi_adding_with_filtering.test | 2 +- ...cation_http_ngsi_adding_with_filtering_in_compounds.test | 2 +- ...ation_http_ngsi_adding_without_any_source_attribute.test | 2 +- ...ion_http_ngsi_adding_without_some_source_attributes.test | 2 +- ...p_ngsi_mapping_partial_replacement_attribute_simple.test | 4 ++-- ...ustom_notification_http_ngsi_mapping_with_filtering.test | 2 +- ...qtt_ngsi_adding_full_replacement_attribute_compound.test | 2 +- ..._mqtt_ngsi_adding_full_replacement_attribute_simple.test | 2 +- ..._ngsi_adding_partial_replacement_attribute_compound.test | 6 +++--- ...tt_ngsi_adding_partial_replacement_attribute_simple.test | 4 ++-- ...custom_notification_mqtt_ngsi_adding_with_filtering.test | 2 +- ...cation_mqtt_ngsi_adding_with_filtering_in_compounds.test | 2 +- ...ation_mqtt_ngsi_adding_without_any_source_attribute.test | 2 +- ...ion_mqtt_ngsi_adding_without_some_source_attributes.test | 2 +- ...t_ngsi_mapping_partial_replacement_attribute_simple.test | 4 ++-- ...ustom_notification_mqtt_ngsi_mapping_with_filtering.test | 2 +- 20 files changed, 28 insertions(+), 28 deletions(-) diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_full_replacement_attribute_compound.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_full_replacement_attribute_compound.test index ebe0cb5d8b..3be1a4fdb1 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_full_replacement_attribute_compound.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_full_replacement_attribute_compound.test @@ -133,7 +133,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_full_replacement_attribute_simple.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_full_replacement_attribute_simple.test index 8a15bc1a87..bf02538d27 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_full_replacement_attribute_simple.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_full_replacement_attribute_simple.test @@ -137,7 +137,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_partial_replacement_attribute_compound.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_partial_replacement_attribute_compound.test index 663b4f7043..ad3063dd71 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_partial_replacement_attribute_compound.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_partial_replacement_attribute_compound.test @@ -133,7 +133,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { @@ -304,13 +304,13 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "metadata": {}, "type": "StructuredValue", "value": [ - "{\"z\":10,\"w\":\"ttt\"} thing", + "{\"w\":\"ttt\",\"z\":10} thing", { "one": [ "foo [4,null,{\"r\":2}] bar", "this is: null" ], - "two": "some other{\"z\":10,\"w\":\"ttt\"}" + "two": "some other{\"w\":\"ttt\",\"z\":10}" } ] }, diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_partial_replacement_attribute_simple.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_partial_replacement_attribute_simple.test index 2e3d8db9de..1e86e378d1 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_partial_replacement_attribute_simple.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_partial_replacement_attribute_simple.test @@ -129,7 +129,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { @@ -287,7 +287,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "A3": { "metadata": {}, "type": "Boolean", - "value": "this is an stringfied object: {\"z\":10,\"w\":\"ttt\"}" + "value": "this is an stringfied object: {\"w\":\"ttt\",\"z\":10}" }, "A4": { "metadata": {}, diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_with_filtering.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_with_filtering.test index 4255493534..af9e62b5bd 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_with_filtering.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_with_filtering.test @@ -138,7 +138,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_with_filtering_in_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_with_filtering_in_compounds.test index acd38a5ec8..71e4610065 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_with_filtering_in_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_with_filtering_in_compounds.test @@ -134,7 +134,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_without_any_source_attribute.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_without_any_source_attribute.test index b0c1b40ebf..f48d5143c3 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_without_any_source_attribute.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_without_any_source_attribute.test @@ -138,7 +138,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10}, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_without_some_source_attributes.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_without_some_source_attributes.test index 73bb5235d2..63854b353e 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_without_some_source_attributes.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_adding_without_some_source_attributes.test @@ -138,7 +138,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_mapping_partial_replacement_attribute_simple.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_mapping_partial_replacement_attribute_simple.test index 74dad59ff2..1bfbbda4de 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_mapping_partial_replacement_attribute_simple.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_mapping_partial_replacement_attribute_simple.test @@ -137,7 +137,7 @@ payload='{ "type": "irrelevant" }, "A4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10}, "type": "irrelevant" }, "A5": { @@ -273,7 +273,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "A4": { "metadata": {}, "type": "Text", - "value": "this is A4: {\"z\":10,\"w\":\"ttt\"}" + "value": "this is A4: {\"w\":\"ttt\",\"z\":10}" }, "A5": { "metadata": {}, diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_mapping_with_filtering.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_mapping_with_filtering.test index bcbf50f9ab..dc7dde790c 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_mapping_with_filtering.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_mapping_with_filtering.test @@ -138,7 +138,7 @@ payload='{ "type": "irrelevant" }, "A4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "A5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_full_replacement_attribute_compound.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_full_replacement_attribute_compound.test index c672a5e42c..6fc6da457c 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_full_replacement_attribute_compound.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_full_replacement_attribute_compound.test @@ -134,7 +134,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_full_replacement_attribute_simple.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_full_replacement_attribute_simple.test index c730e29d9a..5660df0539 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_full_replacement_attribute_simple.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_full_replacement_attribute_simple.test @@ -138,7 +138,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10}, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_partial_replacement_attribute_compound.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_partial_replacement_attribute_compound.test index 1987f33e6d..22d53e8eb2 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_partial_replacement_attribute_compound.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_partial_replacement_attribute_compound.test @@ -134,7 +134,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { @@ -296,13 +296,13 @@ MQTT message at topic topic1: "metadata": {}, "type": "StructuredValue", "value": [ - "{\"z\":10,\"w\":\"ttt\"} thing", + "{\"w\":\"ttt\",\"z\":10} thing", { "one": [ "foo [4,null,{\"r\":2}] bar", "this is: null" ], - "two": "some other{\"z\":10,\"w\":\"ttt\"}" + "two": "some other{\"w\":\"ttt\",\"z\":10}" } ] }, diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_partial_replacement_attribute_simple.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_partial_replacement_attribute_simple.test index c8bb90c233..55e35128bf 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_partial_replacement_attribute_simple.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_partial_replacement_attribute_simple.test @@ -130,7 +130,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { @@ -279,7 +279,7 @@ MQTT message at topic topic1: "A3": { "metadata": {}, "type": "Boolean", - "value": "this is an stringfied object: {\"z\":10,\"w\":\"ttt\"}" + "value": "this is an stringfied object: {\"w\":\"ttt\",\"z\":10}" }, "A4": { "metadata": {}, diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_with_filtering.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_with_filtering.test index f653f540bd..ffce938ef4 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_with_filtering.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_with_filtering.test @@ -139,7 +139,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_with_filtering_in_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_with_filtering_in_compounds.test index 07793365df..a917a86bf6 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_with_filtering_in_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_with_filtering_in_compounds.test @@ -135,7 +135,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_without_any_source_attribute.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_without_any_source_attribute.test index 68314e30d4..7f5d72bae0 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_without_any_source_attribute.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_without_any_source_attribute.test @@ -139,7 +139,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_without_some_source_attributes.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_without_some_source_attributes.test index e3984c6247..6863f958ce 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_without_some_source_attributes.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_adding_without_some_source_attributes.test @@ -139,7 +139,7 @@ payload='{ "type": "irrelevant" }, "B4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "B5": { diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_partial_replacement_attribute_simple.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_partial_replacement_attribute_simple.test index 9b3cf16f91..019b9acc92 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_partial_replacement_attribute_simple.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_partial_replacement_attribute_simple.test @@ -138,7 +138,7 @@ payload='{ "type": "irrelevant" }, "A4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10 }, "type": "irrelevant" }, "A5": { @@ -265,7 +265,7 @@ MQTT message at topic topic1: "A4": { "metadata": {}, "type": "Text", - "value": "this is A4: {\"z\":10,\"w\":\"ttt\"}" + "value": "this is A4: {\"w\":\"ttt\",\"z\":10}" }, "A5": { "metadata": {}, diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_with_filtering.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_with_filtering.test index 01ddd52702..13e0a73029 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_with_filtering.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_mapping_with_filtering.test @@ -139,7 +139,7 @@ payload='{ "type": "irrelevant" }, "A4": { - "value": { "z": 10, "w": "ttt"}, + "value": { "w": "ttt", "z": 10}, "type": "irrelevant" }, "A5": { From 30d4556c5d57df1fd85dc0b9049f56ca6c475b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Apr 2024 16:56:52 +0200 Subject: [PATCH 226/390] FIX libcjexl download in CI build.sh --- ci/deb/build.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index aea9ed627d..25479aeb9b 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -172,6 +172,20 @@ echo "===================================== PREPARE ============================ echo "Builder: create temp folders" rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} +# FIXME PR: unhardwire release number +if [ -z "${REPO_ACCESS_TOKEN}" ]; then + echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" +else + res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/0.0.0) + if [ "$res_code" -eq 200 ]; then + echo "Builder: downloading cjexl lib" + ASSET_ID=$(curl -s -S -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/0.0.0 | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') + curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $REPO_ACCESS_TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID + else + echo "Builder: error $res_code accessing cjexl release. Maybe token is invalid?" + fi +fi + if [ -n "${branch}" ]; then echo "===================================== CLONE ============================================" From 9b02e231a01152541e4cbca6dfcbe26062256aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Apr 2024 17:59:23 +0200 Subject: [PATCH 227/390] FIX add token to gitactions that need it --- .github/workflows/functional.yml | 2 +- .github/workflows/valgrind-nocache.yml | 2 +- .github/workflows/valgrind.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 7b6b61eeea..61b91cfb66 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -60,4 +60,4 @@ jobs: - name: Run functional test run: | - docker run --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional + docker run --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional diff --git a/.github/workflows/valgrind-nocache.yml b/.github/workflows/valgrind-nocache.yml index ea86291e59..fa4edadef8 100644 --- a/.github/workflows/valgrind-nocache.yml +++ b/.github/workflows/valgrind-nocache.yml @@ -58,4 +58,4 @@ jobs: - name: Run valgrind test run: | - docker run --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind + docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 67940e7d17..36933d8ea3 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -55,4 +55,4 @@ jobs: - name: Run valgrind test run: | - docker run --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind + docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind From 30911873609e67dbd799a7ed1a149748cb41c5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Apr 2024 18:09:41 +0200 Subject: [PATCH 228/390] FIX syntax error in GitActions --- .github/workflows/functional.yml | 2 +- .github/workflows/valgrind-nocache.yml | 2 +- .github/workflows/valgrind.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 61b91cfb66..b8b4bcb597 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -60,4 +60,4 @@ jobs: - name: Run functional test run: | - docker run --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional + docker run --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional diff --git a/.github/workflows/valgrind-nocache.yml b/.github/workflows/valgrind-nocache.yml index fa4edadef8..e8ff06e31b 100644 --- a/.github/workflows/valgrind-nocache.yml +++ b/.github/workflows/valgrind-nocache.yml @@ -58,4 +58,4 @@ jobs: - name: Run valgrind test run: | - docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind + docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 36933d8ea3..33010b755d 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -55,4 +55,4 @@ jobs: - name: Run valgrind test run: | - docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN ${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind + docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind From 280d8f71900c88cd611e9313560e7cd98a8a63e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 24 Apr 2024 18:31:49 +0200 Subject: [PATCH 229/390] FIX GitAction unit tests --- .github/workflows/unit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 722a7bb8db..fa1c722db5 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -31,4 +31,4 @@ jobs: - name: Run unit tests run: | - docker run --network host -t --rm -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts unit + docker run --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts unit From 21699835811039625f6ffd8aeb330410a348ff06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Apr 2024 16:18:50 +0200 Subject: [PATCH 230/390] FIX improve buid.sh --- ci/deb/build.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index 25479aeb9b..107cb11f30 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -173,14 +173,17 @@ echo "Builder: create temp folders" rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} # FIXME PR: unhardwire release number +CJEXL_VERSION="0.0.0" if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/0.0.0) + res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION) if [ "$res_code" -eq 200 ]; then echo "Builder: downloading cjexl lib" - ASSET_ID=$(curl -s -S -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/0.0.0 | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') + ASSET_ID=$(curl -s -S -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $REPO_ACCESS_TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID + MD5SUM=$(md5sum /usr/local/lib/libcjexl.a | awk -F ' ' '{print$1}') + echo "Builder: cjexl lib md5sum is $MD5SUM" else echo "Builder: error $res_code accessing cjexl release. Maybe token is invalid?" fi From 49aa74c1fe2e6d770a27b375835683f03134d0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Apr 2024 16:19:15 +0200 Subject: [PATCH 231/390] FIX ExprResult::release() in the proper places --- src/lib/common/macroSubstitute.cpp | 19 +++++++++++++------ src/lib/expressions/ExprManager.cpp | 3 +++ src/lib/expressions/ExprResult.cpp | 4 ++-- src/lib/expressions/ExprResult.h | 2 +- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index 23b631a710..cc994b70cd 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -54,22 +54,25 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e std::string macroName = stringValue.substr(2, stringValue.size() - 3); ExprResult r = exprMgr.evaluate(exprContextObjectP, macroName); + std::string result; if (r.valueType == orion::ValueTypeNull) { - return notFoundDefault; + result = notFoundDefault; } else { // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. if (exprContextObjectP->isLegacy()) { - return removeQuotes(r.toString()); + result = removeQuotes(r.toString()); } else { - return r.toString(); + result = r.toString(); } } + r.release(); + return result; } else if (exprContextObjectP != NULL) { @@ -101,10 +104,11 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, const std::string key, const std::string& notFoundDefault, bool raw) { ExprResult r = exprMgr.evaluate(exprContextObjectP, key); + std::string result; if (r.valueType == orion::ValueTypeNull) { - return notFoundDefault; + result = notFoundDefault; } else { @@ -120,13 +124,16 @@ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, c { // This means that the expression is in the middle of the string (i.e. partial replacement and not full replacement), // so double quotes have to be be removed - return removeQuotes(s); + result = removeQuotes(s); } else { - return s; + result = s; } } + + r.release(); + return result; } diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 028b8808f7..0ee2fcb23d 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -85,6 +85,9 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st LM_T(LmtExpr, ("evaluating JEXL expression <%s> with context <%s>", _expression.c_str(), context.c_str())); const char* result = cjexl_eval(jexlEngine, _expression.c_str(), context.c_str()); LM_T(LmtExpr, ("JEXL evaluation result: <%s>", result)); + + // The ExprResult::fill() method allocates dynamic memory. So, the callers to evaluate() are supposed to invoke ExprResult::release() + // method to free it r.fill(result); } diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 6bb5bf4e87..e071a7d2ea 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -25,7 +25,7 @@ // The ExprResult class is used as return value in ExprManager::evaluate(). We could return std::string // in that function and simplify (so avoiding the ExprResult class). But in that case float rounding is -// problematic (e.g. 1.999999 instead of 2), as they don't take advanage of the ad hoc logic implemented +// problematic (e.g. 1.999999 instead of 2), as they don't take advantage of the ad hoc logic implemented // in ContextAttribute rendering #include "rapidjson/document.h" @@ -175,7 +175,7 @@ void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value:: * fill - * */ -void ExprResult::fill(std::string result) +void ExprResult::fill(const std::string& result) { // If nothing changes, the returned value would be null (failsafe) valueType = orion::ValueTypeNull; diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index 12dd928497..8b9eeb8233 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -49,7 +49,7 @@ class ExprResult // Use only when valueType is object or vector orion::CompoundValueNode* compoundValueP; - void fill(std::string result); + void fill(const std::string& result); std::string toString(void); void release(void); From c9a0e5866ccbb8be0defd0bfdeba230de55ab876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 25 Apr 2024 16:56:04 +0200 Subject: [PATCH 232/390] FIX add ExprResult constructor to initialize compoundValueP to NULL --- src/lib/expressions/ExprResult.cpp | 20 ++++++++++++++++---- src/lib/expressions/ExprResult.h | 2 ++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index e071a7d2ea..5f34023565 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -50,7 +50,7 @@ static void processDictItem(orion::CompoundValueNode* parentP, const rapidjson:: * processListItem - * */ -void processListItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstValueIterator& iter) +static void processListItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstValueIterator& iter) { orion::CompoundValueNode* nodeP; @@ -112,7 +112,7 @@ void processListItem(orion::CompoundValueNode* parentP, const rapidjson::Value:: * processDictItem - * */ -void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstMemberIterator& iter) +static void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value::ConstMemberIterator& iter) { orion::CompoundValueNode* nodeP; @@ -172,7 +172,19 @@ void processDictItem(orion::CompoundValueNode* parentP, const rapidjson::Value:: /* **************************************************************************** * -* fill - +* ExprResult::ExprResult - +* +*/ +ExprResult::ExprResult() +{ + compoundValueP = NULL; +} + + + +/* **************************************************************************** +* +* ExprResult::fill - * */ void ExprResult::fill(const std::string& result) @@ -244,7 +256,7 @@ void ExprResult::fill(const std::string& result) /* **************************************************************************** * -* toString - +* ExprResult::toString - * * Pretty similar to ContextAttribute::toJsonValue() * diff --git a/src/lib/expressions/ExprResult.h b/src/lib/expressions/ExprResult.h index 8b9eeb8233..9814d3073b 100644 --- a/src/lib/expressions/ExprResult.h +++ b/src/lib/expressions/ExprResult.h @@ -49,6 +49,8 @@ class ExprResult // Use only when valueType is object or vector orion::CompoundValueNode* compoundValueP; + ExprResult(void); + void fill(const std::string& result); std::string toString(void); From ecf7fbd2117e6610bc428ead4c5ea9ca58ffb838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Apr 2024 13:40:51 +0200 Subject: [PATCH 233/390] FIX add expr fields in timing section in GET /statistics --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/admin/statistics.md | 10 +++ src/lib/common/statistics.cpp | 16 +++++ src/lib/common/statistics.h | 101 ++++++++++++++++++++++++++++ src/lib/expressions/ExprManager.cpp | 7 ++ src/lib/ngsiNotify/Notifier.cpp | 2 + src/lib/rest/rest.cpp | 4 ++ 7 files changed, 141 insertions(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 8cb6e6e302..9c42cb5f02 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ - Add: exprLang field in custom notifications (#4004) +- Add: expression context build and evaluation counter in timing section in GET /statistics (#4004) - Fix: lighter operation to get databases list from MongoDB (#4517) \ No newline at end of file diff --git a/doc/manuals/admin/statistics.md b/doc/manuals/admin/statistics.md index 768368691f..a2760412f2 100644 --- a/doc/manuals/admin/statistics.md +++ b/doc/manuals/admin/statistics.md @@ -133,6 +133,10 @@ Provides timing information, i.e. the time that CB passes executing in different "mongoReadWait": 4656.924425628, "mongoWriteWait": 259.347915990, "mongoCommandWait": 0.514811318, + "exprLegacyCtxBld": FIXME PR, + "exprLegacyEval": FIXME PR, + "exprJexlCtxBld": FIXME PR, + "exprJexlEval": FIXME PR, "render": 108.162782114, "total": 6476.593504743 }, @@ -140,6 +144,10 @@ Provides timing information, i.e. the time that CB passes executing in different "mongoBackend": 0.014752309, "mongoReadWait": 0.012018445, "mongoWriteWait": 0.000574611, + "exprLegacyCtxBld": FIXME PR, + "exprLegacyEval": FIXME PR, + "exprJexlCtxBld": FIXME PR, + "exprJexlEval": FIXME PR, "render": 0.000019136, "total": 0.015148915 } @@ -156,6 +164,8 @@ The block includes two main sections: The particular counters are as follows: +FIXME PR: explain expr* fields + * `total`: processing time for the whole request, excluding the time that the HTTP library takes for request/response dispatching (pseudo end-to-end time) * `jsonV1Parse`: time passed in NGSIv1 JSON parsing module (pseudo self-time) diff --git a/src/lib/common/statistics.cpp b/src/lib/common/statistics.cpp index 794eafee2b..1eb5dde069 100644 --- a/src/lib/common/statistics.cpp +++ b/src/lib/common/statistics.cpp @@ -195,6 +195,10 @@ std::string renderTimingStatistics(void) bool accMongoReadWaitTime = (accTimeStat.mongoReadWaitTime.tv_sec != 0) || (accTimeStat.mongoReadWaitTime.tv_nsec != 0); bool accMongoWriteWaitTime = (accTimeStat.mongoWriteWaitTime.tv_sec != 0) || (accTimeStat.mongoWriteWaitTime.tv_nsec != 0); bool accMongoCommandWaitTime = (accTimeStat.mongoCommandWaitTime.tv_sec != 0) || (accTimeStat.mongoCommandWaitTime.tv_nsec != 0); + bool accExprLegacyCtxBldTime = (accTimeStat.exprLegacyCtxBldTime.tv_sec != 0) || (accTimeStat.exprLegacyCtxBldTime.tv_nsec != 0); + bool accExprLegacyEvalTime = (accTimeStat.exprLegacyEvalTime.tv_sec != 0) || (accTimeStat.exprLegacyEvalTime.tv_nsec != 0); + bool accExprJexlCtxBldTime = (accTimeStat.exprJexlCtxBldTime.tv_sec != 0) || (accTimeStat.exprJexlCtxBldTime.tv_nsec != 0); + bool accExprJexlEvalTime = (accTimeStat.exprJexlEvalTime.tv_sec != 0) || (accTimeStat.exprJexlEvalTime.tv_nsec != 0); bool accRenderTime = (accTimeStat.renderTime.tv_sec != 0) || (accTimeStat.renderTime.tv_nsec != 0); bool accReqTime = (accTimeStat.reqTime.tv_sec != 0) || (accTimeStat.reqTime.tv_nsec != 0); @@ -204,6 +208,10 @@ std::string renderTimingStatistics(void) bool lastMongoReadWaitTime = (lastTimeStat.mongoReadWaitTime.tv_sec != 0) || (lastTimeStat.mongoReadWaitTime.tv_nsec != 0); bool lastMongoWriteWaitTime = (lastTimeStat.mongoWriteWaitTime.tv_sec != 0) || (lastTimeStat.mongoWriteWaitTime.tv_nsec != 0); bool lastMongoCommandWaitTime = (lastTimeStat.mongoCommandWaitTime.tv_sec != 0) || (lastTimeStat.mongoCommandWaitTime.tv_nsec != 0); + bool lastExprLegacyCtxBldTime = (lastTimeStat.exprLegacyCtxBldTime.tv_sec != 0) || (lastTimeStat.exprLegacyCtxBldTime.tv_nsec != 0); + bool lastExprLegacyEvalTime = (lastTimeStat.exprLegacyEvalTime.tv_sec != 0) || (lastTimeStat.exprLegacyEvalTime.tv_nsec != 0); + bool lastExprJexlCtxBldTime = (lastTimeStat.exprJexlCtxBldTime.tv_sec != 0) || (lastTimeStat.exprJexlCtxBldTime.tv_nsec != 0); + bool lastExprJexlEvalTime = (lastTimeStat.exprJexlEvalTime.tv_sec != 0) || (lastTimeStat.exprJexlEvalTime.tv_nsec != 0); bool lastRenderTime = (lastTimeStat.renderTime.tv_sec != 0) || (lastTimeStat.renderTime.tv_nsec != 0); bool lastReqTime = (lastTimeStat.reqTime.tv_sec != 0) || (lastTimeStat.reqTime.tv_nsec != 0); @@ -228,6 +236,10 @@ std::string renderTimingStatistics(void) if (accMongoReadWaitTime) accJh.addNumber("mongoReadWait", timeSpecToFloat(accTimeStat.mongoReadWaitTime)); if (accMongoWriteWaitTime) accJh.addNumber("mongoWriteWait", timeSpecToFloat(accTimeStat.mongoWriteWaitTime)); if (accMongoCommandWaitTime) accJh.addNumber("mongoCommandWait", timeSpecToFloat(accTimeStat.mongoCommandWaitTime)); + if (accExprLegacyCtxBldTime) accJh.addNumber("exprLegacyCtxBld", timeSpecToFloat(accTimeStat.exprLegacyCtxBldTime)); + if (accExprLegacyEvalTime) accJh.addNumber("exprLegacyEval", timeSpecToFloat(accTimeStat.exprLegacyEvalTime)); + if (accExprJexlCtxBldTime) accJh.addNumber("exprJexlCtxBld", timeSpecToFloat(accTimeStat.exprJexlCtxBldTime)); + if (accExprJexlEvalTime) accJh.addNumber("exprJexlEval", timeSpecToFloat(accTimeStat.exprJexlEvalTime)); if (accRenderTime) accJh.addNumber("render", timeSpecToFloat(accTimeStat.renderTime)); if (accReqTime) accJh.addNumber("total", timeSpecToFloat(accTimeStat.reqTime)); @@ -243,6 +255,10 @@ std::string renderTimingStatistics(void) if (lastMongoReadWaitTime) lastJh.addNumber("mongoReadWait", timeSpecToFloat(lastTimeStat.mongoReadWaitTime)); if (lastMongoWriteWaitTime) lastJh.addNumber("mongoWriteWait", timeSpecToFloat(lastTimeStat.mongoWriteWaitTime)); if (lastMongoCommandWaitTime) lastJh.addNumber("mongoCommandWait", timeSpecToFloat(lastTimeStat.mongoCommandWaitTime)); + if (lastExprLegacyCtxBldTime) lastJh.addNumber("exprLegacyCtxBld", timeSpecToFloat(lastTimeStat.exprLegacyCtxBldTime)); + if (lastExprLegacyEvalTime) lastJh.addNumber("exprLegacyEval", timeSpecToFloat(lastTimeStat.exprLegacyEvalTime)); + if (lastExprJexlCtxBldTime) lastJh.addNumber("exprJexlCtxBld", timeSpecToFloat(lastTimeStat.exprJexlCtxBldTime)); + if (lastExprJexlEvalTime) lastJh.addNumber("exprJexlEval", timeSpecToFloat(lastTimeStat.exprJexlEvalTime)); if (lastRenderTime) lastJh.addNumber("render", timeSpecToFloat(lastTimeStat.renderTime)); if (lastReqTime) lastJh.addNumber("total", timeSpecToFloat(lastTimeStat.reqTime)); diff --git a/src/lib/common/statistics.h b/src/lib/common/statistics.h index ab4034f5f2..623e074dcf 100644 --- a/src/lib/common/statistics.h +++ b/src/lib/common/statistics.h @@ -208,6 +208,103 @@ +/* **************************************************************************** +* +* TIME_EXPR_CTXBLD_START - +*/ +#define TIME_EXPR_CTXBLD_START() \ + struct timespec exprCtxBldStart; \ + struct timespec exprCtxBldEnd; \ + \ + if (timingStatistics) \ + { \ + clock_gettime(CLOCK_REALTIME, &exprCtxBldStart); \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_CTXBLD_STOP - +*/ +#define TIME_EXPR_CTXBLD_STOP() \ + if (timingStatistics) \ + { \ + struct timespec diff; \ + clock_gettime(CLOCK_REALTIME, &exprCtxBldEnd); \ + clock_difftime(&exprCtxBldEnd, &exprCtxBldStart, &diff); \ + if (legacy) \ + { \ + clock_addtime(&threadLastTimeStat.exprLegacyCtxBldTime, &diff); \ + } \ + else \ + { \ + clock_addtime(&threadLastTimeStat.exprJexlCtxBldTime, &diff); \ + } \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_LEGACY_EVAL_START - +*/ +#define TIME_EXPR_LEGACY_EVAL_START() \ + struct timespec exprLegacyEvalStart; \ + struct timespec exprLegacyEvalEnd; \ + \ + if (timingStatistics) \ + { \ + clock_gettime(CLOCK_REALTIME, &exprLegacyEvalStart); \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_LEGACY_EVAL_STOP - +*/ +#define TIME_EXPR_LEGACY_EVAL_STOP() \ + if (timingStatistics) \ + { \ + struct timespec diff; \ + clock_gettime(CLOCK_REALTIME, &exprLegacyEvalEnd); \ + clock_difftime(&exprLegacyEvalEnd, &exprLegacyEvalStart, &diff); \ + clock_addtime(&threadLastTimeStat.exprLegacyEvalTime, &diff); \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_JEXL_EVAL_START - +*/ +#define TIME_EXPR_JEXL_EVAL_START() \ + struct timespec exprJexlEvalStart; \ + struct timespec exprJexlEvalEnd; \ + \ + if (timingStatistics) \ + { \ + clock_gettime(CLOCK_REALTIME, &exprJexlEvalStart); \ + } + + + +/* **************************************************************************** +* +* TIME_EXPR_JEXL_EVAL_STOP - +*/ +#define TIME_EXPR_JEXL_EVAL_STOP() \ + if (timingStatistics) \ + { \ + struct timespec diff; \ + clock_gettime(CLOCK_REALTIME, &exprJexlEvalEnd); \ + clock_difftime(&exprJexlEvalEnd, &exprJexlEvalStart, &diff); \ + clock_addtime(&threadLastTimeStat.exprJexlEvalTime, &diff); \ + } + + + /* **************************************************************************** * * TimeStat - @@ -220,6 +317,10 @@ typedef struct TimeStat struct timespec mongoReadWaitTime; struct timespec mongoWriteWaitTime; struct timespec mongoCommandWaitTime; + struct timespec exprLegacyCtxBldTime; + struct timespec exprLegacyEvalTime; + struct timespec exprJexlCtxBldTime; + struct timespec exprJexlEvalTime; struct timespec renderTime; struct timespec reqTime; } TimeStat; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 0ee2fcb23d..0e59f59bb6 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -28,6 +28,7 @@ #include "logMsg/logMsg.h" #include "orionTypes/OrionValueType.h" +#include "common/statistics.h" // Interface to use libcjexl.a extern "C" { @@ -66,6 +67,8 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st if (exprContextObjectP->isLegacy()) { // std::map based evaluation. Only pure replacement is supported + + TIME_EXPR_LEGACY_EVAL_START(); LM_T(LmtExpr, ("evaluating legacy expression: <%s>", _expression.c_str())); std::map* replacementsP = exprContextObjectP->getMap(); @@ -77,10 +80,13 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st r.stringValue = iter->second; LM_T(LmtExpr, ("legacy evaluation result: <%s>", r.stringValue.c_str())); } + TIME_EXPR_LEGACY_EVAL_STOP(); } else { // JEXL based evaluation + + TIME_EXPR_JEXL_EVAL_START(); std::string context = exprContextObjectP->getJexlContext(); LM_T(LmtExpr, ("evaluating JEXL expression <%s> with context <%s>", _expression.c_str(), context.c_str())); const char* result = cjexl_eval(jexlEngine, _expression.c_str(), context.c_str()); @@ -89,6 +95,7 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st // The ExprResult::fill() method allocates dynamic memory. So, the callers to evaluate() are supposed to invoke ExprResult::release() // method to free it r.fill(result); + TIME_EXPR_JEXL_EVAL_STOP(); } return r; diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index b13d113b5a..2877049fd7 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -326,6 +326,7 @@ static SenderThreadParams* buildSenderParamsCustom // updated. In legacy model, if the key already exists, the operation is ignored (so previous value is preserved). Taking // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence // over macros comming from macros of the same name we conditionally add them depending the case + TIME_EXPR_CTXBLD_START(); exprContext.add("id", en.id); exprContext.add("type", en.type); if (!legacy) @@ -344,6 +345,7 @@ static SenderThreadParams* buildSenderParamsCustom exprContext.add("servicePath", en.servicePath); exprContext.add("authToken", xauthToken); } + TIME_EXPR_CTXBLD_STOP(); // // 1. Verb/Method diff --git a/src/lib/rest/rest.cpp b/src/lib/rest/rest.cpp index cc35ff57fd..3ddd0a85e7 100644 --- a/src/lib/rest/rest.cpp +++ b/src/lib/rest/rest.cpp @@ -618,6 +618,10 @@ static void requestCompleted clock_addtime(&accTimeStat.mongoWriteWaitTime, &threadLastTimeStat.mongoWriteWaitTime); clock_addtime(&accTimeStat.mongoReadWaitTime, &threadLastTimeStat.mongoReadWaitTime); clock_addtime(&accTimeStat.mongoCommandWaitTime, &threadLastTimeStat.mongoCommandWaitTime); + clock_addtime(&accTimeStat.exprLegacyCtxBldTime, &threadLastTimeStat.exprLegacyCtxBldTime); + clock_addtime(&accTimeStat.exprLegacyEvalTime, &threadLastTimeStat.exprLegacyEvalTime); + clock_addtime(&accTimeStat.exprJexlCtxBldTime, &threadLastTimeStat.exprJexlCtxBldTime); + clock_addtime(&accTimeStat.exprJexlEvalTime, &threadLastTimeStat.exprJexlEvalTime); clock_addtime(&accTimeStat.renderTime, &threadLastTimeStat.renderTime); clock_addtime(&accTimeStat.reqTime, &threadLastTimeStat.reqTime); From 4db6066af441002a20ea0318e222533ce2846234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Apr 2024 16:58:23 +0200 Subject: [PATCH 234/390] FIX PoC to measure times CJEXL --- src/lib/expressions/ExprManager.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 0e59f59bb6..7bd204ccce 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -51,6 +51,23 @@ extern "C" { void ExprManager::init(void) { jexlEngine = cjexl_new_engine(); + LM_T(LmtExpr, ("JEXL engine has been initialized")); + + // Create context + ExprContextObject eco; + eco.add("x", 4.0); + eco.add("y", 11.0); + LM_T(LmtExpr, ("jexl context has been built")); + + // Call evaluate method + ExprResult r = evaluate(&eco, "x + y"); + LM_T(LmtExpr, ("jexl evaluation is done")); + + // Print result + LM_T(LmtExpr, ("jexl evaluation result is obtainted")); + LM_T(LmtExpr, ("jexl result: %f", r.numberValue)); + + cjexl_free_engine(jexlEngine); } From b95dd16e1ec8a4011f5c8c4c3a4b886abc9aa7e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Apr 2024 17:26:48 +0200 Subject: [PATCH 235/390] FIX PoC to compare CJEXL with std::map --- src/lib/expressions/ExprManager.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 7bd204ccce..40e86a69fd 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -54,13 +54,12 @@ void ExprManager::init(void) LM_T(LmtExpr, ("JEXL engine has been initialized")); // Create context - ExprContextObject eco; + ExprContextObject eco(false); eco.add("x", 4.0); - eco.add("y", 11.0); LM_T(LmtExpr, ("jexl context has been built")); // Call evaluate method - ExprResult r = evaluate(&eco, "x + y"); + ExprResult r = evaluate(&eco, "x"); LM_T(LmtExpr, ("jexl evaluation is done")); // Print result From 64ed3693d5b93da572d001c704d913fe315c4277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 26 Apr 2024 17:27:01 +0200 Subject: [PATCH 236/390] Revert last two commits with PoC This reverts commit b95dd16e1ec8a4011f5c8c4c3a4b886abc9aa7e3. --- src/lib/expressions/ExprManager.cpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 40e86a69fd..0e59f59bb6 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -51,22 +51,6 @@ extern "C" { void ExprManager::init(void) { jexlEngine = cjexl_new_engine(); - LM_T(LmtExpr, ("JEXL engine has been initialized")); - - // Create context - ExprContextObject eco(false); - eco.add("x", 4.0); - LM_T(LmtExpr, ("jexl context has been built")); - - // Call evaluate method - ExprResult r = evaluate(&eco, "x"); - LM_T(LmtExpr, ("jexl evaluation is done")); - - // Print result - LM_T(LmtExpr, ("jexl evaluation result is obtainted")); - LM_T(LmtExpr, ("jexl result: %f", r.numberValue)); - - cjexl_free_engine(jexlEngine); } From 8339877b9ab8640b540d39877ef3edc350d785cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 20:43:44 +0000 Subject: [PATCH 237/390] Bump jinja2 from 3.1.3 to 3.1.4 in /doc Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.3 to 3.1.4. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.3...3.1.4) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 4252df910c..b362ac268b 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.2.4 Pygments==2.15.0 Markdown==3.3.4 -jinja2==3.1.3 +jinja2==3.1.4 From 9434b9c95a6a9a644f25fa7b2de41fc5bda01bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 09:01:35 +0200 Subject: [PATCH 238/390] FIX revert exprLang --- CHANGES_NEXT_RELEASE | 1 - doc/manuals/admin/database_model.md | 1 - doc/manuals/orion-api.md | 2 - src/lib/apiTypesV2/HttpInfo.cpp | 6 - src/lib/apiTypesV2/HttpInfo.h | 2 - src/lib/apiTypesV2/MqttInfo.cpp | 6 - src/lib/apiTypesV2/MqttInfo.h | 2 - src/lib/expressions/ExprManager.cpp | 1 + src/lib/jsonParseV2/parseSubscription.cpp | 40 - .../mongoBackend/MongoCommonSubscription.cpp | 7 - src/lib/mongoBackend/dbConstants.h | 1 - src/lib/ngsiNotify/Notifier.cpp | 5 +- .../notification_templates_cache_refresh.test | 3 +- ...n_templates_many_notifications_legacy.test | 449 ----------- ...s_with_decoded_chars_in_notifications.test | 3 +- .../http_info_for_sub_update.test | 6 +- .../custom_url_validity_check.test | 3 +- ...tom_notification_http_json_basic_crud.test | 25 +- ...stom_notification_http_json_constants.test | 4 +- ...m_notification_http_json_replacements.test | 4 +- ...ication_http_json_replacements_inside.test | 4 +- ...n_http_json_replacements_inside_multi.test | 4 +- ...n_http_json_replacements_intermediate.test | 4 +- ...tom_notification_mqtt_json_basic_crud.test | 25 +- ...stom_notification_mqtt_json_constants.test | 4 +- ...m_notification_mqtt_json_replacements.test | 4 +- ...ication_mqtt_json_replacements_inside.test | 4 +- ...n_mqtt_json_replacements_inside_multi.test | 4 +- ...n_mqtt_json_replacements_intermediate.test | 4 +- .../self_notification_multi_hop.test | 9 +- .../self_notification_one_hop.test | 3 +- .../mqtt_custom_subscription_basic_crud.test | 8 +- .../mqtt_custom_subscription_update.test | 12 +- .../3001_mqtt/mqtt_notification_custom.test | 3 +- ...tification_problem_with_custom_header.test | 4 +- .../null_payload_get.test | 6 +- .../covered_custom_notification_legacy.test | 217 ------ .../per_sub_httpTimeout_crud_custom.test | 15 +- ...r_sub_httpTimeout_notification_custom.test | 6 +- .../mqtt_custom_notifications_auth.test | 12 +- ...http_custom_exprlang_field_basic_crud.test | 664 ----------------- .../jexl_expr_attrs_weird_syntax.test | 12 +- .../legacy_expr_attrs_weird_syntax.test | 216 ------ ...mqtt_custom_exprlang_field_basic_crud.test | 699 ------------------ ...m_notification_http_ngsi_attr_no_type.test | 3 +- ...tom_notification_http_ngsi_basic_crud.test | 25 +- ...cation_http_ngsi_basic_crud_compounds.test | 21 +- ...fication_http_ngsi_basic_crud_partial.test | 21 +- ...m_notification_mqtt_ngsi_attr_no_type.test | 3 +- ...tom_notification_mqtt_ngsi_basic_crud.test | 25 +- ...cation_mqtt_ngsi_basic_crud_compounds.test | 21 +- ...fication_mqtt_ngsi_basic_crud_partial.test | 21 +- .../mqttCustom_retain_basic_crud.test | 30 +- .../ngsi_patching_special_attr_types.test | 6 +- 54 files changed, 120 insertions(+), 2570 deletions(-) delete mode 100644 test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test delete mode 100644 test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test delete mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test delete mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test delete mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 9c42cb5f02..8876c3fb18 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,2 @@ -- Add: exprLang field in custom notifications (#4004) - Add: expression context build and evaluation counter in timing section in GET /statistics (#4004) - Fix: lighter operation to get databases list from MongoDB (#4517) \ No newline at end of file diff --git a/doc/manuals/admin/database_model.md b/doc/manuals/admin/database_model.md index 1c3ad517c6..ae438dbef6 100644 --- a/doc/manuals/admin/database_model.md +++ b/doc/manuals/admin/database_model.md @@ -329,7 +329,6 @@ notifications. It is a number between 0 and 1800000. If defined to 0 or omitted, in the Orion API. More detail of this functionality [here](../orion-api.md#ngsi-payload-patching). The value of this field is an object with a `attrs` key which value is a simplified version of `attrs` in [the entities collection](#entities-collection). -- **exprLang**: optional field to store the expression language. Only for custom subscriptions. If omitted `legacy` is assumed. - **lastFailure**: the time (as integer number, meaning seconds) when last notification failure occurred. Not present if the subscription has never failed. - **lastFailureReason**: text describing the cause of the last failure. diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 50493f53c1..5e8e6e9bef 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -3947,7 +3947,6 @@ A `httpCustom` object contains the following subfields. | `payload` | ✓ | string | Text-based payload to be used in notifications. In case of empty string or omitted, the default payload (see [Notification Messages](#notification-messages) sections) is used. If `null`, notification will not include any payload. | | `json` | ✓ | object | JSON-based payload to be used in notifications. See [JSON Payloads](#json-payloads) section for more details. | | `ngsi` | ✓ | object | NGSI patching for payload to be used in notifications. See [NGSI payload patching](#ngsi-payload-patching) section for more details. | -| `exprLang` | ✓ | string | Expression language to use, either `legacy` or `jexl` (default is `jexl`) | | `timeout` | ✓ | number | Maximum time (in milliseconds) the subscription waits for the response. The maximum value allowed for this parameter is 1800000 (30 minutes). If `timeout` is defined to 0 or omitted, then the value passed as `-httpTimeout` CLI parameter is used. See section in the [Command line options](admin/cli.md#command-line-options) for more details. | `payload`, `json` or `ngsi` cannot be used at the same time, they are mutually exclusive. @@ -3969,7 +3968,6 @@ A `mqttCustom` object contains the following subfields. | `payload` | ✓ | string | Text-based payload to be used in notifications. In case of empty string or omitted, the default payload (see [Notification Messages](#notification-messages) sections) is used. If `null`, notification will not include any payload. | | `json` | ✓ | object | JSON-based payload to be used in notifications. See [JSON Payloads](#json-payloads) section for more details. | | `ngsi` | ✓ | object | NGSI patching for payload to be used in notifications. See [NGSI payload patching](#ngsi-payload-patching) section for more details. | -| `exprLang` | ✓ | string | Expression language to use, either `legacy` or `jexl` (default is `jexl`) | `payload`, `json` or `ngsi` cannot be used at the same time, they are mutually exclusive. diff --git a/src/lib/apiTypesV2/HttpInfo.cpp b/src/lib/apiTypesV2/HttpInfo.cpp index cc4c000283..d4eb24114c 100644 --- a/src/lib/apiTypesV2/HttpInfo.cpp +++ b/src/lib/apiTypesV2/HttpInfo.cpp @@ -106,8 +106,6 @@ std::string HttpInfo::toJson() { jh.addRaw("headers", objectToJson(headers)); } - - jh.addString("exprLang", this->exprLang); } return jh.str(); @@ -234,9 +232,6 @@ void HttpInfo::fill(const orion::BSONObj& bo) orion::BSONObj headers = getObjectFieldF(bo, CSUB_HEADERS); headers.toStringMap(&this->headers); } - - // expression language used in custom notifications - this->exprLang = bo.hasField(CSUB_EXPRLANG) ? getStringFieldF(bo, CSUB_EXPRLANG) : "legacy"; } } @@ -257,7 +252,6 @@ void HttpInfo::fill(const HttpInfo& _httpInfo) this->custom = _httpInfo.custom; this->includePayload = _httpInfo.includePayload; this->timeout = _httpInfo.timeout; - this->exprLang = _httpInfo.exprLang; this->json = _httpInfo.json == NULL? NULL : _httpInfo.json->clone(); diff --git a/src/lib/apiTypesV2/HttpInfo.h b/src/lib/apiTypesV2/HttpInfo.h index 8d32651374..0af3d273ef 100644 --- a/src/lib/apiTypesV2/HttpInfo.h +++ b/src/lib/apiTypesV2/HttpInfo.h @@ -57,8 +57,6 @@ struct HttpInfo bool includePayload; long long timeout; - std::string exprLang; - HttpInfo(); std::string toJson(); diff --git a/src/lib/apiTypesV2/MqttInfo.cpp b/src/lib/apiTypesV2/MqttInfo.cpp index c8829b0bb2..e604f21e1a 100644 --- a/src/lib/apiTypesV2/MqttInfo.cpp +++ b/src/lib/apiTypesV2/MqttInfo.cpp @@ -102,8 +102,6 @@ std::string MqttInfo::toJson() jh.addRaw("ngsi", this->ngsi.toJson(NGSI_V2_NORMALIZED, true)); break; } - - jh.addString("exprLang", this->exprLang); } return jh.str(); @@ -218,9 +216,6 @@ void MqttInfo::fill(const orion::BSONObj& bo) this->ngsi.attributeVector.fill(getObjectFieldF(ngsiObj, ENT_ATTRS)); } } - - // expression language used in custom notifications - this->exprLang = bo.hasField(CSUB_EXPRLANG) ? getStringFieldF(bo, CSUB_EXPRLANG) : "legacy"; } } @@ -243,7 +238,6 @@ void MqttInfo::fill(const MqttInfo& _mqttInfo) this->providedAuth = _mqttInfo.providedAuth; this->user = _mqttInfo.user; this->passwd = _mqttInfo.passwd; - this->exprLang = _mqttInfo.exprLang; this->json = _mqttInfo.json == NULL ? NULL : _mqttInfo.json->clone(); diff --git a/src/lib/apiTypesV2/MqttInfo.h b/src/lib/apiTypesV2/MqttInfo.h index bbc3b9260d..738e73c6a9 100644 --- a/src/lib/apiTypesV2/MqttInfo.h +++ b/src/lib/apiTypesV2/MqttInfo.h @@ -53,8 +53,6 @@ struct MqttInfo CustomPayloadType payloadType; bool includePayload; - std::string exprLang; - bool providedAuth; std::string user; std::string passwd; diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 0e59f59bb6..f54f91ef7c 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -55,6 +55,7 @@ void ExprManager::init(void) + /* **************************************************************************** * * ExprManager::evaluate - diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index 5fbe2faa38..593161b202 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -1023,26 +1023,6 @@ static std::string parseNotification(ConnectionInfo* ciP, SubscriptionUpdate* su return r; } - // exprLang - Opt exprLangOpt = getStringOpt(httpCustom, "exprLang", "exprLang httpCustom notification"); - std::string exprLang = "jexl"; - - if (!exprLangOpt.ok()) - { - return badInput(ciP, exprLangOpt.error); - } - - if (exprLangOpt.given) - { - exprLang = exprLangOpt.value; - if ((exprLang != "jexl") && (exprLang != "legacy")) - { - return badInput(ciP, "not valid exprLang, valid ones are jexl or legacy"); - } - } - - subsP->notification.httpInfo.exprLang = exprLang; - subsP->notification.httpInfo.custom = true; } else if (notification.HasMember("mqtt")) @@ -1153,26 +1133,6 @@ static std::string parseNotification(ConnectionInfo* ciP, SubscriptionUpdate* su return r; } - // exprLang - Opt exprLangOpt = getStringOpt(mqttCustom, "exprLang", "exprLang mqttCustom notification"); - std::string exprLang = "jexl"; - - if (!exprLangOpt.ok()) - { - return badInput(ciP, exprLangOpt.error); - } - - if (exprLangOpt.given) - { - exprLang = exprLangOpt.value; - if ((exprLang != "jexl") && (exprLang != "legacy")) - { - return badInput(ciP, "not valid exprLang, valid ones are jexl or legacy"); - } - } - - subsP->notification.mqttInfo.exprLang = exprLang; - subsP->notification.mqttInfo.custom = true; } diff --git a/src/lib/mongoBackend/MongoCommonSubscription.cpp b/src/lib/mongoBackend/MongoCommonSubscription.cpp index a58ea5f55b..dd6bcf3101 100644 --- a/src/lib/mongoBackend/MongoCommonSubscription.cpp +++ b/src/lib/mongoBackend/MongoCommonSubscription.cpp @@ -183,9 +183,6 @@ static void setCustomHttpInfo(const HttpInfo& httpInfo, orion::BSONObjBuilder* b b->append(CSUB_NGSI, bob.obj()); } - - b->append(CSUB_EXPRLANG, httpInfo.exprLang); - LM_T(LmtMongo, ("Subscription exprLang: %s", httpInfo.exprLang.c_str())); } @@ -258,9 +255,6 @@ static void setCustomMqttInfo(const ngsiv2::MqttInfo& mqttInfo, orion::BSONObjBu b->append(CSUB_NGSI, bob.obj()); } - - b->append(CSUB_EXPRLANG, mqttInfo.exprLang); - LM_T(LmtMongo, ("Subscription exprLang: %s", mqttInfo.exprLang.c_str())); } @@ -632,7 +626,6 @@ void setOnlyChanged(const Subscription& sub, orion::BSONObjBuilder* b) } - /* **************************************************************************** * * setCovered - diff --git a/src/lib/mongoBackend/dbConstants.h b/src/lib/mongoBackend/dbConstants.h index 241507efd9..b55cf29925 100644 --- a/src/lib/mongoBackend/dbConstants.h +++ b/src/lib/mongoBackend/dbConstants.h @@ -119,7 +119,6 @@ #define CSUB_NGSI "ngsi" #define CSUB_BLACKLIST "blacklist" #define CSUB_ONLYCHANGED "onlyChanged" -#define CSUB_EXPRLANG "exprLang" #define CSUB_COVERED "covered" #define CSUB_NOTIFYONMETADATACHANGE "notifyOnMetadataChange" #define CSUB_LASTFAILURE "lastFailure" diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 2877049fd7..bd8c66738d 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -315,10 +315,9 @@ static SenderThreadParams* buildSenderParamsCustom std::map headers; Entity& en = notifyCerP->entity; - const std::string exprLang = notification.type == ngsiv2::HttpNotification ? notification.httpInfo.exprLang : notification.mqttInfo.exprLang; - // Used by several macroSubstitute() calls along this function - bool legacy = (exprLang == "legacy"); + // FIXME PR: unhardwire legacy == false + bool legacy = false; ExprContextObject exprContext(legacy); // FIXME PR: no-legacy context is now based in JsonHelper, which may mess with key repetition. Check this diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test index f1b0c00cbb..853d3e6abd 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_cache_refresh.test @@ -157,7 +157,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 713 +Content-Length: 695 [ { @@ -167,7 +167,6 @@ Content-Length: 713 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "A1": "${A1}", "A2": "${A2}", diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test deleted file mode 100644 index d624115962..0000000000 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications_legacy.test +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Notification Templates - many notifications (legacy expressions - ---SHELL-INIT-- -dbInit CB -brokerStart CB -accumulatorStart --pretty-print - ---SHELL-- - -# -# 01. Create regular subscription for E1 -# 02. Create same subscription but with 'httpCustom' instead of 'http' -# 03. Create custom subscription for E1 -# 04. Create E1/T1 -# 05. Create E1/T2 -# 06. Create E1/T3 -# 07. Dump accumulator and see 9 notifications -# - -echo "01. Create regular subscription for E1" -echo "======================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1" - } - ], - "condition": { - "attrs": [] - } - }, - "notification": { - "http": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" - } - }, - "throttling": 0 -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create same subscription but with 'httpCustom' instead of 'http'" -echo "====================================================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1" - } - ], - "condition": { - "attrs": [] - } - }, - "notification": { - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" - }, - "exprLang": "legacy" - }, - "throttling": 0 -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "03. Create custom subscription for E1" -echo "=====================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1" - } - ], - "condition": { - "attrs": [] - } - }, - "notification": { - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "method": "PUT", - "payload": "{ %22A1%22: %22${A1}%22, %22A2%22: %22${A2}%22, %22A3%22: %22${A3}%22 }", - "qs": { "id": "${id}", "type": "${type}", "a1": "${A1}", "a2": "${A2}", "a3": "${A3}" }, - "headers": { "entity-id": "${id}", "entity-type": "${type}", "A1": "${A1}", "A2": "${A2}", "A3": "${A3}" }, - "exprLang": "legacy" - } - }, - "throttling": 0 -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "04. Create E1/T1" -echo "================" -payload='{ - "id": "E1", - "type": "T1", - "A1": true -}' -orionCurl --url /v2/entities?options=keyValues --payload "$payload" -echo -echo - - -echo "05. Create E1/T2" -echo "================" -payload='{ - "id": "E1", - "type": "T2", - "A2": null -}' -orionCurl --url /v2/entities?options=keyValues --payload "$payload" -echo -echo - - -echo "06. Create E1/T3" -echo "================" -payload='{ - "id": "E1", - "type": "T3", - "A3": 3 -}' -orionCurl --url /v2/entities?options=keyValues --payload "$payload" -echo -echo - - -echo "07. Dump accumulator and see 9 notifications" -echo "============================================" -sleep 0.5 # Without this sleep, sometimes the last notification is missing -accumulatorDump -echo -echo - - ---REGEXPECT-- -01. Create regular subscription for E1 -====================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -02. Create same subscription but with 'httpCustom' instead of 'http' -==================================================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -03. Create custom subscription for E1 -===================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -04. Create E1/T1 -================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T1 -Content-Length: 0 - - - -05. Create E1/T2 -================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T2 -Content-Length: 0 - - - -06. Create E1/T3 -================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T3 -Content-Length: 0 - - - -07. Dump accumulator and see 9 notifications -#SORT_START -============================================ -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 129 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A1": { - "metadata": {}, - "type": "Boolean", - "value": true - }, - "id": "E1", - "type": "T1" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 129 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A1": { - "metadata": {}, - "type": "Boolean", - "value": true - }, - "id": "E1", - "type": "T1" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 -Fiware-Servicepath: / -Entity-Id: E1 -Content-Length: 36 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -A1: true -Entity-Type: T1 -Content-Type: text/plain; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "A1": "true", - "A2": "", - "A3": "" -} -#SORT_END -#SORT_START -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 126 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A2": { - "metadata": {}, - "type": "None", - "value": null - }, - "id": "E1", - "type": "T2" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 126 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A2": { - "metadata": {}, - "type": "None", - "value": null - }, - "id": "E1", - "type": "T2" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a2=null&id=E1&type=T2 -Fiware-Servicepath: / -Entity-Id: E1 -Content-Length: 36 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -A2: null -Entity-Type: T2 -Content-Type: text/plain; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "A1": "", - "A2": "null", - "A3": "" -} -#SORT_END -#SORT_START -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 125 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A3": { - "metadata": {}, - "type": "Number", - "value": 3 - }, - "id": "E1", - "type": "T3" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 125 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "data": [ - { - "A3": { - "metadata": {}, - "type": "Number", - "value": 3 - }, - "id": "E1", - "type": "T3" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 -Fiware-Servicepath: / -A3: 3 -Entity-Id: E1 -Content-Length: 33 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Entity-Type: T3 -Content-Type: text/plain; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) - -{ - "A1": "", - "A2": "", - "A3": "3" -} -======================================= -#SORT_END - - ---TEARDOWN-- -brokerStop CB -dbDrop CB -accumulatorStop diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test index a31092de4a..9a548cec40 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_with_decoded_chars_in_notifications.test @@ -137,7 +137,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 568 +Content-Length: 550 { "id": "REGEX([0-9a-f]{24})", @@ -146,7 +146,6 @@ Content-Length: 568 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "hs1": "1", "hs2": "${A1}" diff --git a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test index 2bdcc9c72c..7a4956a172 100644 --- a/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test +++ b/test/functionalTest/cases/2203_http_info_for_sub_update/http_info_for_sub_update.test @@ -173,7 +173,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 513 +Content-Length: 495 [ { @@ -183,7 +183,6 @@ Content-Length: 513 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-Type": "application/json", "H1": "${A1}" @@ -232,7 +231,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 515 +Content-Length: 497 [ { @@ -242,7 +241,6 @@ Content-Length: 515 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-Type": "application/json", "H2": "${A2}" diff --git a/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test b/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test index 5969637f5d..57a7a22c99 100644 --- a/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test +++ b/test/functionalTest/cases/2280_custom_url_validity_check/custom_url_validity_check.test @@ -142,7 +142,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 304 +Content-Length: 286 [ { @@ -152,7 +152,6 @@ Content-Length: 304 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "method": "PUT", "url": "${abc}" }, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test index 89828afff7..c50d612856 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_basic_crud.test @@ -254,7 +254,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 343 +Content-Length: 325 { "id": "REGEX([0-9a-f]{24})", @@ -263,7 +263,6 @@ Content-Length: 343 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "x1": 1, "x2": 2 @@ -302,7 +301,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -311,7 +310,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -347,7 +345,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 339 +Content-Length: 321 { "id": "REGEX([0-9a-f]{24})", @@ -356,7 +354,6 @@ Content-Length: 339 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "yy", "zz" @@ -395,7 +392,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 335 +Content-Length: 317 { "id": "REGEX([0-9a-f]{24})", @@ -404,7 +401,6 @@ Content-Length: 335 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ 1, 2, @@ -444,7 +440,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -453,7 +449,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -491,7 +486,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -500,7 +495,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -536,7 +530,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 352 +Content-Length: 334 { "id": "REGEX([0-9a-f]{24})", @@ -545,7 +539,6 @@ Content-Length: 352 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "z4": true, "z5": [ @@ -579,7 +572,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 691 +Content-Length: 655 [ { @@ -589,7 +582,6 @@ Content-Length: 691 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -616,7 +608,6 @@ Content-Length: 691 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "z4": true, "z5": [ diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test index d49ac8a108..2bb34d3d32 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_constants.test @@ -180,7 +180,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 916 +Content-Length: 880 [ { @@ -190,7 +190,6 @@ Content-Length: 916 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "array": [ "22", @@ -246,7 +245,6 @@ Content-Length: 916 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "foo", 10, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test index e0a2d267ec..08f26a3682 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 838 +Content-Length: 802 [ { @@ -197,7 +197,6 @@ Content-Length: 838 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "array": "${array}", "bool": "${bool}", @@ -231,7 +230,6 @@ Content-Length: 838 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "${text}", "${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test index e1b86b3f9f..758775b4e3 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 851 +Content-Length: 815 [ { @@ -197,7 +197,6 @@ Content-Length: 851 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "A": "ns:${text}", "B": "ns:${number}", @@ -231,7 +230,6 @@ Content-Length: 851 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "ns:${text}", "ns:${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test index 191994521c..95f7928e31 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_inside_multi.test @@ -187,7 +187,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 957 +Content-Length: 921 [ { @@ -197,7 +197,6 @@ Content-Length: 957 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "A": "ns:${text}:${object}", "B": "ns:${number}:${array}", @@ -231,7 +230,6 @@ Content-Length: 957 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "ns:${text}:${object}", "ns:${number}:${array}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test index 568d36fc19..7a19421ab8 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_http_json_replacements_intermediate.test @@ -157,7 +157,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 751 +Content-Length: 715 [ { @@ -167,7 +167,6 @@ Content-Length: 751 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": { "x": { "x1": "${A}", @@ -203,7 +202,6 @@ Content-Length: 751 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "json": [ "22", { diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test index dccebf8230..0c24073d89 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_basic_crud.test @@ -261,7 +261,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 376 +Content-Length: 358 { "id": "REGEX([0-9a-f]{24})", @@ -270,7 +270,6 @@ Content-Length: 376 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "x1": 1, "x2": 2 @@ -312,7 +311,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -321,7 +320,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -360,7 +358,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 372 +Content-Length: 354 { "id": "REGEX([0-9a-f]{24})", @@ -369,7 +367,6 @@ Content-Length: 372 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "yy", "zz" @@ -411,7 +408,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 368 +Content-Length: 350 { "id": "REGEX([0-9a-f]{24})", @@ -420,7 +417,6 @@ Content-Length: 368 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ 1, 2, @@ -463,7 +459,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -472,7 +468,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -513,7 +508,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -522,7 +517,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -561,7 +555,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 385 +Content-Length: 367 { "id": "REGEX([0-9a-f]{24})", @@ -570,7 +564,6 @@ Content-Length: 385 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "z4": true, "z5": [ @@ -607,7 +600,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 757 +Content-Length: 721 [ { @@ -617,7 +610,6 @@ Content-Length: 757 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -647,7 +639,6 @@ Content-Length: 757 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "z4": true, "z5": [ diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test index e84b665130..9428ed0e64 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_constants.test @@ -186,7 +186,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 982 +Content-Length: 946 [ { @@ -196,7 +196,6 @@ Content-Length: 982 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "array": [ "22", @@ -255,7 +254,6 @@ Content-Length: 982 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "foo", 10, diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test index 993180a8ce..954ed3bdba 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements.test @@ -193,7 +193,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 904 +Content-Length: 868 [ { @@ -203,7 +203,6 @@ Content-Length: 904 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "array": "${array}", "bool": "${bool}", @@ -240,7 +239,6 @@ Content-Length: 904 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "${text}", "${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test index 7133b2e675..3a6f4a15c5 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside.test @@ -192,7 +192,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 917 +Content-Length: 881 [ { @@ -202,7 +202,6 @@ Content-Length: 917 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "A": "ns:${text}", "B": "ns:${number}", @@ -239,7 +238,6 @@ Content-Length: 917 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "ns:${text}", "ns:${number}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test index 58c287e5d5..940492dc04 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_inside_multi.test @@ -193,7 +193,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1023 +Content-Length: 987 [ { @@ -203,7 +203,6 @@ Content-Length: 1023 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "A": "ns:${text}:${object}", "B": "ns:${number}:${array}", @@ -240,7 +239,6 @@ Content-Length: 1023 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "ns:${text}:${object}", "ns:${number}:${array}", diff --git a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test index 2fc44f8b6b..020245e27b 100644 --- a/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test +++ b/test/functionalTest/cases/2560_custom_notification_json/custom_notification_mqtt_json_replacements_intermediate.test @@ -163,7 +163,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 817 +Content-Length: 781 [ { @@ -173,7 +173,6 @@ Content-Length: 817 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": { "x": { "x1": "${A}", @@ -212,7 +211,6 @@ Content-Length: 817 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "json": [ "22", { diff --git a/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test b/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test index 6fa972f30b..2fe079ec78 100644 --- a/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test +++ b/test/functionalTest/cases/2937_self_notification_protection/self_notification_multi_hop.test @@ -373,7 +373,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 582 +Content-Length: 564 { "id": "REGEX([0-9a-f]{24})", @@ -382,7 +382,6 @@ Content-Length: 582 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-type": "application/json" }, @@ -416,7 +415,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 582 +Content-Length: 564 { "id": "REGEX([0-9a-f]{24})", @@ -425,7 +424,6 @@ Content-Length: 582 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-type": "application/json" }, @@ -459,7 +457,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 582 +Content-Length: 564 { "id": "REGEX([0-9a-f]{24})", @@ -468,7 +466,6 @@ Content-Length: 582 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-type": "application/json" }, diff --git a/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test b/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test index 12727d8b09..ffaffdb044 100644 --- a/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test +++ b/test/functionalTest/cases/2937_self_notification_protection/self_notification_one_hop.test @@ -128,7 +128,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 584 +Content-Length: 566 [ { @@ -138,7 +138,6 @@ Content-Length: 584 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "headers": { "Content-type": "application/json" }, diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test index 36ca87149f..d482101617 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_basic_crud.test @@ -282,7 +282,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 2536 +Content-Length: 2428 [ { @@ -295,7 +295,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "temperature:${temperature}", "qos": 0, "retain": false, @@ -328,7 +327,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "temperature:${temperature}", "qos": 2, "retain": false, @@ -361,7 +359,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "/orion/${dest}", @@ -393,7 +390,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 2, "retain": false, "topic": "/orion/${dest}", @@ -425,7 +421,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": null, "qos": 0, "retain": false, @@ -458,7 +453,6 @@ Content-Length: 2536 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": null, "qos": 2, "retain": false, diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test index 1d65654747..531e251861 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_custom_subscription_update.test @@ -204,7 +204,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 407 +Content-Length: 389 { "description": "Original sub", @@ -214,7 +214,6 @@ Content-Length: 407 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "t:${t}", "qos": 2, "retain": false, @@ -253,7 +252,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 407 +Content-Length: 389 { "description": "Modified sub", @@ -263,7 +262,6 @@ Content-Length: 407 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "t:${t}", "qos": 2, "retain": false, @@ -302,7 +300,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 400 +Content-Length: 382 { "description": "Modified sub", @@ -312,7 +310,6 @@ Content-Length: 400 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": null, "qos": 0, "retain": false, @@ -395,7 +392,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 390 +Content-Length: 372 { "description": "Modified sub", @@ -405,7 +402,6 @@ Content-Length: 390 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 1, "retain": false, "topic": "/orionbk/${d}", diff --git a/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test b/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test index 597cfa5457..ecc29badba 100644 --- a/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test +++ b/test/functionalTest/cases/3001_mqtt/mqtt_notification_custom.test @@ -245,7 +245,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 527 +Content-Length: 509 [ { @@ -258,7 +258,6 @@ Content-Length: 527 "lastNotification": "REGEX(.*)", "lastSuccess": "REGEX(.*)", "mqttCustom": { - "exprLang": "jexl", "payload": "{ %22A%22: %22${A}%22 }", "qos": 0, "retain": false, diff --git a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test b/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test index 9e63211726..b4e9cc5695 100644 --- a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test +++ b/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test @@ -159,7 +159,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1048 +Content-Length: 1012 [ { @@ -171,7 +171,6 @@ Content-Length: 1048 "attrsFormat": "legacy", "covered": false, "httpCustom": { - "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "lastNotification": "REGEX(.*)", @@ -202,7 +201,6 @@ Content-Length: 1048 "attrsFormat": "legacy", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "{ %22A1%22: %22Value of A1: ${A1}%22 }", "url": "http://localhost:9997/notify" }, diff --git a/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test b/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test index 13e3163639..0955bd6c86 100644 --- a/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test +++ b/test/functionalTest/cases/3272_custom_notification_sending_0-length_request/null_payload_get.test @@ -189,7 +189,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1272 +Content-Length: 1200 [ { @@ -199,7 +199,6 @@ Content-Length: 1272 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "abc", "url": "http://localhost:9997/notify" }, @@ -225,7 +224,6 @@ Content-Length: 1272 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "onlyChangedAttrs": false @@ -250,7 +248,6 @@ Content-Length: 1272 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": null, "url": "http://localhost:9997/notify" }, @@ -276,7 +273,6 @@ Content-Length: 1272 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "url": "http://localhost:9997/notify" }, "onlyChangedAttrs": false diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test deleted file mode 100644 index 8321f78322..0000000000 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification_legacy.test +++ /dev/null @@ -1,217 +0,0 @@ -# Copyright 2022 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Covered custom notifictions (legacy expressions) - ---SHELL-INIT-- -dbInit CB -brokerStart CB -accumulatorStart - ---SHELL-- - -# -# 01. Create covered custom subscriptions for E1 covering attributes A1 and A2 -# 02. Create not covered custom subscriptions for E2 -# 03. Create E1 with attribute A1=1 -# 04. Dump & reset, see notifications A1=1 and A2=null in custom payload -# 05. Create E2 with attribute A1=1 -# 06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) -# - -echo "01. Create covered custom subscriptions for E1 covering attributes A1 and A2" -echo "============================================================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1", - "type": "T" - } - ] - }, - "notification": { - "covered": true, - "attrs": [ "A1", "A2" ], - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "payload": "{ %22A1-value%22: ${A1}, %22A2-value%22: ${A2} }", - "headers": { - "content-type": "application/json%3B charset%3Dutf-8" - }, - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create not covered custom subscriptions for E2" -echo "==================================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E2", - "type": "T" - } - ] - }, - "notification": { - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "payload": "{ %22A1-value%22: ${A1}, %22A2-value%22: ${A2} }", - "headers": { - "content-type": "application/json%3B charset%3Dutf-8" - } - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "03. Create E1 with attribute A1=1" -echo "=================================" -payload='{ - "id": "E1", - "type": "T", - "A1": { - "value": 1, - "type": "Number" - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "04. Dump & reset, see notifications A1=1 and A2=null in custom payload" -echo "======================================================================" -accumulatorDump -accumulatorReset -echo -echo - - -echo "05. Create E2 with attribute A1=1" -echo "=================================" -payload='{ - "id": "E2", - "type": "T", - "A1": { - "value": 1, - "type": "Number" - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON)" -echo "========================================================================================" -accumulatorDump -accumulatorReset -echo -echo - - ---REGEXPECT-- -01. Create covered custom subscriptions for E1 covering attributes A1 and A2 -============================================================================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -02. Create not covered custom subscriptions for E2 -================================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -03. Create E1 with attribute A1=1 -================================= -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T -Content-Length: 0 - - - -04. Dump & reset, see notifications A1=1 and A2=null in custom payload -====================================================================== -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 35 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ "A1-value": 1, "A2-value": null }======================================= - - -05. Create E2 with attribute A1=1 -================================= -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E2?type=T -Content-Length: 0 - - - -06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) -======================================================================================== -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 31 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ "A1-value": 1, "A2-value": }======================================= - - ---TEARDOWN-- -brokerStop CB -dbDrop CB -accumulatorStop diff --git a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test index badbe98993..d874dbefdd 100644 --- a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test +++ b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_crud_custom.test @@ -327,7 +327,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 379 +Content-Length: 361 { "description": "HTTP sub", @@ -337,7 +337,6 @@ Content-Length: 379 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 1000, "url": "http://localhost:1234" @@ -374,7 +373,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 379 +Content-Length: 361 { "description": "HTTP sub", @@ -384,7 +383,6 @@ Content-Length: 379 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 2000, "url": "http://localhost:1234" @@ -421,7 +419,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 364 +Content-Length: 346 { "description": "HTTP sub", @@ -431,7 +429,6 @@ Content-Length: 364 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "url": "http://localhost:1234" }, @@ -467,7 +464,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 379 +Content-Length: 361 { "description": "HTTP sub", @@ -477,7 +474,6 @@ Content-Length: 379 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 1000, "url": "http://localhost:1234" @@ -524,7 +520,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 364 +Content-Length: 346 { "description": "HTTP sub", @@ -534,7 +530,6 @@ Content-Length: 364 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "url": "http://localhost:1234" }, diff --git a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test index bbdcdaebb0..0937cb5cec 100644 --- a/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test +++ b/test/functionalTest/cases/3842_per_sub_httpTimeout/per_sub_httpTimeout_notification_custom.test @@ -175,7 +175,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 514 +Content-Length: 496 { "description": "HTTP sub", @@ -185,7 +185,6 @@ Content-Length: 514 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 12000, "url": "http://localhost:9997/noresponse" @@ -218,7 +217,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 550 +Content-Length: 532 { "description": "HTTP sub", @@ -229,7 +228,6 @@ Content-Length: 550 "covered": false, "failsCounter": 1, "httpCustom": { - "exprLang": "jexl", "payload": "Custom payload", "timeout": 8000, "url": "http://localhost:9997/noresponse" diff --git a/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test b/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test index c50332dc8f..59a65b45f0 100644 --- a/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test +++ b/test/functionalTest/cases/3914_mqtt_notifications_auth/mqtt_custom_notifications_auth.test @@ -317,7 +317,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 384 +Content-Length: 366 { "id": "REGEX([0-9a-f\-]{24})", @@ -326,7 +326,6 @@ Content-Length: 384 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, @@ -372,7 +371,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 385 +Content-Length: 367 { "id": "REGEX([0-9a-f\-]{24})", @@ -381,7 +380,6 @@ Content-Length: 385 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, @@ -427,7 +425,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 352 +Content-Length: 334 { "id": "REGEX([0-9a-f\-]{24})", @@ -436,7 +434,6 @@ Content-Length: 352 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "/orion", @@ -480,7 +477,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 384 +Content-Length: 366 { "id": "REGEX([0-9a-f\-]{24})", @@ -489,7 +486,6 @@ Content-Length: 384 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "passwd": "*****", "qos": 0, "retain": false, diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test deleted file mode 100644 index 656e55da9b..0000000000 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/http_custom_exprlang_field_basic_crud.test +++ /dev/null @@ -1,664 +0,0 @@ -# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Basic CRUD for exprLang field (http variant) - ---SHELL-INIT-- -dbInit CB -brokerStart CB 0-255 - ---SHELL-- - -# -# 01. Create subscription with invalid exprLang, see error -# 02. Create subscription with exprLang jexl -# 03a. Get subscription and see exprLang jexl -# 03b. Get subscription (as a list) and see exprLang jexl -# 04. Update subscription with exprLang legacy -# 05a. Get subscription and see exprLang legacy -# 05b. Get subscription (as a list) and see exprLang legacy -# 06. Update subscription without exprLang -# 07a. Get subscription and see exprLang jexl -# 07b. Get subscription (as a list) and see exprLang jexl -# 08. Update subscription with exprLang legacy -# 09a. Get subscription and see exprLang legacy -# 09b. Get subscription (as a list) and see exprLang legacy -# 10. Update subscription with invalid exprLang, see error -# 11a. Get subscription and see exprLang legacy -# 11b. Get subscription (as a list) and see exprLang legacy -# - -echo "01. Create subscription with invalid exprLang, see error" -echo "========================================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern" : ".*", - "type": "T" - } - ] - }, - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": "foo" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create subscription with exprLang jexl" -echo "==========================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern" : ".*", - "type": "T" - } - ] - }, - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": "jexl" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") - - -echo "03a. Get subscription and see exprLang jexl" -echo "===========================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "03b. Get subscription (as a list) and see exprLang jexl" -echo "=======================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "04. Update subscription with exprLang legacy" -echo "============================================" -payload='{ - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "05a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "05b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "06. Update subscription without exprLang" -echo "========================================" -payload='{ - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "topic": "orion" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "07a. Get subscription and see exprLang jexl" -echo "===========================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "07b. Get subscription (as a list) and see exprLang jexl" -echo "=======================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "08. Update subscription with exprLang legacy" -echo "============================================" -payload='{ - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "09a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "09b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "10. Update subscription with invalid exprLang, see error" -echo "========================================================" -payload='{ - "notification": { - "httpCustom": { - "url": "http://localhost:9998", - "exprLang": 42 - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "11a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "11b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - ---REGEXPECT-- -01. Create subscription with invalid exprLang, see error -======================================================== -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 88 - -{ - "description": "not valid exprLang, valid ones are jexl or legacy", - "error": "BadRequest" -} - - -02. Create subscription with exprLang jexl -========================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -03a. Get subscription and see exprLang jexl -=========================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 320 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "jexl", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -03b. Get subscription (as a list) and see exprLang jexl -======================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "jexl", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -04. Update subscription with exprLang legacy -============================================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -05a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -05b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 324 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -06. Update subscription without exprLang -======================================== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -07a. Get subscription and see exprLang jexl -=========================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 320 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "jexl", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -07b. Get subscription (as a list) and see exprLang jexl -======================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "jexl", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -08. Update subscription with exprLang legacy -============================================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -09a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -09b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 324 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -10. Update subscription with invalid exprLang, see error -======================================================== -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 87 - -{ - "description": "exprLang httpCustom notification is not a string", - "error": "BadRequest" -} - - -11a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 322 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -11b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 324 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "httpCustom": { - "exprLang": "legacy", - "url": "http://localhost:9998" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test index ac53119941..79ecfd9eeb 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -31,17 +31,15 @@ accumulatorStart --pretty-print --SHELL-- # -# Same as legacy_expr_attrs_weird_syntax.test, but using jexl expression language -# -# 01. 01. Create custom sub with custom expression: R1=A-B and R2=A:B +# 01. Create custom sub with custom expression: A-B and A:B # 02. Create entity E1 with A-B 1 and A:B 2 # 03. Update entity E1 with A-B 3 and A:B 4 # 04. Dump accumulator and see notifications (R1 null, R2 null and R1 null, R2 null) # -echo "01. Create custom sub with custom expression: R1=A-B and R2=A:B" -echo "===============================================================" +echo "01. Create custom sub with custom expression: A-B and A:B" +echo "=========================================================" payload='{ "subject": { "entities": [ @@ -117,8 +115,8 @@ echo --REGEXPECT-- -01. Create custom sub with custom expression: R1=A-B and R2=A:B -=============================================================== +01. Create custom sub with custom expression: A-B and A:B +========================================================= HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test deleted file mode 100644 index 15da6fc5bb..0000000000 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/legacy_expr_attrs_weird_syntax.test +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Legacy expression using attributes with weird syntax - ---SHELL-INIT-- -dbInit CB -brokerStart CB -accumulatorStart --pretty-print - ---SHELL-- - -# -# Same as jexl_expr_attrs_weird_syntax.test, but using legacy expression language -# -# 01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B -# 02. Create entity E1 with A-B 1 and A:B 2 -# 03. Update entity E1 with A-B 3 and A:B 4 -# 04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4) -# - - -echo "01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B" -echo "======================================================================" -payload='{ - "subject": { - "entities": [ - { - "id" : "E1", - "type": "T" - } - ] - }, - "notification": { - "httpCustom": { - "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", - "exprLang": "legacy", - "ngsi": { - "R1": { - "value": "${A-B}", - "type": "Calculated" - }, - "R2": { - "value": "${A:B}", - "type": "Calculated" - } - } - }, - "attrs": [ "R1", "R2" ] - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create entity E1 with A-B 1 and A:B 2" -echo "=========================================" -payload='{ - "id": "E1", - "type": "T", - "A-B": { - "value": 1, - "type": "Number" - }, - "A:B": { - "value": 2, - "type": "Number" - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "03. Update entity E1 with A-B 3 and A:B 4" -echo "=========================================" -payload='{ - "A-B": { - "value": 3, - "type": "Number" - }, - "A:B": { - "value": 4, - "type": "Number" - } -}' -orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" -echo -echo - - -echo "04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4)" -echo "======================================================================" -accumulatorDump -echo -echo - - ---REGEXPECT-- -01. Create custom sub with custom legacy expression: R1=A-B and R2=A:B -====================================================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -02. Create entity E1 with A-B 1 and A:B 2 -========================================= -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T -Content-Length: 0 - - - -03. Update entity E1 with A-B 3 and A:B 4 -========================================= -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -04. Dump accumulator and see notifications (R1 1, R2 2 and R1 3, R2 4) -====================================================================== -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 179 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "data": [ - { - "R1": { - "metadata": {}, - "type": "Calculated", - "value": 1 - }, - "R2": { - "metadata": {}, - "type": "Calculated", - "value": 2 - }, - "id": "E1", - "type": "T" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://127.0.0.1:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 179 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: normalized -Host: 127.0.0.1:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "data": [ - { - "R1": { - "metadata": {}, - "type": "Calculated", - "value": 3 - }, - "R2": { - "metadata": {}, - "type": "Calculated", - "value": 4 - }, - "id": "E1", - "type": "T" - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - ---TEARDOWN-- -brokerStop CB -dbDrop CB -accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test deleted file mode 100644 index b155766fea..0000000000 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/mqtt_custom_exprlang_field_basic_crud.test +++ /dev/null @@ -1,699 +0,0 @@ -# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Basic CRUD for exprLang field (mqtt variant) - ---SHELL-INIT-- -dbInit CB -brokerStart CB 0-255 - ---SHELL-- - -# -# 01. Create subscription with invalid exprLang, see error -# 02. Create subscription with exprLang jexl -# 03a. Get subscription and see exprLang jexl -# 03b. Get subscription (as a list) and see exprLang jexl -# 04. Update subscription with exprLang legacy -# 05a. Get subscription and see exprLang legacy -# 05b. Get subscription (as a list) and see exprLang legacy -# 06. Update subscription without exprLang -# 07a. Get subscription and see exprLang jexl -# 07b. Get subscription (as a list) and see exprLang jexl -# 08. Update subscription with exprLang legacy -# 09a. Get subscription and see exprLang legacy -# 09b. Get subscription (as a list) and see exprLang legacy -# 10. Update subscription with invalid exprLang, see error -# 11a. Get subscription and see exprLang legacy -# 11b. Get subscription (as a list) and see exprLang legacy -# - -echo "01. Create subscription with invalid exprLang, see error" -echo "========================================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern" : ".*", - "type": "T" - } - ] - }, - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": "foo" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create subscription with exprLang jexl" -echo "==========================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern" : ".*", - "type": "T" - } - ] - }, - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": "jexl" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") - - -echo "03a. Get subscription and see exprLang jexl" -echo "===========================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "03b. Get subscription (as a list) and see exprLang jexl" -echo "=======================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "04. Update subscription with exprLang legacy" -echo "============================================" -payload='{ - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "05a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "05b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "06. Update subscription without exprLang" -echo "========================================" -payload='{ - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "07a. Get subscription and see exprLang jexl" -echo "===========================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "07b. Get subscription (as a list) and see exprLang jexl" -echo "=======================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "08. Update subscription with exprLang legacy" -echo "============================================" -payload='{ - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": "legacy" - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "09a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "09b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "10. Update subscription with invalid exprLang, see error" -echo "========================================================" -payload='{ - "notification": { - "mqttCustom": { - "url": "mqtt://localhost:1883", - "topic": "orion", - "exprLang": 42 - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH -echo -echo - - -echo "11a. Get subscription and see exprLang legacy" -echo "=============================================" -orionCurl --url /v2/subscriptions/$SUB_ID -echo -echo - - -echo "11b. Get subscription (as a list) and see exprLang legacy" -echo "=========================================================" -orionCurl --url /v2/subscriptions -echo -echo - ---REGEXPECT-- -01. Create subscription with invalid exprLang, see error -======================================================== -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 88 - -{ - "description": "not valid exprLang, valid ones are jexl or legacy", - "error": "BadRequest" -} - - -02. Create subscription with exprLang jexl -========================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -03a. Get subscription and see exprLang jexl -=========================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 359 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "jexl", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -03b. Get subscription (as a list) and see exprLang jexl -======================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "jexl", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -04. Update subscription with exprLang legacy -============================================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -05a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -05b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 363 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -06. Update subscription without exprLang -======================================== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -07a. Get subscription and see exprLang jexl -=========================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 359 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "jexl", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -07b. Get subscription (as a list) and see exprLang jexl -======================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "jexl", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -08. Update subscription with exprLang legacy -============================================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -09a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -09b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 363 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - -10. Update subscription with invalid exprLang, see error -======================================================== -HTTP/1.1 400 Bad Request -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 87 - -{ - "description": "exprLang mqttCustom notification is not a string", - "error": "BadRequest" -} - - -11a. Get subscription and see exprLang legacy -============================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 361 - -{ - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } -} - - -11b. Get subscription (as a list) and see exprLang legacy -========================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 363 - -[ - { - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "normalized", - "covered": false, - "mqttCustom": { - "exprLang": "legacy", - "qos": 0, - "retain": false, - "topic": "orion", - "url": "mqtt://localhost:1883" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*", - "type": "T" - } - ] - } - } -] - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test index ed8fee4246..4f2b9778b6 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_attr_no_type.test @@ -110,7 +110,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 585 +Content-Length: 567 { "id": "REGEX([0-9a-f]{24})", @@ -119,7 +119,6 @@ Content-Length: 585 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "SomeType", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test index 71d19596e5..9d671877f0 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud.test @@ -275,7 +275,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 383 +Content-Length: 365 { "id": "REGEX([0-9a-f]{24})", @@ -284,7 +284,6 @@ Content-Length: 383 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -327,7 +326,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -336,7 +335,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -372,7 +370,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 383 +Content-Length: 365 { "id": "REGEX([0-9a-f]{24})", @@ -381,7 +379,6 @@ Content-Length: 383 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -424,7 +421,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 383 +Content-Length: 365 { "id": "REGEX([0-9a-f]{24})", @@ -433,7 +430,6 @@ Content-Length: 383 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "C": { "type": "Number", @@ -476,7 +472,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -485,7 +481,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -523,7 +518,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -532,7 +527,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -568,7 +562,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 383 +Content-Length: 365 { "id": "REGEX([0-9a-f]{24})", @@ -577,7 +571,6 @@ Content-Length: 383 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", @@ -612,7 +605,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 722 +Content-Length: 686 [ { @@ -622,7 +615,6 @@ Content-Length: 722 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -649,7 +641,6 @@ Content-Length: 722 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test index 3ee0ec39fb..a34bfa6d96 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_compounds.test @@ -293,7 +293,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 432 +Content-Length: 414 { "id": "REGEX([0-9a-f]{24})", @@ -302,7 +302,6 @@ Content-Length: 432 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "StructuredValue", @@ -354,7 +353,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -363,7 +362,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -399,7 +397,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 433 +Content-Length: 415 { "id": "REGEX([0-9a-f]{24})", @@ -408,7 +406,6 @@ Content-Length: 433 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "StructuredValue", @@ -464,7 +461,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 429 +Content-Length: 411 { "id": "REGEX([0-9a-f]{24})", @@ -473,7 +470,6 @@ Content-Length: 429 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "C": { "type": "StructuredValue", @@ -525,7 +521,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -534,7 +530,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -572,7 +567,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -581,7 +576,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -617,7 +611,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 435 +Content-Length: 417 { "id": "REGEX([0-9a-f]{24})", @@ -626,7 +620,6 @@ Content-Length: 435 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "StructuredValue", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test index d6330c8c1f..2e60112a6e 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_basic_crud_partial.test @@ -259,7 +259,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 371 +Content-Length: 353 { "id": "REGEX([0-9a-f]{24})", @@ -268,7 +268,6 @@ Content-Length: 371 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -310,7 +309,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -319,7 +318,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "foo", "url": "http://127.0.0.1:9997/notify" }, @@ -355,7 +353,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 343 { "id": "REGEX([0-9a-f]{24})", @@ -364,7 +362,6 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -405,7 +402,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 351 +Content-Length: 333 { "id": "REGEX([0-9a-f]{24})", @@ -414,7 +411,6 @@ Content-Length: 351 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "id": "E3", "type": "T3" @@ -453,7 +449,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -462,7 +458,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "bar", "url": "http://127.0.0.1:9997/notify" }, @@ -500,7 +495,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 336 +Content-Length: 318 { "id": "REGEX([0-9a-f]{24})", @@ -509,7 +504,6 @@ Content-Length: 336 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "payload": "zzz", "url": "http://127.0.0.1:9997/notify" }, @@ -545,7 +539,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 373 +Content-Length: 355 { "id": "REGEX([0-9a-f]{24})", @@ -554,7 +548,6 @@ Content-Length: 373 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test index 0f62f2759f..8043fd7c4f 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_attr_no_type.test @@ -111,7 +111,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 618 +Content-Length: 600 { "id": "REGEX([0-9a-f]{24})", @@ -120,7 +120,6 @@ Content-Length: 618 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "SomeType", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test index d8421cc232..c08b1ab6ca 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud.test @@ -282,7 +282,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 416 +Content-Length: 398 { "id": "REGEX([0-9a-f]{24})", @@ -291,7 +291,6 @@ Content-Length: 416 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -337,7 +336,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -346,7 +345,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -385,7 +383,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 416 +Content-Length: 398 { "id": "REGEX([0-9a-f]{24})", @@ -394,7 +392,6 @@ Content-Length: 416 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -440,7 +437,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 416 +Content-Length: 398 { "id": "REGEX([0-9a-f]{24})", @@ -449,7 +446,6 @@ Content-Length: 416 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "C": { "type": "Number", @@ -495,7 +491,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -504,7 +500,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -545,7 +540,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -554,7 +549,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -593,7 +587,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 416 +Content-Length: 398 { "id": "REGEX([0-9a-f]{24})", @@ -602,7 +596,6 @@ Content-Length: 416 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", @@ -640,7 +633,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 788 +Content-Length: 752 [ { @@ -650,7 +643,6 @@ Content-Length: 788 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -680,7 +672,6 @@ Content-Length: 788 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test index 2eaf4e7c1e..fcbbb8f196 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_compounds.test @@ -300,7 +300,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 465 +Content-Length: 447 { "id": "REGEX([0-9a-f]{24})", @@ -309,7 +309,6 @@ Content-Length: 465 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "StructuredValue", @@ -364,7 +363,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -373,7 +372,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -412,7 +410,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 466 +Content-Length: 448 { "id": "REGEX([0-9a-f]{24})", @@ -421,7 +419,6 @@ Content-Length: 466 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "StructuredValue", @@ -480,7 +477,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 462 +Content-Length: 444 { "id": "REGEX([0-9a-f]{24})", @@ -489,7 +486,6 @@ Content-Length: 462 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "C": { "type": "StructuredValue", @@ -544,7 +540,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -553,7 +549,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -594,7 +589,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -603,7 +598,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -642,7 +636,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 468 +Content-Length: 450 { "id": "REGEX([0-9a-f]{24})", @@ -651,7 +645,6 @@ Content-Length: 468 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "StructuredValue", diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test index 0ea36db1d8..4ed1f9ce81 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_basic_crud_partial.test @@ -266,7 +266,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 404 +Content-Length: 386 { "id": "REGEX([0-9a-f]{24})", @@ -275,7 +275,6 @@ Content-Length: 404 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "A": { "type": "Number", @@ -320,7 +319,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -329,7 +328,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "foo", "qos": 0, "retain": false, @@ -368,7 +366,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 394 +Content-Length: 376 { "id": "REGEX([0-9a-f]{24})", @@ -377,7 +375,6 @@ Content-Length: 394 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "B": { "type": "Number", @@ -421,7 +418,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 384 +Content-Length: 366 { "id": "REGEX([0-9a-f]{24})", @@ -430,7 +427,6 @@ Content-Length: 384 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "id": "E3", "type": "T3" @@ -472,7 +468,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -481,7 +477,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "bar", "qos": 0, "retain": false, @@ -522,7 +517,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 351 { "id": "REGEX([0-9a-f]{24})", @@ -531,7 +526,6 @@ Content-Length: 369 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "payload": "zzz", "qos": 0, "retain": false, @@ -570,7 +564,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 406 +Content-Length: 388 { "id": "REGEX([0-9a-f]{24})", @@ -579,7 +573,6 @@ Content-Length: 406 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "ngsi": { "D": { "type": "Number", diff --git a/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test b/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test index 4054d27904..9aff1e0a20 100644 --- a/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test +++ b/test/functionalTest/cases/4388_mqtt_retain/mqttCustom_retain_basic_crud.test @@ -262,7 +262,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 358 +Content-Length: 340 { "id": "REGEX([0-9a-f]{24})", @@ -271,7 +271,6 @@ Content-Length: 358 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -301,7 +300,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 360 +Content-Length: 342 [ { @@ -311,7 +310,6 @@ Content-Length: 360 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -350,7 +348,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 359 +Content-Length: 341 { "id": "REGEX([0-9a-f]{24})", @@ -359,7 +357,6 @@ Content-Length: 359 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -389,7 +386,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 343 [ { @@ -399,7 +396,6 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -438,7 +434,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 359 +Content-Length: 341 { "id": "REGEX([0-9a-f]{24})", @@ -447,7 +443,6 @@ Content-Length: 359 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -477,7 +472,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 361 +Content-Length: 343 [ { @@ -487,7 +482,6 @@ Content-Length: 361 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": false, "topic": "orion", @@ -526,7 +520,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 358 +Content-Length: 340 { "id": "REGEX([0-9a-f]{24})", @@ -535,7 +529,6 @@ Content-Length: 358 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -565,7 +558,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 360 +Content-Length: 342 [ { @@ -575,7 +568,6 @@ Content-Length: 360 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -620,7 +612,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 358 +Content-Length: 340 { "id": "REGEX([0-9a-f]{24})", @@ -629,7 +621,6 @@ Content-Length: 358 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", @@ -659,7 +650,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 360 +Content-Length: 342 [ { @@ -669,7 +660,6 @@ Content-Length: 360 "attrsFormat": "normalized", "covered": false, "mqttCustom": { - "exprLang": "jexl", "qos": 0, "retain": true, "topic": "orion", diff --git a/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test b/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test index 191958e178..8352a77b7b 100644 --- a/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test +++ b/test/functionalTest/cases/4435_ngsi_patching_special_attr_types/ngsi_patching_special_attr_types.test @@ -146,7 +146,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 542 +Content-Length: 524 { "description": "DateTime test", @@ -159,7 +159,6 @@ Content-Length: 542 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "dateobservedto": { "type": "DateTime", @@ -206,7 +205,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 546 +Content-Length: 528 { "description": "DateTime test", @@ -219,7 +218,6 @@ Content-Length: 546 "attrsFormat": "normalized", "covered": false, "httpCustom": { - "exprLang": "jexl", "ngsi": { "dateobservedto2": { "type": "DateTime", From 48814b71055bc883c8cce85e0701c2a277f7da60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 10:52:33 +0200 Subject: [PATCH 239/390] FIX legacy term changed to basic --- doc/manuals/admin/statistics.md | 8 +++--- src/lib/common/macroSubstitute.cpp | 8 +++--- src/lib/common/statistics.cpp | 16 +++++------ src/lib/common/statistics.h | 42 ++++++++++++++--------------- src/lib/expressions/ExprContext.cpp | 34 +++++++++++------------ src/lib/expressions/ExprContext.h | 10 +++---- src/lib/expressions/ExprManager.cpp | 10 +++---- src/lib/ngsiNotify/Notifier.cpp | 18 ++++++------- src/lib/rest/rest.cpp | 4 +-- 9 files changed, 75 insertions(+), 75 deletions(-) diff --git a/doc/manuals/admin/statistics.md b/doc/manuals/admin/statistics.md index a2760412f2..7a0eb4f487 100644 --- a/doc/manuals/admin/statistics.md +++ b/doc/manuals/admin/statistics.md @@ -133,8 +133,8 @@ Provides timing information, i.e. the time that CB passes executing in different "mongoReadWait": 4656.924425628, "mongoWriteWait": 259.347915990, "mongoCommandWait": 0.514811318, - "exprLegacyCtxBld": FIXME PR, - "exprLegacyEval": FIXME PR, + "exprBasicCtxBld": FIXME PR, + "exprBasicEval": FIXME PR, "exprJexlCtxBld": FIXME PR, "exprJexlEval": FIXME PR, "render": 108.162782114, @@ -144,8 +144,8 @@ Provides timing information, i.e. the time that CB passes executing in different "mongoBackend": 0.014752309, "mongoReadWait": 0.012018445, "mongoWriteWait": 0.000574611, - "exprLegacyCtxBld": FIXME PR, - "exprLegacyEval": FIXME PR, + "exprBasicCtxBld": FIXME PR, + "exprBasicEval": FIXME PR, "exprJexlCtxBld": FIXME PR, "exprJexlEval": FIXME PR, "render": 0.000019136, diff --git a/src/lib/common/macroSubstitute.cpp b/src/lib/common/macroSubstitute.cpp index cc994b70cd..28dd9c156a 100644 --- a/src/lib/common/macroSubstitute.cpp +++ b/src/lib/common/macroSubstitute.cpp @@ -61,8 +61,8 @@ std::string smartStringValue(const std::string stringValue, ExprContextObject* e } else { - // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. - if (exprContextObjectP->isLegacy()) + // in basic mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. + if (exprContextObjectP->isBasic()) { result = removeQuotes(r.toString()); } @@ -114,8 +114,8 @@ static std::string stringValueOrNothing(ExprContextObject* exprContextObjectP, c { std::string s = r.toString(); - // in legacy mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. - if (exprContextObjectP->isLegacy()) + // in basic mode an extra remove quotes step is needed, to avoid "1" instead of 1 for number, etc. + if (exprContextObjectP->isBasic()) { s = removeQuotes(s); } diff --git a/src/lib/common/statistics.cpp b/src/lib/common/statistics.cpp index 1eb5dde069..10dd8b1631 100644 --- a/src/lib/common/statistics.cpp +++ b/src/lib/common/statistics.cpp @@ -195,8 +195,8 @@ std::string renderTimingStatistics(void) bool accMongoReadWaitTime = (accTimeStat.mongoReadWaitTime.tv_sec != 0) || (accTimeStat.mongoReadWaitTime.tv_nsec != 0); bool accMongoWriteWaitTime = (accTimeStat.mongoWriteWaitTime.tv_sec != 0) || (accTimeStat.mongoWriteWaitTime.tv_nsec != 0); bool accMongoCommandWaitTime = (accTimeStat.mongoCommandWaitTime.tv_sec != 0) || (accTimeStat.mongoCommandWaitTime.tv_nsec != 0); - bool accExprLegacyCtxBldTime = (accTimeStat.exprLegacyCtxBldTime.tv_sec != 0) || (accTimeStat.exprLegacyCtxBldTime.tv_nsec != 0); - bool accExprLegacyEvalTime = (accTimeStat.exprLegacyEvalTime.tv_sec != 0) || (accTimeStat.exprLegacyEvalTime.tv_nsec != 0); + bool accExprBasicCtxBldTime = (accTimeStat.exprBasicCtxBldTime.tv_sec != 0) || (accTimeStat.exprBasicCtxBldTime.tv_nsec != 0); + bool accExprBasicEvalTime = (accTimeStat.exprBasicEvalTime.tv_sec != 0) || (accTimeStat.exprBasicEvalTime.tv_nsec != 0); bool accExprJexlCtxBldTime = (accTimeStat.exprJexlCtxBldTime.tv_sec != 0) || (accTimeStat.exprJexlCtxBldTime.tv_nsec != 0); bool accExprJexlEvalTime = (accTimeStat.exprJexlEvalTime.tv_sec != 0) || (accTimeStat.exprJexlEvalTime.tv_nsec != 0); bool accRenderTime = (accTimeStat.renderTime.tv_sec != 0) || (accTimeStat.renderTime.tv_nsec != 0); @@ -208,8 +208,8 @@ std::string renderTimingStatistics(void) bool lastMongoReadWaitTime = (lastTimeStat.mongoReadWaitTime.tv_sec != 0) || (lastTimeStat.mongoReadWaitTime.tv_nsec != 0); bool lastMongoWriteWaitTime = (lastTimeStat.mongoWriteWaitTime.tv_sec != 0) || (lastTimeStat.mongoWriteWaitTime.tv_nsec != 0); bool lastMongoCommandWaitTime = (lastTimeStat.mongoCommandWaitTime.tv_sec != 0) || (lastTimeStat.mongoCommandWaitTime.tv_nsec != 0); - bool lastExprLegacyCtxBldTime = (lastTimeStat.exprLegacyCtxBldTime.tv_sec != 0) || (lastTimeStat.exprLegacyCtxBldTime.tv_nsec != 0); - bool lastExprLegacyEvalTime = (lastTimeStat.exprLegacyEvalTime.tv_sec != 0) || (lastTimeStat.exprLegacyEvalTime.tv_nsec != 0); + bool lastExprBasicCtxBldTime = (lastTimeStat.exprBasicCtxBldTime.tv_sec != 0) || (lastTimeStat.exprBasicCtxBldTime.tv_nsec != 0); + bool lastExprBasicEvalTime = (lastTimeStat.exprBasicEvalTime.tv_sec != 0) || (lastTimeStat.exprBasicEvalTime.tv_nsec != 0); bool lastExprJexlCtxBldTime = (lastTimeStat.exprJexlCtxBldTime.tv_sec != 0) || (lastTimeStat.exprJexlCtxBldTime.tv_nsec != 0); bool lastExprJexlEvalTime = (lastTimeStat.exprJexlEvalTime.tv_sec != 0) || (lastTimeStat.exprJexlEvalTime.tv_nsec != 0); bool lastRenderTime = (lastTimeStat.renderTime.tv_sec != 0) || (lastTimeStat.renderTime.tv_nsec != 0); @@ -236,8 +236,8 @@ std::string renderTimingStatistics(void) if (accMongoReadWaitTime) accJh.addNumber("mongoReadWait", timeSpecToFloat(accTimeStat.mongoReadWaitTime)); if (accMongoWriteWaitTime) accJh.addNumber("mongoWriteWait", timeSpecToFloat(accTimeStat.mongoWriteWaitTime)); if (accMongoCommandWaitTime) accJh.addNumber("mongoCommandWait", timeSpecToFloat(accTimeStat.mongoCommandWaitTime)); - if (accExprLegacyCtxBldTime) accJh.addNumber("exprLegacyCtxBld", timeSpecToFloat(accTimeStat.exprLegacyCtxBldTime)); - if (accExprLegacyEvalTime) accJh.addNumber("exprLegacyEval", timeSpecToFloat(accTimeStat.exprLegacyEvalTime)); + if (accExprBasicCtxBldTime) accJh.addNumber("exprBasicCtxBld", timeSpecToFloat(accTimeStat.exprBasicCtxBldTime)); + if (accExprBasicEvalTime) accJh.addNumber("exprBasicEval", timeSpecToFloat(accTimeStat.exprBasicEvalTime)); if (accExprJexlCtxBldTime) accJh.addNumber("exprJexlCtxBld", timeSpecToFloat(accTimeStat.exprJexlCtxBldTime)); if (accExprJexlEvalTime) accJh.addNumber("exprJexlEval", timeSpecToFloat(accTimeStat.exprJexlEvalTime)); if (accRenderTime) accJh.addNumber("render", timeSpecToFloat(accTimeStat.renderTime)); @@ -255,8 +255,8 @@ std::string renderTimingStatistics(void) if (lastMongoReadWaitTime) lastJh.addNumber("mongoReadWait", timeSpecToFloat(lastTimeStat.mongoReadWaitTime)); if (lastMongoWriteWaitTime) lastJh.addNumber("mongoWriteWait", timeSpecToFloat(lastTimeStat.mongoWriteWaitTime)); if (lastMongoCommandWaitTime) lastJh.addNumber("mongoCommandWait", timeSpecToFloat(lastTimeStat.mongoCommandWaitTime)); - if (lastExprLegacyCtxBldTime) lastJh.addNumber("exprLegacyCtxBld", timeSpecToFloat(lastTimeStat.exprLegacyCtxBldTime)); - if (lastExprLegacyEvalTime) lastJh.addNumber("exprLegacyEval", timeSpecToFloat(lastTimeStat.exprLegacyEvalTime)); + if (lastExprBasicCtxBldTime) lastJh.addNumber("exprBasicCtxBld", timeSpecToFloat(lastTimeStat.exprBasicCtxBldTime)); + if (lastExprBasicEvalTime) lastJh.addNumber("exprBasicEval", timeSpecToFloat(lastTimeStat.exprBasicEvalTime)); if (lastExprJexlCtxBldTime) lastJh.addNumber("exprJexlCtxBld", timeSpecToFloat(lastTimeStat.exprJexlCtxBldTime)); if (lastExprJexlEvalTime) lastJh.addNumber("exprJexlEval", timeSpecToFloat(lastTimeStat.exprJexlEvalTime)); if (lastRenderTime) lastJh.addNumber("render", timeSpecToFloat(lastTimeStat.renderTime)); diff --git a/src/lib/common/statistics.h b/src/lib/common/statistics.h index 623e074dcf..a5e1195781 100644 --- a/src/lib/common/statistics.h +++ b/src/lib/common/statistics.h @@ -233,9 +233,9 @@ struct timespec diff; \ clock_gettime(CLOCK_REALTIME, &exprCtxBldEnd); \ clock_difftime(&exprCtxBldEnd, &exprCtxBldStart, &diff); \ - if (legacy) \ + if (basic) \ { \ - clock_addtime(&threadLastTimeStat.exprLegacyCtxBldTime, &diff); \ + clock_addtime(&threadLastTimeStat.exprBasicCtxBldTime, &diff); \ } \ else \ { \ @@ -247,30 +247,30 @@ /* **************************************************************************** * -* TIME_EXPR_LEGACY_EVAL_START - +* TIME_EXPR_BASIC_EVAL_START - */ -#define TIME_EXPR_LEGACY_EVAL_START() \ - struct timespec exprLegacyEvalStart; \ - struct timespec exprLegacyEvalEnd; \ - \ - if (timingStatistics) \ - { \ - clock_gettime(CLOCK_REALTIME, &exprLegacyEvalStart); \ +#define TIME_EXPR_BASIC_EVAL_START() \ + struct timespec exprBasicEvalStart; \ + struct timespec exprBasicEvalEnd; \ + \ + if (timingStatistics) \ + { \ + clock_gettime(CLOCK_REALTIME, &exprBasicEvalStart); \ } /* **************************************************************************** * -* TIME_EXPR_LEGACY_EVAL_STOP - +* TIME_EXPR_BASIC_EVAL_STOP - */ -#define TIME_EXPR_LEGACY_EVAL_STOP() \ - if (timingStatistics) \ - { \ - struct timespec diff; \ - clock_gettime(CLOCK_REALTIME, &exprLegacyEvalEnd); \ - clock_difftime(&exprLegacyEvalEnd, &exprLegacyEvalStart, &diff); \ - clock_addtime(&threadLastTimeStat.exprLegacyEvalTime, &diff); \ +#define TIME_EXPR_BASIC_EVAL_STOP() \ + if (timingStatistics) \ + { \ + struct timespec diff; \ + clock_gettime(CLOCK_REALTIME, &exprBasicEvalEnd); \ + clock_difftime(&exprBasicEvalEnd, &exprBasicEvalStart, &diff); \ + clock_addtime(&threadLastTimeStat.exprBasicEvalTime, &diff); \ } @@ -299,7 +299,7 @@ { \ struct timespec diff; \ clock_gettime(CLOCK_REALTIME, &exprJexlEvalEnd); \ - clock_difftime(&exprJexlEvalEnd, &exprJexlEvalStart, &diff); \ + clock_difftime(&exprJexlEvalEnd, &exprJexlEvalStart, &diff); \ clock_addtime(&threadLastTimeStat.exprJexlEvalTime, &diff); \ } @@ -317,8 +317,8 @@ typedef struct TimeStat struct timespec mongoReadWaitTime; struct timespec mongoWriteWaitTime; struct timespec mongoCommandWaitTime; - struct timespec exprLegacyCtxBldTime; - struct timespec exprLegacyEvalTime; + struct timespec exprBasicCtxBldTime; + struct timespec exprBasicEvalTime; struct timespec exprJexlCtxBldTime; struct timespec exprJexlEvalTime; struct timespec renderTime; diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 6f8421ca1e..927809b693 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -34,9 +34,9 @@ * * ExprContextObject::ExprContextObject - */ -ExprContextObject::ExprContextObject(bool _legacy) +ExprContextObject::ExprContextObject(bool _basic) { - legacy = _legacy; + basic = _basic; } @@ -69,7 +69,7 @@ std::map* ExprContextObject::getMap(void) */ void ExprContextObject::add(const std::string &key, const std::string &_value, bool raw) { - if (legacy) + if (basic) { std::string value = _value; if (!raw) @@ -77,7 +77,7 @@ void ExprContextObject::add(const std::string &key, const std::string &_value, b // This is the case of regular string. The raw case is for JSON generated from compound values value = '"' + _value + '"'; } - LM_T(LmtExpr, ("adding to legacy expression context object (string): %s=%s", key.c_str(), value.c_str())); + LM_T(LmtExpr, ("adding to basic expression context object (string): %s=%s", key.c_str(), value.c_str())); repl.insert(std::pair(key, value)); } else @@ -95,9 +95,9 @@ void ExprContextObject::add(const std::string &key, const std::string &_value, b */ void ExprContextObject::add(const std::string &key, double _value) { - if (legacy) + if (basic) { - LM_T(LmtExpr, ("adding to legacy expression context object (double): %s=%f", key.c_str(), _value)); + LM_T(LmtExpr, ("adding to basic expression context object (double): %s=%f", key.c_str(), _value)); repl.insert(std::pair(key, double2string(_value))); } else @@ -115,9 +115,9 @@ void ExprContextObject::add(const std::string &key, double _value) */ void ExprContextObject::add(const std::string &key, bool _value) { - if (legacy) + if (basic) { - LM_T(LmtExpr, ("adding to legacy expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); + LM_T(LmtExpr, ("adding to basic expression context object (bool): %s=%s", key.c_str(), _value ? "true" : "false")); repl.insert(std::pair(key, _value? "true": "false")); } else @@ -135,9 +135,9 @@ void ExprContextObject::add(const std::string &key, bool _value) */ void ExprContextObject::add(const std::string &key) { - if (legacy) + if (basic) { - LM_T(LmtExpr, ("adding to legacy expression context object (none): %s", key.c_str())); + LM_T(LmtExpr, ("adding to basic expression context object (none): %s", key.c_str())); repl.insert(std::pair(key, "null")); } else @@ -155,9 +155,9 @@ void ExprContextObject::add(const std::string &key) */ void ExprContextObject::add(const std::string &key, ExprContextObject exprContextObject) { - if (legacy) + if (basic) { - LM_E(("Runtime Error (this method must not be invoked in legacy mode)")); + LM_E(("Runtime Error (this method must not be invoked in basic mode)")); } else { @@ -175,9 +175,9 @@ void ExprContextObject::add(const std::string &key, ExprContextObject exprContex */ void ExprContextObject::add(const std::string &key, ExprContextList exprContextList) { - if (legacy) + if (basic) { - LM_E(("Runtime Error (this method must not be invoked in legacy mode)")); + LM_E(("Runtime Error (this method must not be invoked in basic mode)")); } else { @@ -191,11 +191,11 @@ void ExprContextObject::add(const std::string &key, ExprContextList exprContextL /* **************************************************************************** * -* ExprContextObject::isLegacy - +* ExprContextObject::isBasic - */ -bool ExprContextObject::isLegacy(void) +bool ExprContextObject::isBasic(void) { - return legacy; + return basic; } diff --git a/src/lib/expressions/ExprContext.h b/src/lib/expressions/ExprContext.h index aa92e3094a..0e994be3b2 100644 --- a/src/lib/expressions/ExprContext.h +++ b/src/lib/expressions/ExprContext.h @@ -40,12 +40,12 @@ class ExprContextList; // forward declaration class ExprContextObject { private: - bool legacy; - JsonObjectHelper jh; // used in regular (i.e. not legacy) mode - std::map repl; // used in legacy mode + bool basic; + JsonObjectHelper jh; // used in regular (i.e. not basic) mode + std::map repl; // used in basic mode public: - ExprContextObject(bool legacy = false); + ExprContextObject(bool basic = false); std::string getJexlContext(void); std::map* getMap(void); @@ -57,7 +57,7 @@ class ExprContextObject void add(const std::string& key, ExprContextObject exprContextObject); void add(const std::string& key, ExprContextList exprContextList); - bool isLegacy(void); + bool isBasic(void); }; class ExprContextList diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index f54f91ef7c..d06bfb6d06 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -65,12 +65,12 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st ExprResult r; r.valueType = orion::ValueTypeNull; - if (exprContextObjectP->isLegacy()) + if (exprContextObjectP->isBasic()) { // std::map based evaluation. Only pure replacement is supported - TIME_EXPR_LEGACY_EVAL_START(); - LM_T(LmtExpr, ("evaluating legacy expression: <%s>", _expression.c_str())); + TIME_EXPR_BASIC_EVAL_START(); + LM_T(LmtExpr, ("evaluating basic expression: <%s>", _expression.c_str())); std::map* replacementsP = exprContextObjectP->getMap(); @@ -79,9 +79,9 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st { r.valueType = orion::ValueTypeString; r.stringValue = iter->second; - LM_T(LmtExpr, ("legacy evaluation result: <%s>", r.stringValue.c_str())); + LM_T(LmtExpr, ("basic evaluation result: <%s>", r.stringValue.c_str())); } - TIME_EXPR_LEGACY_EVAL_STOP(); + TIME_EXPR_BASIC_EVAL_STOP(); } else { diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index bd8c66738d..0542e7aa56 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -316,19 +316,19 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - // FIXME PR: unhardwire legacy == false - bool legacy = false; - ExprContextObject exprContext(legacy); + // FIXME PR: unhardwire basic == false + bool basic = false; + ExprContextObject exprContext(basic); - // FIXME PR: no-legacy context is now based in JsonHelper, which may mess with key repetition. Check this - // It seems that add() semantics are different in legacy and jexl mode. In jexl mode, if the key already exists, it is - // updated. In legacy model, if the key already exists, the operation is ignored (so previous value is preserved). Taking + // FIXME PR: jexl context (i.e. no basic) is now based in JsonHelper, which may mess with key repetition. Check this + // It seems that add() semantics are different in basic and jexl mode. In jexl mode, if the key already exists, it is + // updated. In basic model, if the key already exists, the operation is ignored (so previous value is preserved). Taking // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence // over macros comming from macros of the same name we conditionally add them depending the case TIME_EXPR_CTXBLD_START(); exprContext.add("id", en.id); exprContext.add("type", en.type); - if (!legacy) + if (!basic) { exprContext.add("service", tenant); exprContext.add("servicePath", en.servicePath); @@ -336,9 +336,9 @@ static SenderThreadParams* buildSenderParamsCustom } for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { - en.attributeVector[ix]->addToContext(&exprContext, legacy); + en.attributeVector[ix]->addToContext(&exprContext, basic); } - if (legacy) + if (basic) { exprContext.add("service", tenant); exprContext.add("servicePath", en.servicePath); diff --git a/src/lib/rest/rest.cpp b/src/lib/rest/rest.cpp index 3ddd0a85e7..f09fd37529 100644 --- a/src/lib/rest/rest.cpp +++ b/src/lib/rest/rest.cpp @@ -618,8 +618,8 @@ static void requestCompleted clock_addtime(&accTimeStat.mongoWriteWaitTime, &threadLastTimeStat.mongoWriteWaitTime); clock_addtime(&accTimeStat.mongoReadWaitTime, &threadLastTimeStat.mongoReadWaitTime); clock_addtime(&accTimeStat.mongoCommandWaitTime, &threadLastTimeStat.mongoCommandWaitTime); - clock_addtime(&accTimeStat.exprLegacyCtxBldTime, &threadLastTimeStat.exprLegacyCtxBldTime); - clock_addtime(&accTimeStat.exprLegacyEvalTime, &threadLastTimeStat.exprLegacyEvalTime); + clock_addtime(&accTimeStat.exprBasicCtxBldTime, &threadLastTimeStat.exprBasicCtxBldTime); + clock_addtime(&accTimeStat.exprBasicEvalTime, &threadLastTimeStat.exprBasicEvalTime); clock_addtime(&accTimeStat.exprJexlCtxBldTime, &threadLastTimeStat.exprJexlCtxBldTime); clock_addtime(&accTimeStat.exprJexlEvalTime, &threadLastTimeStat.exprJexlEvalTime); clock_addtime(&accTimeStat.renderTime, &threadLastTimeStat.renderTime); From 81e98c6669815e9a6edb36563e04ccbbb40118f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 12:22:59 +0200 Subject: [PATCH 240/390] ADD get_cjex.sh --- get_cjexl.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 get_cjexl.sh diff --git a/get_cjexl.sh b/get_cjexl.sh new file mode 100644 index 0000000000..c906af50b2 --- /dev/null +++ b/get_cjexl.sh @@ -0,0 +1,33 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +CJEXL_VERSION=$1 +TOKEN=$2 + +res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION) +if [ "$res_code" -eq 200 ]; then + echo "get_cjexl: downloading cjexl lib" + ASSET_ID=$(curl -s -S -H "Authorization: token $TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') + curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID + MD5SUM=$(md5sum /usr/local/lib/libcjexl.a | awk -F ' ' '{print$1}') + echo "get_cjexl: cjexl lib md5sum is $MD5SUM" +else + echo "get_cjexl: error $res_code accessing cjexl release. Maybe token is invalid?" +fi \ No newline at end of file From 8eefa7eef6b11215550be99413213c4316d3c313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 14:59:40 +0200 Subject: [PATCH 241/390] FIX Dockerfile --- ci/deb/build.sh | 1 + docker/Dockerfile | 4 ++++ docker/Dockerfile.alpine | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index 107cb11f30..fcdd48fa83 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -173,6 +173,7 @@ echo "Builder: create temp folders" rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} # FIXME PR: unhardwire release number +# FIXME PR: use get_cjexl script CJEXL_VERSION="0.0.0" if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" diff --git a/docker/Dockerfile b/docker/Dockerfile index b5b80157d4..13033a9343 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -29,11 +29,13 @@ ARG GITHUB_REPOSITORY=fiware-orion ARG GIT_NAME ARG GIT_REV_ORION ARG CLEAN_DEV_TOOLS +ARG REPO_ACCESS_TOKEN ENV ORION_USER ${ORION_USER:-orion} ENV GIT_NAME ${GIT_NAME:-telefonicaid} ENV GIT_REV_ORION ${GIT_REV_ORION:-master} ENV CLEAN_DEV_TOOLS ${CLEAN_DEV_TOOLS:-1} +ENV REPO_ACCESS_TOKEN ${REPO_ACCESS_TOKEN:-""} SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -102,6 +104,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ + bash get_cjexl.sh 0.0.0 ${REPO_ACCESS_TOKEN} && \ make && \ make install && \ # reduce size of installed binaries @@ -125,6 +128,7 @@ RUN \ /opt/mongo-c-driver-1.24.3 \ /usr/local/include/mongo \ /usr/local/lib/libmongoclient.a \ + /usr/local/lib/libcjexl.a \ /opt/rapidjson-1.1.0 \ /opt/v1.1.0.tar.gz \ /usr/local/include/rapidjson \ diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 524335f677..2c4ed4a02e 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -32,11 +32,13 @@ ARG GITHUB_REPOSITORY=fiware-orion ARG GIT_NAME ARG GIT_REV_ORION ARG CLEAN_DEV_TOOLS +ARG REPO_ACCESS_TOKEN ENV ORION_USER ${ORION_USER:-orion} ENV GIT_NAME ${GIT_NAME:-telefonicaid} ENV GIT_REV_ORION ${GIT_REV_ORION:-master} ENV CLEAN_DEV_TOOLS ${CLEAN_DEV_TOOLS:-1} +ENV REPO_ACCESS_TOKEN ${REPO_ACCESS_TOKEN:-""} SHELL ["/bin/ash", "-o", "pipefail", "-c"] @@ -108,6 +110,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ + bash get_cjexl.sh 0.0.0 ${REPO_ACCESS_TOKEN} && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ @@ -133,6 +136,7 @@ RUN \ /opt/mongo-c-driver-1.24.3 \ /usr/local/include/mongo \ /usr/local/lib/libmongoclient.a \ + /usr/local/lib/libcjexl.a \ /opt/rapidjson-1.1.0 \ /opt/v1.1.0.tar.gz \ /usr/local/include/rapidjson \ From 915188554156d8d6eaab58db27ed646a504b4f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 15:36:44 +0200 Subject: [PATCH 242/390] FIX add new jexl test cases --- .../jexl_several_expressions.test | 232 ++++++++++++++++++ .../jexl_syntax_errors.test | 178 ++++++++++++++ .../jexl_transformation_unknown.test | 144 +++++++++++ 3 files changed, 554 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test new file mode 100644 index 0000000000..82ffb1ccaa --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test @@ -0,0 +1,232 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (several expressions in same subscription) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression sum: A+B, mul: A*B +# 02. Create entity E1 with A=2 and B=3 +# 03. Update entity E1 with A=2.1 and B=-3.8 +# 04. Dump accumulator and see two notifications ({sum: 5, mul: 6}, {sum: -1.7, mul: -7.98}) +# + + +echo "01. Create custom sub with custom expression sum: A+B, mul: A*B" +echo "===============================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "sum": { + "value": "${A+B}", + "type": "Calculated" + }, + "mul": { + "value": "${A*B}", + "type": "Calculated" + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=2 and B=3" +echo "=====================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 2, + "type": "Number" + }, + "B": { + "value": 3, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A=2.1 and B=-3.8" +echo "==========================================" +payload='{ + "A": { + "value": 2.1, + "type": "Number" + }, + "B": { + "value": -3.8, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "Dump accumulator and see two notifications ({sum: 5, mul: 6}, {sum: -1.7, mul: -7.98})" +echo "======================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: A+B, mul: A*B +=============================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=2 and B=3 +===================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A=2.1 and B=-3.8 +========================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +Dump accumulator and see two notifications ({sum: 5, mul: 6}, {sum: -1.7, mul: -7.98}) +====================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 273 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": 3 + }, + "id": "E1", + "mul": { + "metadata": {}, + "type": "Calculated", + "value": 6 + }, + "sum": { + "metadata": {}, + "type": "Calculated", + "value": 5 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 285 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2.1 + }, + "B": { + "metadata": {}, + "type": "Number", + "value": -3.8 + }, + "id": "E1", + "mul": { + "metadata": {}, + "type": "Calculated", + "value": -7.98 + }, + "sum": { + "metadata": {}, + "type": "Calculated", + "value": -1.7 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test new file mode 100644 index 0000000000..4359b235bc --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test @@ -0,0 +1,178 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification with syntax errors + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression with several JEXL expressions with syntax errors +# 02. Create entity E1 with A=[ foobar ] +# 03. Dump accumulator and see notification with expressions resolved to null (except e0 = TRUE) +# + + +echo "01. Create custom sub with custom expression with several JEXL expressions with syntax errors" +echo "=============================================================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "e0": { + "value": "${A[0]|includes('\''foo'\'')|toString|uppercase}", + "type": "Calculated" + }, + "e1": { + "value": "${A[0|includes('\''foo'\'')|toString|uppercase}", + "type": "Calculated" + }, + "e2": { + "value": "${A[0]|includes('\''foo)|toString|uppercase}", + "type": "Calculated" + }, + "e3": { + "value": "${A[0]|includes('\''foo'\''|toString|uppercase}", + "type": "Calculated" + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=[ foobar ]" +echo "======================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": [ "foobar" ], + "type": "StructuredValue" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Dump accumulator and see notification with expressions resolved to null (except e0 = TRUE)" +echo "==============================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression with several JEXL expressions with syntax errors +============================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=[ foobar ] +====================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Dump accumulator and see notification with expressions resolved to null (except e0 = TRUE) +============================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 359 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "StructuredValue", + "value": [ + "foobar" + ] + }, + "e0": { + "metadata": {}, + "type": "Calculated", + "value": "TRUE" + }, + "e1": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "e2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "e3": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test new file mode 100644 index 0000000000..3d536559b4 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test @@ -0,0 +1,144 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using unknown transformation) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression low: A|patatacase +# 02. Create entity E1 with A=IwantAllInLOWERCase +# 03. Dump accumulator and see notifications (low: null) +# + + +echo "01. Create custom sub with custom expression low: A|patatacase" +echo "==============================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "low": { + "value": "${A|patatacase}", + "type": "Calculated" + } + } + }, + "attrs": [ "low" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=IwantAllInLOWERCase" +echo "===============================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": "IwantAllInLOWERCase", + "type": "Text" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Dump accumulator and see notifications (low: null)" +echo "======================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression low: A|patatacase +============================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=IwantAllInLOWERCase +=============================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Dump accumulator and see notifications (low: null) +====================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "low": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 76234f49836c353caeb679d138368a44b1508023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 17:40:56 +0200 Subject: [PATCH 243/390] FIX some FIXME PR marks --- ci/deb/build.sh | 14 +------------- src/lib/ngsiNotify/Notifier.cpp | 10 +++++----- .../jexl_basic_attrs_in_update.test | 10 +++++----- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index fcdd48fa83..5a5393f877 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -172,22 +172,10 @@ echo "===================================== PREPARE ============================ echo "Builder: create temp folders" rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} -# FIXME PR: unhardwire release number -# FIXME PR: use get_cjexl script -CJEXL_VERSION="0.0.0" if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION) - if [ "$res_code" -eq 200 ]; then - echo "Builder: downloading cjexl lib" - ASSET_ID=$(curl -s -S -H "Authorization: token $REPO_ACCESS_TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') - curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $REPO_ACCESS_TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID - MD5SUM=$(md5sum /usr/local/lib/libcjexl.a | awk -F ' ' '{print$1}') - echo "Builder: cjexl lib md5sum is $MD5SUM" - else - echo "Builder: error $res_code accessing cjexl release. Maybe token is invalid?" - fi + bash /opt/fiware-orion/get_cjexl.sh 0.0.0 $REPO_ACCESS_TOKEN fi if [ -n "${branch}" ]; then diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 0542e7aa56..a04d7c124b 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -316,15 +316,15 @@ static SenderThreadParams* buildSenderParamsCustom Entity& en = notifyCerP->entity; // Used by several macroSubstitute() calls along this function - // FIXME PR: unhardwire basic == false + // FIXME PR: unhardwire basic == false depending on conditional compiling bool basic = false; ExprContextObject exprContext(basic); - // FIXME PR: jexl context (i.e. no basic) is now based in JsonHelper, which may mess with key repetition. Check this // It seems that add() semantics are different in basic and jexl mode. In jexl mode, if the key already exists, it is - // updated. In basic model, if the key already exists, the operation is ignored (so previous value is preserved). Taking - // into account that in the case of an attribute with name "service", "servicePath" and "authToken", must have precedence - // over macros comming from macros of the same name we conditionally add them depending the case + // updated (in other words, the last added keys is the one that takes precedence). In basic model, if the key already + // exists, the operation is ignored (in other words, the first added key is the one that takes precedence). Taking + // into account that in the case of an attribute with name "service", "servicePath" or "authToken", it must have precedence + // over the ones comming from headers of the same name, we conditionally add them depending the case TIME_EXPR_CTXBLD_START(); exprContext.add("id", en.id); exprContext.add("type", en.type); diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test index 601f69d25a..6e31d9b36e 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test @@ -35,7 +35,7 @@ accumulatorStart --pretty-print # 02. Create entity E1 with A=1 and B=2 # 03. Update entity E1 with A=foo and B=bar # 04. Update entity E1 with A=2.1 and B=-3.8 -# 05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7) +# 05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7) # @@ -120,8 +120,8 @@ echo echo -echo "05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7)" -echo "===============================================================================" +echo "05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7)" +echo "=================================================================================" accumulatorDump echo echo @@ -164,8 +164,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -05. Dump accumulator and see two notifications (sum: 3, sum: foobar, sum: -1.7) -=============================================================================== +05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7) +================================================================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 221 From 4bf61333d920361a6fcbd59433d9cdb3db2a4838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 8 May 2024 18:36:06 +0200 Subject: [PATCH 244/390] FIX conditional compilinig depending on cjexl availability --- CMakeLists.txt | 11 ++++++++++- src/lib/expressions/ExprManager.cpp | 17 +++++++++++++++++ src/lib/ngsiNotify/Notifier.cpp | 8 ++++++-- src/lib/serviceRoutines/versionTreat.cpp | 4 ++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b871e47ad8..0118dbef19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -230,11 +230,20 @@ SET (BOOST_MT # SET for common static libs. We use 1.24.3 as reference version. find_package (mongoc-1.0 1.24.3 EXACT) +# Is cjexl lib available? +find_library (HAVE_CJEXL cjexl PATHS /usr/lib /usr/lib64 /usr/local/lib64 /usr/local/lib) +if (HAVE_CJEXL) + message("Using cjexl") +else (HAVE_CJEXL) + message("Not using cjexl") + add_definitions(-DEXPR_BASIC) +endif (HAVE_CJEXL) + # Static libs common to contextBroker and unitTest binaries SET (COMMON_STATIC_LIBS microhttpd.a mosquitto.a - cjexl.a + ${HAVE_CJEXL} mongo::mongoc_static ) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index d06bfb6d06..47e492f223 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -30,6 +30,22 @@ #include "orionTypes/OrionValueType.h" #include "common/statistics.h" +#ifdef EXPR_BASIC +// Never called, but need to be defined to avoid compilation errors +static void* cjexl_new_engine() +{ + return NULL; +} + +static void cjexl_free_engine(void* ptr) +{ +} + +static const char* cjexl_eval(void* ptr, const char* script_ptr, const char* context_ptr) +{ + return ""; +} +#else // Interface to use libcjexl.a extern "C" { void* cjexl_new_engine(); @@ -42,6 +58,7 @@ extern "C" { extern "C" { const char* cjexl_eval(void* ptr, const char* script_ptr, const char* context_ptr); } +#endif /* **************************************************************************** diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index a04d7c124b..df16cbb610 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -315,9 +315,13 @@ static SenderThreadParams* buildSenderParamsCustom std::map headers; Entity& en = notifyCerP->entity; - // Used by several macroSubstitute() calls along this function - // FIXME PR: unhardwire basic == false depending on conditional compiling +#ifdef EXPR_BASIC + bool basic = true; +#else bool basic = false; +#endif + + // Used by several macroSubstitute() calls along this function ExprContextObject exprContext(basic); // It seems that add() semantics are different in basic and jexl mode. In jexl mode, if the key already exists, it is diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index eec99efedb..8a0fa24038 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -47,10 +47,12 @@ #include #include +#ifndef EXPR_BASIC // Interface to use libcjexl.a extern "C" { const char* cjexl_version(); } +#endif /* **************************************************************************** * @@ -94,7 +96,9 @@ std::string libVersions(void) total += curl + "\"" + curlVersion + "\"" + ",\n"; total += mosq + "\"" + mosqVersion + "\"" + ",\n"; total += mhd + "\"" + MHD_get_version() + "\"" + ",\n"; +#ifndef EXPR_BASIC total += cjexl + "\"" + cjexl_version() + "\"" + ",\n"; +#endif #ifdef OLD_SSL_VERSION_FORMAT // Needed by openssl 1.1.1n in Debian 11 and before total += ssl + "\"" + SHLIB_VERSION_NUMBER "\"" + ",\n"; From 25b463cf910f36a5f8edb7014ce32bac6e5da293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 13 May 2024 14:03:00 +0200 Subject: [PATCH 245/390] ADD jexl_missing_attribute_as_null.test test --- .../jexl_missing_attribute_as_null.test | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test new file mode 100644 index 0000000000..7cdc57b975 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test @@ -0,0 +1,247 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (missing attribute replaced as null) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression B: (A==null)?'is null':A +# 02. Create entity E1 with A=1 +# 03. Create entity E2 with A=null +# 04. Create entity E3 without A +# 05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +# + + +echo "01. Create custom sub with custom expression B: (A==null)?'is null':A" +echo "=====================================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B": { + "value": "${(A==null)?'\''is null'\'':A}", + "type": "Calculated" + } + } + }, + "attrs": [ "B" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Create entity E2 with A=null" +echo "================================" +payload='{ + "id": "E2", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "04. Create entity E3 without A" +echo "==============================" +payload='{ + "id": "E3", + "type": "T", + "C": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null)" +echo "===============================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression B: (A==null)?'is null':A +===================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Create entity E2 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +04. Create entity E3 without A +============================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E3?type=T +Content-Length: 0 + + + +05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +=============================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 127 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 135 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "id": "E2", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 135 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "id": "E3", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 153ad23622e729e6af0d1ee154e5af4deb6cdba1 Mon Sep 17 00:00:00 2001 From: pkel-kn Date: Tue, 14 May 2024 08:17:30 +0200 Subject: [PATCH 246/390] Update walkthrough_apiv2.md Consistent syntax highlighting in API walkthrough --- doc/manuals/user/walkthrough_apiv2.md | 120 +++++++++++++------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/doc/manuals/user/walkthrough_apiv2.md b/doc/manuals/user/walkthrough_apiv2.md index 568b208a7f..7268a838c2 100644 --- a/doc/manuals/user/walkthrough_apiv2.md +++ b/doc/manuals/user/walkthrough_apiv2.md @@ -117,10 +117,10 @@ host and/or port can be specified) and echoes whatever it receives in the terminal window where it is executed. Run it using the following command: -``` -# cd /dir/where/accumulator-server/is/downloaded -# chmod a+x accumulator-server.py -# ./accumulator-server.py --port 1028 --url /accumulate --host ::1 --pretty-print -v +```shell +cd /dir/where/accumulator-server/is/downloaded +chmod a+x accumulator-server.py +./accumulator-server.py --port 1028 --url /accumulate --host ::1 --pretty-print -v ``` Note this script requires Flask version 2.0.2 (along with Werkzeug 2.0.2) and paho-mqtt version 1.6.1, which can be installed using @@ -148,7 +148,7 @@ following: - For POST: -``` +```shell curl localhost:1026/ -s -S [headers] -d @- < -s -S [headers] -X PUT -d @- < -s -S [headers] -X PATCH -d @- < -s -S [headers] ``` - For DELETE: -``` +```shell curl localhost:1026/ -s -S [headers] -X DELETE ``` @@ -187,14 +187,14 @@ Regarding \[headers\] you have to include the following ones: - Accept header to specify the payload format in which you want to receive the response. You should explicitly specify JSON. -``` +```shell curl ... -H 'Accept: application/json' ... ``` - If using payload in the request (i.e. POST, PUT or PATCH), you have to supply the `Context-Type` HTTP header to specify the format (JSON). -``` +```shell curl ... -H 'Content-Type: application/json' ... ``` @@ -214,7 +214,7 @@ Some additional remarks: `python` by the particular Python executable in your case (e.g. in some cases it could be `| python3 -mjson.tool`). -``` +```shell (curl ... | python -mjson.tool) <22' -s -S -H 'Accept: application/json' | python -mjson.tool ``` @@ -565,7 +565,7 @@ source of context information. Let's assume that this application in a given moment wants to set the temperature and pressure of Room1 to 26.5 ºC and 763 mmHg respectively, so it issues the following request: -``` +```shell curl localhost:1026/v2/entities/Room1/attrs -s -S -H 'Content-Type: application/json' -X PATCH -d @- < Date: Tue, 14 May 2024 11:45:21 +0200 Subject: [PATCH 247/390] FIX simplify attribute_as_null test case and add extra one --- .../jexl_missing_attribute_as_null.test | 12 +- .../jexl_missing_attribute_as_null2.test | 247 ++++++++++++++++++ 2 files changed, 253 insertions(+), 6 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test index 7cdc57b975..a7af691e9c 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test @@ -31,7 +31,7 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with custom expression B: (A==null)?'is null':A +# 01. Create custom sub with custom expression B: A||'is null' # 02. Create entity E1 with A=1 # 03. Create entity E2 with A=null # 04. Create entity E3 without A @@ -39,8 +39,8 @@ accumulatorStart --pretty-print # -echo "01. Create custom sub with custom expression B: (A==null)?'is null':A" -echo "=====================================================================" +echo "01. Create custom sub with custom expression B: A||'is null'" +echo "============================================================" # NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) payload='{ "subject": { @@ -56,7 +56,7 @@ payload='{ "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", "ngsi": { "B": { - "value": "${(A==null)?'\''is null'\'':A}", + "value": "${A||'\''is null'\''}", "type": "Calculated" } } @@ -122,8 +122,8 @@ echo --REGEXPECT-- -01. Create custom sub with custom expression B: (A==null)?'is null':A -===================================================================== +01. Create custom sub with custom expression B: A||'is null' +============================================================ HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test new file mode 100644 index 0000000000..a7af691e9c --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test @@ -0,0 +1,247 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (missing attribute replaced as null) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression B: A||'is null' +# 02. Create entity E1 with A=1 +# 03. Create entity E2 with A=null +# 04. Create entity E3 without A +# 05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +# + + +echo "01. Create custom sub with custom expression B: A||'is null'" +echo "============================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B": { + "value": "${A||'\''is null'\''}", + "type": "Calculated" + } + } + }, + "attrs": [ "B" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Create entity E2 with A=null" +echo "================================" +payload='{ + "id": "E2", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "04. Create entity E3 without A" +echo "==============================" +payload='{ + "id": "E3", + "type": "T", + "C": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null)" +echo "===============================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression B: A||'is null' +============================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Create entity E2 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +04. Create entity E3 without A +============================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E3?type=T +Content-Length: 0 + + + +05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +=============================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 127 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 135 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "id": "E2", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 135 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "B": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "id": "E3", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 264a9a6203a787e1c8af5c923878637cf00c994c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 12:38:50 +0200 Subject: [PATCH 248/390] FIX improve missing attribute as null jexl tests --- .../jexl_missing_attribute_as_null.test | 131 +++++++++++++++--- .../jexl_missing_attribute_as_null2.test | 83 +++++------ 2 files changed, 156 insertions(+), 58 deletions(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test index a7af691e9c..9d711fc6a7 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test @@ -31,16 +31,16 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with custom expression B: A||'is null' +# 01. Create custom sub with custom expression B0: (A==null)?-1:A, B1: A||'is null', B2: A||-1, B3: A||0-1||, B4: A||('-1'|parseInt), B5: A||0 # 02. Create entity E1 with A=1 # 03. Create entity E2 with A=null # 04. Create entity E3 without A -# 05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +# 05. Dump accumulator and see three notifications (<1, 1, null, 1, 1, 1>, <'is null', 'is null', null, -1, -1, 0>, ) # -echo "01. Create custom sub with custom expression B: A||'is null'" -echo "============================================================" +echo "01. Create custom sub with custom expression B0: (A==null)?-1:A, B1: A||'is null', B2: A||-1, B3: A||0-1||, B4: A||'-1'||parseInt, B5: A||0" +echo "===========================================================================================================================================" # NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) payload='{ "subject": { @@ -55,13 +55,33 @@ payload='{ "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", "ngsi": { - "B": { + "B0": { + "value": "${(A==null)?'\''is null'\'':A}", + "type": "Calculated" + }, + "B1": { "value": "${A||'\''is null'\''}", "type": "Calculated" + }, + "B2": { + "value": "${A||-1}", + "type": "Calculated" + }, + "B3": { + "value": "${A||0-1}", + "type": "Calculated" + }, + "B4": { + "value": "${A||('\''-1'\''|parseInt)}", + "type": "Calculated" + }, + "B5": { + "value": "${A||0}", + "type": "Calculated" } } }, - "attrs": [ "B" ] + "attrs": [ "B0", "B1", "B2", "B3", "B4", "B5" ] } }' orionCurl --url /v2/subscriptions --payload "$payload" @@ -114,16 +134,16 @@ echo echo -echo "05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null)" -echo "===============================================================================" +echo "05. Dump accumulator and see three notifications (<1, 1, null, 1, 1, 1>, <'is null', 'is null', null, -1, -1, 0>, )" +echo "=====================================================================================================================================================" accumulatorDump echo echo --REGEXPECT-- -01. Create custom sub with custom expression B: A||'is null' -============================================================ +01. Create custom sub with custom expression B0: (A==null)?-1:A, B1: A||'is null', B2: A||-1, B3: A||0-1||, B4: A||'-1'||parseInt, B5: A||0 +=========================================================================================================================================== HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -162,11 +182,11 @@ Content-Length: 0 -05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) -=============================================================================== +05. Dump accumulator and see three notifications (<1, 1, null, 1, 1, 1>, <'is null', 'is null', null, -1, -1, 0>, ) +===================================================================================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 127 +Content-Length: 386 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -177,7 +197,32 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "B0": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "B1": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "B2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "B3": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "B4": { + "metadata": {}, + "type": "Calculated", + "value": 1 + }, + "B5": { "metadata": {}, "type": "Calculated", "value": 1 @@ -191,7 +236,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 135 +Content-Length: 404 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -202,11 +247,36 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "B0": { + "metadata": {}, + "type": "Calculated", + "value": "is null" + }, + "B1": { "metadata": {}, "type": "Calculated", "value": "is null" }, + "B2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "B3": { + "metadata": {}, + "type": "Calculated", + "value": -1 + }, + "B4": { + "metadata": {}, + "type": "Calculated", + "value": -1 + }, + "B5": { + "metadata": {}, + "type": "Calculated", + "value": 0 + }, "id": "E2", "type": "T" } @@ -216,7 +286,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 135 +Content-Length: 399 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -227,11 +297,36 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "B0": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "B1": { "metadata": {}, "type": "Calculated", "value": "is null" }, + "B2": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "B3": { + "metadata": {}, + "type": "Calculated", + "value": -1 + }, + "B4": { + "metadata": {}, + "type": "Calculated", + "value": -1 + }, + "B5": { + "metadata": {}, + "type": "Calculated", + "value": 0 + }, "id": "E3", "type": "T" } diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test index a7af691e9c..8c0fd939f4 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -JEXL expression in custom notification (missing attribute replaced as null) +JEXL expression in custom notification (missing attribute replaced as null additional case) --SHELL-INIT-- dbInit CB @@ -31,17 +31,16 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with custom expression B: A||'is null' -# 02. Create entity E1 with A=1 -# 03. Create entity E2 with A=null -# 04. Create entity E3 without A -# 05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) +# 01. Create custom sub with custom expression S: (A||0-1)+(B||0-2) +# 02. Create entity E1 with A=1, B=2 +# 03. Create entity E2 with B=1 +# 04. Create entity E3 without A or B +# 05. Dump accumulator and see three notifications (S: 3, S: 0, S: -3) # -echo "01. Create custom sub with custom expression B: A||'is null'" -echo "============================================================" -# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +echo "01. Create custom sub with custom expression S: (A||0-1)+(B||0-2)" +echo "=================================================================" payload='{ "subject": { "entities": [ @@ -55,13 +54,13 @@ payload='{ "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", "ngsi": { - "B": { - "value": "${A||'\''is null'\''}", + "S": { + "value": "${(A||0-1)+(B||0-2)}", "type": "Calculated" } } }, - "attrs": [ "B" ] + "attrs": [ "S" ] } }' orionCurl --url /v2/subscriptions --payload "$payload" @@ -69,14 +68,18 @@ echo echo -echo "02. Create entity E1 with A=1" -echo "=============================" +echo "02. Create entity E1 with A=1, B=2" +echo "==================================" payload='{ "id": "E1", "type": "T", "A": { "value": 1, "type": "Number" + }, + "B": { + "value": 2, + "type": "Number" } }' orionCurl --url /v2/entities --payload "$payload" @@ -84,13 +87,13 @@ echo echo -echo "03. Create entity E2 with A=null" -echo "================================" +echo "03. Create entity E2 with B=1" +echo "=============================" payload='{ "id": "E2", "type": "T", - "A": { - "value": null, + "B": { + "value": 1, "type": "Number" } }' @@ -99,8 +102,8 @@ echo echo -echo "04. Create entity E3 without A" -echo "==============================" +echo "04. Create entity E3 without A or B" +echo "===================================" payload='{ "id": "E3", "type": "T", @@ -114,16 +117,16 @@ echo echo -echo "05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null)" -echo "===============================================================================" +echo "05. Dump accumulator and see three notifications (S: 3, S: 0, S: -3)" +echo "====================================================================" accumulatorDump echo echo --REGEXPECT-- -01. Create custom sub with custom expression B: A||'is null' -============================================================ +01. Create custom sub with custom expression S: (A||0-1)+(B||0-2) +================================================================= HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -132,8 +135,8 @@ Content-Length: 0 -02. Create entity E1 with A=1 -============================= +02. Create entity E1 with A=1, B=2 +================================== HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -142,8 +145,8 @@ Content-Length: 0 -03. Create entity E2 with A=null -================================ +03. Create entity E2 with B=1 +============================= HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -152,8 +155,8 @@ Content-Length: 0 -04. Create entity E3 without A -============================== +04. Create entity E3 without A or B +=================================== HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -162,8 +165,8 @@ Content-Length: 0 -05. Dump accumulator and see three notifications (B: 1, B: is null, B: is null) -=============================================================================== +05. Dump accumulator and see three notifications (S: 3, S: 0, S: -3) +==================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 127 @@ -177,10 +180,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "S": { "metadata": {}, "type": "Calculated", - "value": 1 + "value": 3 }, "id": "E1", "type": "T" @@ -191,7 +194,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 135 +Content-Length: 127 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -202,10 +205,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "S": { "metadata": {}, "type": "Calculated", - "value": "is null" + "value": 0 }, "id": "E2", "type": "T" @@ -216,7 +219,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 135 +Content-Length: 128 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -227,10 +230,10 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "B": { + "S": { "metadata": {}, "type": "Calculated", - "value": "is null" + "value": -3 }, "id": "E3", "type": "T" From dc35a4a81050c827a8347c1182373134a24e9b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 13:34:21 +0200 Subject: [PATCH 249/390] ADD GitAction to publish docker in Dockerhub for branches --- .github/workflows/publishimage-master.yml | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/publishimage-master.yml diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml new file mode 100644 index 0000000000..c302600bad --- /dev/null +++ b/.github/workflows/publishimage-master.yml @@ -0,0 +1,35 @@ +name: Publish Docker image (master) + +# The workflow will push images on every tag in the format x.y.z +# FIXME PR: change label and branch before merging into master + +on: + push: + branches: + #- master + - feature/4004_jexl_expressions + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build Docker image + #run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + run: docker build -t telefonicaiot/fiware-orion:jexl --build-arg GIT_REV_ORION=feature/4004_jexl_expressions --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + + - name: Push Docker image + #run: docker push telefonicaiot/fiware-orion:latest + run: docker push telefonicaiot/fiware-orion:jexl From 2a75eb1fcc311b5891217934913ce425d06295b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 13:38:22 +0200 Subject: [PATCH 250/390] FIX GitAction login step --- .github/workflows/publishimage-master.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index c302600bad..07347c86c7 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -23,8 +23,8 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build Docker image #run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . From e789afcd7dbaae8fd4c281d3b00847f00754ca30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 14:15:16 +0200 Subject: [PATCH 251/390] FIX GitAction dockerhub credential fix --- .github/workflows/publishimage-master.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index 07347c86c7..a238eb15fb 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -23,8 +23,8 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_TEF_USERNAME }} + password: ${{ secrets.DOCKERHUB_TEF_TOKEN }} - name: Build Docker image #run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . From 7281e1f9abd3991a01d71388781153c4a9ff51fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 17 May 2024 14:47:13 +0200 Subject: [PATCH 252/390] ADD GitAction for build docker for tags --- ...blishimage.yml => publishimage-fiware.yml} | 4 +-- .github/workflows/publishimage-master.yml | 12 +++---- .github/workflows/publishimage-tag.yml | 35 +++++++++++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) rename .github/workflows/{publishimage.yml => publishimage-fiware.yml} (89%) create mode 100644 .github/workflows/publishimage-tag.yml diff --git a/.github/workflows/publishimage.yml b/.github/workflows/publishimage-fiware.yml similarity index 89% rename from .github/workflows/publishimage.yml rename to .github/workflows/publishimage-fiware.yml index af56a99bae..07f10beb01 100644 --- a/.github/workflows/publishimage.yml +++ b/.github/workflows/publishimage-fiware.yml @@ -1,6 +1,6 @@ -name: Publish pre Image from master +name: Publish pre Image from master in FIWARE dockerhub -# The workflow will push PRE images from master on every merge. The images will be tagged with PRE and the next minor increase on the +# The workflow will push PRE images from master on every merge to the fiware dockerhub organization. The images will be tagged with PRE and the next minor increase on the # semver(based on the github releases) # It will NOT produce releases and release images or the 'latest' tag from master. Both (releases and 'latest' tag) rely on the # dockerhub autobuild feature diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index a238eb15fb..2811dea64f 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -1,13 +1,11 @@ name: Publish Docker image (master) -# The workflow will push images on every tag in the format x.y.z -# FIXME PR: change label and branch before merging into master +# The workflow will push images for master on every merge on: push: branches: - #- master - - feature/4004_jexl_expressions + - master jobs: build-and-push: @@ -27,9 +25,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TEF_TOKEN }} - name: Build Docker image - #run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . - run: docker build -t telefonicaiot/fiware-orion:jexl --build-arg GIT_REV_ORION=feature/4004_jexl_expressions --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . - name: Push Docker image - #run: docker push telefonicaiot/fiware-orion:latest - run: docker push telefonicaiot/fiware-orion:jexl + run: docker push telefonicaiot/fiware-orion:latest diff --git a/.github/workflows/publishimage-tag.yml b/.github/workflows/publishimage-tag.yml new file mode 100644 index 0000000000..6cd6b94b40 --- /dev/null +++ b/.github/workflows/publishimage-tag.yml @@ -0,0 +1,35 @@ +name: Publish Docker image (tag) + +# The workflow will push images on every tag in the format x.y.z + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_TEF_USERNAME }} + password: ${{ secrets.DOCKERHUB_TEF_TOKEN }} + + - name: Extract version from tag + id: extract_version + run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Build Docker image + run: docker build -t telefonicaiot/fiware-orion:${{ env.VERSION }} --build-arg GIT_REV_ORION=${{ env.VERSION }} --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + + - name: Push Docker image + run: docker push telefonicaiot/fiware-orion:${{ env.VERSION }} From 8e92831ea026fbdc4545fcf93fd0d889fd71d5ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 20 May 2024 17:35:44 +0200 Subject: [PATCH 253/390] Update test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test --- .../cases/4004_jexl_expressions_in_subs/jexl_null_values.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test index 35bfbe4567..2027736e1a 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test @@ -86,7 +86,7 @@ echo "03. Update entity E1 with A=null again (with forcedUpdate)" echo "==========================================================" payload='{ "A": { - "value": "null", + "value": null, "type": "Text" } }' From b3f56524632310b7b02c71b439430519aad350a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 05:27:51 +0000 Subject: [PATCH 254/390] --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test/acceptance/behave/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/behave/requirements.txt b/test/acceptance/behave/requirements.txt index d824bf71d0..503ddad1ba 100644 --- a/test/acceptance/behave/requirements.txt +++ b/test/acceptance/behave/requirements.txt @@ -3,7 +3,7 @@ argparse==1.3.0 behave==1.2.5 PyHamcrest==1.8.3 # Blindly update from 2.20.0 (test/acceptance is no longer in use) -requests==2.31.0 +requests==2.32.0 xmltodict==0.9.2 fabric==1.10.2 pymongo==4.6.3 # changed since 3.0.3 due to vulnerability check From 12c4574b842146f60792914205e34c4e3f827a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 May 2024 09:38:18 +0200 Subject: [PATCH 255/390] ADD jexl_transformation_full.test --- .../jexl_transformation_full.test | 365 ++++++++++++++++++ ...n.test => jexl_transformation_simple.test} | 0 2 files changed, 365 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test rename test/functionalTest/cases/4004_jexl_expressions_in_subs/{jexl_transformation.test => jexl_transformation_simple.test} (100%) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test new file mode 100644 index 0000000000..7508139a98 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -0,0 +1,365 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +JEXL expression in custom notification (using all transformations) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub using all transformations +# 02. Create entity E1 with A to P attributes +# 03. Dump accumulator and see expected notification +# + + +echo "01. Create custom sub using all transformations" +echo "===============================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "idPattern": ".*", + "type": "T" + } + ] + }, + "notification": { + "attrs": [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P" + ], + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "A": { + "type": "Text", + "value": "${A+(3|toString)}" + }, + "B": { + "type": "Text", + "value": "${B|replaceStr('\''A'\'','\''B'\'')}" + }, + "C": { + "type": "Number", + "value": "${C+1}" + }, + "D": { + "type": "Text", + "value": "${D|uppercase}" + }, + "E": { + "type": "TextUnrestricted", + "value": "${E|lowercase}" + }, + "F": { + "type": "Text", + "value": "${F[1]|trim}" + }, + "G": { + "type": "Number", + "value": "${G|toFixed(1)}" + }, + "H": { + "type": "Text", + "value": "${H|includes('\''N'\'')}" + }, + "I": { + "type": "Text", + "value": "${I|indexOf('\''test'\'')}" + }, + "J": { + "type": "Text", + "value": "${(J|isNaN)}" + }, + "K": { + "type": "Text", + "value": "${K|typeOf}" + }, + "L": { + "type": "TextUnrestricted", + "value": "${(L|split('\'' '\''))}" + }, + "M": { + "type": "Number", + "value": "${M|round(2)}" + }, + "N": { + "type": "Text", + "value": "${N|substring(1,2)}" + }, + "O": { + "type": "Text", + "value": "${Z||'\''Is null'\''}" + }, + "P": { + "type": "Number", + "value": "${P|len}" + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A to P attributes" +echo "===========================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "type": "Text", + "value": "NA" + }, + "B": { + "type": "Text", + "value": "NA" + }, + "C": { + "type": "Number", + "value": 0.17 + }, + "D": { + "type": "Text", + "value": "rice" + }, + "E": { + "type": "TextUnrestricted", + "value": "NA" + }, + "F": { + "type": "Json", + "value": [ + "Dolot sequitud", + " trimable" + ], + "metadata": {} + }, + "G": { + "type": "Number", + "value": 0.85 + }, + "H": { + "type": "Text", + "value": "NA" + }, + "I": { + "type": "Text", + "value": "Ipsum test", + "metadata": {} + }, + "J": { + "type": "Text", + "value": "NA" + }, + "K": { + "type": "Text", + "value": "NA" + }, + "L": { + "type": "Json", + "value": "Lorem limoso" + }, + "M": { + "type": "Number", + "value": 0.3 + }, + "N": { + "type": "Text", + "value": "NA" + }, + "O": { + "type": "Text", + "value": null + }, + "P": { + "type": "Text", + "value": "NA" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Dump accumulator and see expected notification" +echo "==================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub using all transformations +=============================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A to P attributes +=========================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Dump accumulator and see expected notification +================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 882 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Text", + "value": "NA3" + }, + "B": { + "metadata": {}, + "type": "Text", + "value": "NB" + }, + "C": { + "metadata": {}, + "type": "Number", + "value": 1.17 + }, + "D": { + "metadata": {}, + "type": "Text", + "value": "RICE" + }, + "E": { + "metadata": {}, + "type": "TextUnrestricted", + "value": "na" + }, + "F": { + "metadata": {}, + "type": "Text", + "value": "trimable" + }, + "G": { + "metadata": {}, + "type": "Number", + "value": 0.8 + }, + "H": { + "metadata": {}, + "type": "Text", + "value": true + }, + "I": { + "metadata": {}, + "type": "Text", + "value": 6 + }, + "J": { + "metadata": {}, + "type": "Text", + "value": true + }, + "K": { + "metadata": {}, + "type": "Text", + "value": "String" + }, + "L": { + "metadata": {}, + "type": "TextUnrestricted", + "value": [ + "Lorem", + "limoso" + ] + }, + "M": { + "metadata": {}, + "type": "Number", + "value": 0 + }, + "N": { + "metadata": {}, + "type": "Text", + "value": "A" + }, + "O": { + "metadata": {}, + "type": "Text", + "value": "Is null" + }, + "P": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test similarity index 100% rename from test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation.test rename to test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test From 62d1a22ede43c3eb015ebafcfd39bc7b8fffc6eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 21 May 2024 16:15:15 +0200 Subject: [PATCH 256/390] FIX remove FIXME PR mark --- src/lib/expressions/ExprResult.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/expressions/ExprResult.cpp b/src/lib/expressions/ExprResult.cpp index 5f34023565..eec2e415d6 100644 --- a/src/lib/expressions/ExprResult.cpp +++ b/src/lib/expressions/ExprResult.cpp @@ -273,8 +273,6 @@ std::string ExprResult::toString(void) } else if (valueType == orion::ValueTypeString) { - // FIXME PR: does this break the no legacy - //return "\"" + toJsonString(stringValue) + "\""; return "\"" + stringValue + "\""; } else if ((valueType == orion::ValueTypeObject)||(valueType == orion::ValueTypeVector)) From ecf2590e2e5e4460e0e4f5785ed8b361fe381285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 10:24:23 +0200 Subject: [PATCH 257/390] FIX CMAKE_MINIMUM_REQUIRED --- src/lib/alarmMgr/CMakeLists.txt | 2 +- src/lib/apiTypesV2/CMakeLists.txt | 2 +- src/lib/cache/CMakeLists.txt | 2 +- src/lib/common/CMakeLists.txt | 2 +- src/lib/convenience/CMakeLists.txt | 2 +- src/lib/jsonParse/CMakeLists.txt | 2 +- src/lib/jsonParseV2/CMakeLists.txt | 2 +- src/lib/logMsg/CMakeLists.txt | 2 +- src/lib/logSummary/CMakeLists.txt | 2 +- src/lib/metricsMgr/CMakeLists.txt | 2 +- src/lib/mongoBackend/CMakeLists.txt | 2 +- src/lib/mongoDriver/CMakeLists.txt | 2 +- src/lib/mqtt/CMakeLists.txt | 2 +- src/lib/ngsi/CMakeLists.txt | 2 +- src/lib/ngsi10/CMakeLists.txt | 2 +- src/lib/ngsi9/CMakeLists.txt | 2 +- src/lib/ngsiNotify/CMakeLists.txt | 2 +- src/lib/orionTypes/CMakeLists.txt | 2 +- src/lib/parse/CMakeLists.txt | 2 +- src/lib/rest/CMakeLists.txt | 2 +- src/lib/serviceRoutines/CMakeLists.txt | 2 +- src/lib/serviceRoutinesV2/CMakeLists.txt | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/lib/alarmMgr/CMakeLists.txt b/src/lib/alarmMgr/CMakeLists.txt index b75130768b..edda27d5f8 100644 --- a/src/lib/alarmMgr/CMakeLists.txt +++ b/src/lib/alarmMgr/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES AlarmManager.cpp diff --git a/src/lib/apiTypesV2/CMakeLists.txt b/src/lib/apiTypesV2/CMakeLists.txt index 6981627be3..18be8b1975 100644 --- a/src/lib/apiTypesV2/CMakeLists.txt +++ b/src/lib/apiTypesV2/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES Entity.cpp diff --git a/src/lib/cache/CMakeLists.txt b/src/lib/cache/CMakeLists.txt index dc708c0ab9..86854d8801 100644 --- a/src/lib/cache/CMakeLists.txt +++ b/src/lib/cache/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES subCache.cpp diff --git a/src/lib/common/CMakeLists.txt b/src/lib/common/CMakeLists.txt index ba5cfb9242..07ba47566f 100644 --- a/src/lib/common/CMakeLists.txt +++ b/src/lib/common/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES globals.cpp diff --git a/src/lib/convenience/CMakeLists.txt b/src/lib/convenience/CMakeLists.txt index 65f74232f0..65760b1253 100644 --- a/src/lib/convenience/CMakeLists.txt +++ b/src/lib/convenience/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES AppendContextElementRequest.cpp diff --git a/src/lib/jsonParse/CMakeLists.txt b/src/lib/jsonParse/CMakeLists.txt index 3e2ea75a24..1ea8952bdc 100644 --- a/src/lib/jsonParse/CMakeLists.txt +++ b/src/lib/jsonParse/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES jsonRequest.cpp diff --git a/src/lib/jsonParseV2/CMakeLists.txt b/src/lib/jsonParseV2/CMakeLists.txt index 42d4a1d5a9..04842e9361 100644 --- a/src/lib/jsonParseV2/CMakeLists.txt +++ b/src/lib/jsonParseV2/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES jsonRequestTreat.cpp diff --git a/src/lib/logMsg/CMakeLists.txt b/src/lib/logMsg/CMakeLists.txt index 3567c3444c..c41a9f0ad4 100644 --- a/src/lib/logMsg/CMakeLists.txt +++ b/src/lib/logMsg/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (HEADERS logMsg.h diff --git a/src/lib/logSummary/CMakeLists.txt b/src/lib/logSummary/CMakeLists.txt index 6413d6cb29..f1ba9ee62e 100644 --- a/src/lib/logSummary/CMakeLists.txt +++ b/src/lib/logSummary/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES logSummary.cpp diff --git a/src/lib/metricsMgr/CMakeLists.txt b/src/lib/metricsMgr/CMakeLists.txt index 5ccc5ef47c..aa22c37e64 100644 --- a/src/lib/metricsMgr/CMakeLists.txt +++ b/src/lib/metricsMgr/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES MetricsManager.cpp diff --git a/src/lib/mongoBackend/CMakeLists.txt b/src/lib/mongoBackend/CMakeLists.txt index 62a5c03127..25960522f5 100644 --- a/src/lib/mongoBackend/CMakeLists.txt +++ b/src/lib/mongoBackend/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES MongoGlobal.cpp diff --git a/src/lib/mongoDriver/CMakeLists.txt b/src/lib/mongoDriver/CMakeLists.txt index 386740d790..e5153dd283 100644 --- a/src/lib/mongoDriver/CMakeLists.txt +++ b/src/lib/mongoDriver/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES BSONTypes.cpp diff --git a/src/lib/mqtt/CMakeLists.txt b/src/lib/mqtt/CMakeLists.txt index 2cb4ededc0..4782ba0648 100644 --- a/src/lib/mqtt/CMakeLists.txt +++ b/src/lib/mqtt/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES MqttConnectionManager.cpp diff --git a/src/lib/ngsi/CMakeLists.txt b/src/lib/ngsi/CMakeLists.txt index a8e50634af..52a4d8bcf2 100644 --- a/src/lib/ngsi/CMakeLists.txt +++ b/src/lib/ngsi/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES AttributeExpression.cpp diff --git a/src/lib/ngsi10/CMakeLists.txt b/src/lib/ngsi10/CMakeLists.txt index 36e4aee8aa..4b18063d00 100644 --- a/src/lib/ngsi10/CMakeLists.txt +++ b/src/lib/ngsi10/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES NotifyContextRequest.cpp diff --git a/src/lib/ngsi9/CMakeLists.txt b/src/lib/ngsi9/CMakeLists.txt index 7f196b64cb..3d1d676c15 100644 --- a/src/lib/ngsi9/CMakeLists.txt +++ b/src/lib/ngsi9/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES RegisterContextRequest.cpp diff --git a/src/lib/ngsiNotify/CMakeLists.txt b/src/lib/ngsiNotify/CMakeLists.txt index 7d64a23001..704fd86b4e 100644 --- a/src/lib/ngsiNotify/CMakeLists.txt +++ b/src/lib/ngsiNotify/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES doNotify.cpp diff --git a/src/lib/orionTypes/CMakeLists.txt b/src/lib/orionTypes/CMakeLists.txt index 7a7e67d93f..27e16cb9d3 100644 --- a/src/lib/orionTypes/CMakeLists.txt +++ b/src/lib/orionTypes/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES areas.cpp diff --git a/src/lib/parse/CMakeLists.txt b/src/lib/parse/CMakeLists.txt index b609f9b5bf..0ab9b01611 100644 --- a/src/lib/parse/CMakeLists.txt +++ b/src/lib/parse/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES CompoundValueNode.cpp diff --git a/src/lib/rest/CMakeLists.txt b/src/lib/rest/CMakeLists.txt index 5a4ea98262..2b490bd3f9 100644 --- a/src/lib/rest/CMakeLists.txt +++ b/src/lib/rest/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES rest.cpp diff --git a/src/lib/serviceRoutines/CMakeLists.txt b/src/lib/serviceRoutines/CMakeLists.txt index f88cfcdc8f..9b9c8d6a4d 100644 --- a/src/lib/serviceRoutines/CMakeLists.txt +++ b/src/lib/serviceRoutines/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES versionTreat.cpp diff --git a/src/lib/serviceRoutinesV2/CMakeLists.txt b/src/lib/serviceRoutinesV2/CMakeLists.txt index c0cfaf7d38..a7b3c7c89d 100644 --- a/src/lib/serviceRoutinesV2/CMakeLists.txt +++ b/src/lib/serviceRoutinesV2/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES getEntities.cpp From 21d8f2868c183346cd515f9d6e888c647611c204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 11:38:02 +0200 Subject: [PATCH 258/390] FIX CMakeList.txt --- src/lib/expressions/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/expressions/CMakeLists.txt b/src/lib/expressions/CMakeLists.txt index 4f938e3634..27346743a0 100644 --- a/src/lib/expressions/CMakeLists.txt +++ b/src/lib/expressions/CMakeLists.txt @@ -18,7 +18,7 @@ # For those usages not covered by this license please contact with # iot_support at tid dot es -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 3.0) SET (SOURCES ExprManager.cpp From d5a14947f18893c5e25e25944e8b4661057d2ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 14:07:45 +0200 Subject: [PATCH 259/390] FIX conditional testing depending on expr flavour --- src/app/contextBroker/contextBroker.cpp | 6 +++ .../cases/0000_cli/version.test | 2 +- .../jexl_array_filtering.test | 1 + .../jexl_basic_attrs_in_update.test | 1 + .../jexl_basic_attrs_not_in_update.test | 1 + .../jexl_expr_attrs_weird_syntax.test | 1 + .../jexl_json_navigation.test | 1 + .../jexl_missing_attribute_as_null.test | 1 + .../jexl_missing_attribute_as_null2.test | 1 + .../jexl_not_defined_attrs.test | 1 + .../jexl_null_values.test | 1 + .../jexl_several_expressions.test | 1 + .../jexl_syntax_errors.test | 1 + .../jexl_transformation_full.test | 1 + .../jexl_transformation_in_id_and_type.test | 1 + .../jexl_transformation_simple.test | 1 + .../jexl_transformation_unknown.test | 1 + .../jexl_transformation_with_arguments.test | 1 + test/functionalTest/testHarness.sh | 50 +++++++++++++++++++ 19 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index ec994937fa..413c63a85b 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -1045,6 +1045,12 @@ int main(int argC, char* argV[]) std::string versionString = std::string(ORION_VERSION) + " (git version: " + GIT_HASH + ")"; + #ifdef EXPR_BASIC + versionString += " flavours: basic-expr"; + #else + versionString += " flavours: jexl-expr"; + #endif + paConfig("man synopsis", (void*) "[options]"); paConfig("man shortdescription", (void*) "Options:"); paConfig("man description", (void*) description); diff --git a/test/functionalTest/cases/0000_cli/version.test b/test/functionalTest/cases/0000_cli/version.test index bac1621329..8d034ea9f2 100644 --- a/test/functionalTest/cases/0000_cli/version.test +++ b/test/functionalTest/cases/0000_cli/version.test @@ -30,7 +30,7 @@ broker version contextBroker --version --REGEXPECT-- -REGEX(\d+\.\d+\.\d+.* \(git version: .*\)) +REGEX(\d+\.\d+\.\d+.* \(git version: .*\) flavours:.*) Copyright 2013-2024 Telefonica Investigacion y Desarrollo, S.A.U Orion Context Broker is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test index d6428e2074..7762f53a9a 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_array_filtering.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (array filtering) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test index 6e31d9b36e..3bc2121392 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_in_update.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL basic expression in custom notification (source attributes in udpate) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test index f6c634a077..7a284c23aa 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_basic_attrs_not_in_update.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL basic expression in custom notification (source attributes not in udpate) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test index 79ecfd9eeb..862f671a54 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_expr_attrs_weird_syntax.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression using attributes with weird syntax diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test index 05e8a0b061..f286e7f2c8 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_json_navigation.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (JSON navigation in objects and arrays) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test index 9d711fc6a7..d90db1f687 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (missing attribute replaced as null) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test index 8c0fd939f4..2d8d503914 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_missing_attribute_as_null2.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (missing attribute replaced as null additional case) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test index 4fe7de3997..3af9b1bdef 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_not_defined_attrs.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (not defined attributes) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test index 2027736e1a..070ab89ce8 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (null values) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test index 82ffb1ccaa..b1b06e070a 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_several_expressions.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (several expressions in same subscription) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test index 4359b235bc..0804b22c8d 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_syntax_errors.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification with syntax errors diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index 7508139a98..decc289255 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using all transformations) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test index fc128a4e4d..d1a0fca669 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_in_id_and_type.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using transformation in id and type) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test index 693818d970..05ffeb6471 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_simple.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using transformation) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test index 3d536559b4..c4ad0adf67 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_unknown.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using unknown transformation) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test index 5616763fde..2bc3422ef1 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_with_arguments.test @@ -19,6 +19,7 @@ # iot_support at tid dot es # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour --NAME-- JEXL expression in custom notification (using transformation with arguments) diff --git a/test/functionalTest/testHarness.sh b/test/functionalTest/testHarness.sh index c309c4e8c2..9b30488480 100755 --- a/test/functionalTest/testHarness.sh +++ b/test/functionalTest/testHarness.sh @@ -96,6 +96,8 @@ declare -A skipV typeset -i skips declare -A disabledTestV typeset -i disabledTests +declare -A notInFlavourTestV +typeset -i notInFlavourTests export DIFF=$SCRIPT_HOME/testDiff.py testError=0 @@ -104,6 +106,7 @@ okOnThird=0 okOnPlus3=0 skips=0 disabledTests=0 +notInFlavourTests=0 # ----------------------------------------------------------------------------- @@ -937,6 +940,30 @@ function testDisabled +# ----------------------------------------------------------------------------- +# +# testMatchExprFlavour +# +function testMatchExprFlavour +{ + testcase=$1 + + if grep -q JEXL_EXPR_FLAVOUR $testcase + then + if $(contextBroker --version | grep -q jexl-expr) + then + echo NOT Disabled + else + echo "Disabled" + echo "Disabled" > /tmp/valgrind.out + fi + else + echo NOT Disabled + fi +} + + + # ------------------------------------------------------------------------------ # # Main loop @@ -973,6 +1000,17 @@ do continue fi + # + # Should the test be skipped due to it doesn't mach in the contextBroker flavour? + # + notInFlavour=$(testMatchExprFlavour $testFile) + if [ "$notInFlavour" == "Disabled" ] + then + notInFlavourTestV[$notInFlavourTests]=$testNo': '$testFile + notInFlavourTests=$notInFlavourTests+1 + continue + fi + if [ "$ixList" != "" ] then hit=$(echo ' '$ixList' ' | grep ' '$testNo' ') @@ -1190,4 +1228,16 @@ then done fi +if [ $notInFlavourTests != 0 ] +then + echo + echo WARNING: $notInFlavourTests test cases were not executed due to contexBroker not matching flavour: + ix=0 + while [ $ix -lt $notInFlavourTests ] + do + echo " o " ${notInFlavourTestV[$ix]} + ix=$ix+1 + done +fi + exit $exitCode From 68425b99f648bcb61ab5c52fd3100ba71e7ba102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 15:06:42 +0200 Subject: [PATCH 260/390] FIX to make basic behaviour the same than jexl regarding null --- src/lib/expressions/ExprManager.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 47e492f223..c56587cdf3 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -94,8 +94,18 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st std::map::iterator iter = replacementsP->find(_expression); if (iter != replacementsP->end()) { - r.valueType = orion::ValueTypeString; r.stringValue = iter->second; + + // To have the same behaviour than in JEXL case + if (r.stringValue == "null") + { + r.valueType = orion::ValueTypeNull; + } + else + { + r.valueType = orion::ValueTypeString; + } + LM_T(LmtExpr, ("basic evaluation result: <%s>", r.stringValue.c_str())); } TIME_EXPR_BASIC_EVAL_STOP(); From 11f66eec966de0eb867ff12ff77c4f74da37caca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 16:22:25 +0200 Subject: [PATCH 261/390] ADD noLibVersions option for version operation --- CHANGES_NEXT_RELEASE | 3 +- src/lib/common/globals.h | 1 + src/lib/rest/ConnectionInfo.cpp | 1 + src/lib/serviceRoutines/versionTreat.cpp | 13 +++++- .../version_via_rest.test | 14 +------ .../cases/0501_cors/version_request.test | 42 +++---------------- .../fiware_correlator.test | 28 ++----------- 7 files changed, 27 insertions(+), 75 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 25236adf1a..55711ac133 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ -- Fix: lighter operation to get databases list from MongoDB (#4517) \ No newline at end of file +- Add: GET /version?options=noLibVersions to simplify the response avoding the "libversions" field +- Fix: lighter operation to get databases list from MongoDB (#4517) diff --git a/src/lib/common/globals.h b/src/lib/common/globals.h index 7fae71855c..29d115c643 100644 --- a/src/lib/common/globals.h +++ b/src/lib/common/globals.h @@ -128,6 +128,7 @@ #define OPT_OVERRIDEMETADATA "overrideMetadata" #define OPT_SKIPFORWARDING "skipForwarding" #define OPT_FULL_COUNTERS "fullCounters" +#define OPT_NO_LIB_VERSIONS "noLibVersions" // used in GET /version operation /* **************************************************************************** diff --git a/src/lib/rest/ConnectionInfo.cpp b/src/lib/rest/ConnectionInfo.cpp index c5a94d30d2..2c4f8c550c 100644 --- a/src/lib/rest/ConnectionInfo.cpp +++ b/src/lib/rest/ConnectionInfo.cpp @@ -54,6 +54,7 @@ static const char* validOptions[] = OPT_FLOW_CONTROL, OPT_SKIPFORWARDING, OPT_FULL_COUNTERS, + OPT_NO_LIB_VERSIONS, // FIXME P3: initial notification feature was removed in CB 3.2.0, but we leave this // here for a while, so existing clients using options=skipInitialNotification don't break diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index 981c81f2b6..80f926dcc5 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -132,6 +132,8 @@ std::string versionTreat ParseData* parseDataP ) { + bool noLibVersions = ciP->uriParamOptions[OPT_NO_LIB_VERSIONS]; + if (isOriginAllowedForCORS(ciP->httpHeaders.origin)) { ciP->httpHeader.push_back(HTTP_ACCESS_CONTROL_ALLOW_ORIGIN); @@ -168,8 +170,15 @@ std::string versionTreat out += " \"compiled_in\" : \"" + std::string(COMPILED_IN) + "\",\n"; out += " \"release_date\" : \"" + std::string(RELEASE_DATE) + "\",\n"; out += " \"machine\" : \"" + std::string(MACHINE_ARCH) + "\",\n"; - out += " \"doc\" : \"" + std::string(API_DOC) + "\"," "\n" + " " + libVersions(); - out += " }\n"; + if (noLibVersions) + { + out += " \"doc\" : \"" + std::string(API_DOC) + "\"\n"; + } + else + { + out += " \"doc\" : \"" + std::string(API_DOC) + "\"," "\n" + " " + libVersions(); + out += " }\n"; + } out += "}\n"; out += "}\n"; diff --git a/test/functionalTest/cases/0000_version_operation/version_via_rest.test b/test/functionalTest/cases/0000_version_operation/version_via_rest.test index ddd58071be..76f8c1ac2a 100644 --- a/test/functionalTest/cases/0000_version_operation/version_via_rest.test +++ b/test/functionalTest/cases/0000_version_operation/version_via_rest.test @@ -34,7 +34,7 @@ brokerStart CB # echo "01. Getting version via REST" echo "============================" -orionCurl --url "/version" --noPayloadCheck +orionCurl --url "/version?options=noLibVersions" --noPayloadCheck echo echo @@ -58,17 +58,7 @@ Content-Length: REGEX(\d+) "compiled_in" : "REGEX(.*)", "release_date" : "REGEX(.*)", "machine" : "REGEX(.*)", - "doc" : "REGEX(.*)", - "libversions": { - "boost": "REGEX(.*)", - "libcurl": "REGEX(.*)", - "libmosquitto": "REGEX(.*)", - "libmicrohttpd": "REGEX(.*)", - "openssl": "REGEX(.*)", - "rapidjson": "REGEX(.*)", - "mongoc": "REGEX(.*)", - "bson": "REGEX(.*)" - } + "doc" : "REGEX(.*)" } } diff --git a/test/functionalTest/cases/0501_cors/version_request.test b/test/functionalTest/cases/0501_cors/version_request.test index 2cdcafe8a1..b94e02495f 100644 --- a/test/functionalTest/cases/0501_cors/version_request.test +++ b/test/functionalTest/cases/0501_cors/version_request.test @@ -35,19 +35,19 @@ brokerStart CB 0-255 IPV4 -corsOrigin arrakis --SHELL-- echo "01. GET Version (Access-Control-Allow-Origin header not included)" echo "=================================================================" -orionCurl --url /version --noPayloadCheck +orionCurl --url /version?options=noLibVersions --noPayloadCheck echo echo echo "02. GET Version with arrakis origin (Access-Control-Allow-Origin header included)" echo "=================================================================================" -orionCurl --url /version -X GET --origin arrakis --noPayloadCheck +orionCurl --url /version?options=noLibVersions -X GET --origin arrakis --noPayloadCheck echo echo echo "03. GET Version with caladan origin (Access-Control-Allow-Origin header not included)" echo "=====================================================================================" -orionCurl --url /version -X GET --origin caladan --noPayloadCheck +orionCurl --url /version?options=noLibVersions -X GET --origin caladan --noPayloadCheck echo echo @@ -88,17 +88,7 @@ Content-Length: REGEX(\d+) "compiled_in" : "REGEX(.*)", "release_date" : "REGEX(.*)", "machine" : "REGEX(.*)", - "doc" : "REGEX(.*)", - "libversions": { - "boost": "REGEX(.*)", - "libcurl": "REGEX(.*)", - "libmosquitto": "REGEX(.*)", - "libmicrohttpd": "REGEX(.*)", - "openssl": "REGEX(.*)", - "rapidjson": "REGEX(.*)", - "mongoc": "REGEX(.*)", - "bson": "REGEX(.*)" - } + "doc" : "REGEX(.*)" } } @@ -124,17 +114,7 @@ Content-Length: REGEX(\d+) "compiled_in" : "REGEX(.*)", "release_date" : "REGEX(.*)", "machine" : "REGEX(.*)", - "doc" : "REGEX(.*)", - "libversions": { - "boost": "REGEX(.*)", - "libcurl": "REGEX(.*)", - "libmosquitto": "REGEX(.*)", - "libmicrohttpd": "REGEX(.*)", - "openssl": "REGEX(.*)", - "rapidjson": "REGEX(.*)", - "mongoc": "REGEX(.*)", - "bson": "REGEX(.*)" - } + "doc" : "REGEX(.*)" } } @@ -158,17 +138,7 @@ Content-Length: REGEX(\d+) "compiled_in" : "REGEX(.*)", "release_date" : "REGEX(.*)", "machine" : "REGEX(.*)", - "doc" : "REGEX(.*)", - "libversions": { - "boost": "REGEX(.*)", - "libcurl": "REGEX(.*)", - "libmosquitto": "REGEX(.*)", - "libmicrohttpd": "REGEX(.*)", - "openssl": "REGEX(.*)", - "rapidjson": "REGEX(.*)", - "mongoc": "REGEX(.*)", - "bson": "REGEX(.*)" - } + "doc" : "REGEX(.*)" } } diff --git a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test index 5e52e7eea5..9df7b7d6ca 100644 --- a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test +++ b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test @@ -36,14 +36,14 @@ brokerStart CB 0 echo "01. version request without correlator, see new correlator in response" echo "======================================================================" -orionCurl --url /version --noPayloadCheck +orionCurl --url /version?options=noLibVersions --noPayloadCheck echo echo echo "02. version request with correlator, see same correlator in response" echo "====================================================================" -orionCurl --url /version --noPayloadCheck --header "Fiware-Correlator: OLD_CORRELATOR" +orionCurl --url /version?options=noLibVersions --noPayloadCheck --header "Fiware-Correlator: OLD_CORRELATOR" echo echo @@ -67,17 +67,7 @@ Content-Length: REGEX(\d+) "compiled_in" : "REGEX(.*)", "release_date" : "REGEX(.*)", "machine" : "REGEX(.*)", - "doc" : "REGEX(.*)", - "libversions": { - "boost": "REGEX(.*)", - "libcurl": "REGEX(.*)", - "libmosquitto": "REGEX(.*)", - "libmicrohttpd": "REGEX(.*)", - "openssl": "REGEX(.*)", - "rapidjson": "REGEX(.*)", - "mongoc": "REGEX(.*)", - "bson": "REGEX(.*)" - } + "doc" : "REGEX(.*)" } } @@ -101,17 +91,7 @@ Content-Length: REGEX(\d+) "compiled_in" : "REGEX(.*)", "release_date" : "REGEX(.*)", "machine" : "REGEX(.*)", - "doc" : "REGEX(.*)", - "libversions": { - "boost": "REGEX(.*)", - "libcurl": "REGEX(.*)", - "libmosquitto": "REGEX(.*)", - "libmicrohttpd": "REGEX(.*)", - "openssl": "REGEX(.*)", - "rapidjson": "REGEX(.*)", - "mongoc": "REGEX(.*)", - "bson": "REGEX(.*)" - } + "doc" : "REGEX(.*)" } } From 77532147819dfd976206c657b23ee5d8b1bfc43e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 18:33:32 +0200 Subject: [PATCH 262/390] FIX comment raised in code review --- CHANGES_NEXT_RELEASE | 2 +- src/lib/common/globals.h | 2 +- src/lib/rest/ConnectionInfo.cpp | 2 +- src/lib/serviceRoutines/versionTreat.cpp | 10 +++++----- .../cases/0000_version_operation/version_via_rest.test | 2 +- .../cases/0501_cors/version_request.test | 6 +++--- .../1916_fiware_correlator/fiware_correlator.test | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 55711ac133..e8ee65ca95 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Add: GET /version?options=noLibVersions to simplify the response avoding the "libversions" field +- Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) diff --git a/src/lib/common/globals.h b/src/lib/common/globals.h index 29d115c643..4533e0f5f1 100644 --- a/src/lib/common/globals.h +++ b/src/lib/common/globals.h @@ -128,7 +128,7 @@ #define OPT_OVERRIDEMETADATA "overrideMetadata" #define OPT_SKIPFORWARDING "skipForwarding" #define OPT_FULL_COUNTERS "fullCounters" -#define OPT_NO_LIB_VERSIONS "noLibVersions" // used in GET /version operation +#define OPT_LIB_VERSIONS "libVersions" // used in GET /version operation /* **************************************************************************** diff --git a/src/lib/rest/ConnectionInfo.cpp b/src/lib/rest/ConnectionInfo.cpp index 2c4f8c550c..1fd902aa18 100644 --- a/src/lib/rest/ConnectionInfo.cpp +++ b/src/lib/rest/ConnectionInfo.cpp @@ -54,7 +54,7 @@ static const char* validOptions[] = OPT_FLOW_CONTROL, OPT_SKIPFORWARDING, OPT_FULL_COUNTERS, - OPT_NO_LIB_VERSIONS, + OPT_LIB_VERSIONS, // FIXME P3: initial notification feature was removed in CB 3.2.0, but we leave this // here for a while, so existing clients using options=skipInitialNotification don't break diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index 80f926dcc5..f4b070f378 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -132,7 +132,7 @@ std::string versionTreat ParseData* parseDataP ) { - bool noLibVersions = ciP->uriParamOptions[OPT_NO_LIB_VERSIONS]; + bool showLibVersions = ciP->uriParamOptions[OPT_LIB_VERSIONS]; if (isOriginAllowedForCORS(ciP->httpHeaders.origin)) { @@ -170,14 +170,14 @@ std::string versionTreat out += " \"compiled_in\" : \"" + std::string(COMPILED_IN) + "\",\n"; out += " \"release_date\" : \"" + std::string(RELEASE_DATE) + "\",\n"; out += " \"machine\" : \"" + std::string(MACHINE_ARCH) + "\",\n"; - if (noLibVersions) + if (showLibVersions) { - out += " \"doc\" : \"" + std::string(API_DOC) + "\"\n"; + out += " \"doc\" : \"" + std::string(API_DOC) + "\"," "\n" + " " + libVersions(); + out += " }\n"; } else { - out += " \"doc\" : \"" + std::string(API_DOC) + "\"," "\n" + " " + libVersions(); - out += " }\n"; + out += " \"doc\" : \"" + std::string(API_DOC) + "\"\n"; } out += "}\n"; out += "}\n"; diff --git a/test/functionalTest/cases/0000_version_operation/version_via_rest.test b/test/functionalTest/cases/0000_version_operation/version_via_rest.test index 76f8c1ac2a..65e7e43204 100644 --- a/test/functionalTest/cases/0000_version_operation/version_via_rest.test +++ b/test/functionalTest/cases/0000_version_operation/version_via_rest.test @@ -34,7 +34,7 @@ brokerStart CB # echo "01. Getting version via REST" echo "============================" -orionCurl --url "/version?options=noLibVersions" --noPayloadCheck +orionCurl --url "/version" --noPayloadCheck echo echo diff --git a/test/functionalTest/cases/0501_cors/version_request.test b/test/functionalTest/cases/0501_cors/version_request.test index b94e02495f..c464672650 100644 --- a/test/functionalTest/cases/0501_cors/version_request.test +++ b/test/functionalTest/cases/0501_cors/version_request.test @@ -35,19 +35,19 @@ brokerStart CB 0-255 IPV4 -corsOrigin arrakis --SHELL-- echo "01. GET Version (Access-Control-Allow-Origin header not included)" echo "=================================================================" -orionCurl --url /version?options=noLibVersions --noPayloadCheck +orionCurl --url /version --noPayloadCheck echo echo echo "02. GET Version with arrakis origin (Access-Control-Allow-Origin header included)" echo "=================================================================================" -orionCurl --url /version?options=noLibVersions -X GET --origin arrakis --noPayloadCheck +orionCurl --url /version -X GET --origin arrakis --noPayloadCheck echo echo echo "03. GET Version with caladan origin (Access-Control-Allow-Origin header not included)" echo "=====================================================================================" -orionCurl --url /version?options=noLibVersions -X GET --origin caladan --noPayloadCheck +orionCurl --url /version -X GET --origin caladan --noPayloadCheck echo echo diff --git a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test index 9df7b7d6ca..57e300b66c 100644 --- a/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test +++ b/test/functionalTest/cases/1916_fiware_correlator/fiware_correlator.test @@ -36,14 +36,14 @@ brokerStart CB 0 echo "01. version request without correlator, see new correlator in response" echo "======================================================================" -orionCurl --url /version?options=noLibVersions --noPayloadCheck +orionCurl --url /version --noPayloadCheck echo echo echo "02. version request with correlator, see same correlator in response" echo "====================================================================" -orionCurl --url /version?options=noLibVersions --noPayloadCheck --header "Fiware-Correlator: OLD_CORRELATOR" +orionCurl --url /version --noPayloadCheck --header "Fiware-Correlator: OLD_CORRELATOR" echo echo From 30058fda1ee69cadc7e7a10580d2aa5dcd42af94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 22 May 2024 18:43:09 +0200 Subject: [PATCH 263/390] FIX unit test --- test/unittests/serviceRoutines/versionTreat_test.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/unittests/serviceRoutines/versionTreat_test.cpp b/test/unittests/serviceRoutines/versionTreat_test.cpp index 9950e5d6f0..71eded8972 100644 --- a/test/unittests/serviceRoutines/versionTreat_test.cpp +++ b/test/unittests/serviceRoutines/versionTreat_test.cpp @@ -73,7 +73,6 @@ TEST(versionTreat, ok) // " \"compiled_in\" : \".*\"\n" // " \"machine\" : \".*\"\n" // " \"doc\" : \".*\"\n" - // " \"libversions\" : (drill down) "\n" // "}\n" // "}\n"; // bool match; @@ -89,15 +88,6 @@ TEST(versionTreat, ok) EXPECT_TRUE(strstr(out.c_str(), "release_date") != NULL); EXPECT_TRUE(strstr(out.c_str(), "machine") != NULL); EXPECT_TRUE(strstr(out.c_str(), "doc") != NULL); - EXPECT_TRUE(strstr(out.c_str(), "libversions") != NULL); - - EXPECT_TRUE(strstr(out.c_str(), "boost") != NULL); - EXPECT_TRUE(strstr(out.c_str(), "libcurl") != NULL); - EXPECT_TRUE(strstr(out.c_str(), "libmicrohttpd") != NULL); - EXPECT_TRUE(strstr(out.c_str(), "openssl") != NULL); - EXPECT_TRUE(strstr(out.c_str(), "rapidjson") != NULL); - EXPECT_TRUE(strstr(out.c_str(), "mongoc") != NULL); - EXPECT_TRUE(strstr(out.c_str(), "bson") != NULL); extern const char* orionUnitTestVersion; std::string expected = std::string("\"version\" : \"") + orionUnitTestVersion + "\""; From 42c3a9ac377e7753995cdbd87eec76ce5647c2e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 May 2024 17:49:16 +0200 Subject: [PATCH 264/390] ADD more test for null values cases --- .../jexl_null_values_all_cases.test | 269 ++++++++++++++ ...exl_null_values_all_cases_covered_sub.test | 276 ++++++++++++++ .../null_values_all_cases.test | 340 +++++++++++++++++ .../null_values_all_cases_covered_sub.test | 347 ++++++++++++++++++ 4 files changed, 1232 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test new file mode 100644 index 0000000000..f40c7e825a --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test @@ -0,0 +1,269 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +Expression in custom notification (null values in all cases with JEXL expression fail) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create custom sub using payload: "${A|patata}" +# 02. Create custom sub using payload: "X:${A|patata}:Y" +# 03. Create custom sub using json: "${A|patata}" and "X:${A|patata}:Y" +# 04. Create custom sub using ngsi: "${A|patata}" and "X:${A|patata}:Y" +# 05. Create entity E1 with A=1 +# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# + + +echo '01. Create custom sub using payload: "${A|patata}"' +echo "==================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "${A|patata}" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '02. Create custom sub using payload: "X:${A|patata}:Y"' +echo "======================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "X:${A|patata}:Y" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Create custom sub using json: "${A|patata}" and "X:${A|patata}:Y"' +echo "=====================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "json": { + "B1": "${A|patata}", + "B2": "X:${A|patata}:Y" + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '04. Create custom sub with using ngsi: "${A|patata}" and "X:${A|patata}:Y"' +echo "==========================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B1": { + "value": "${A|patata}", + "type": "Calculated" + }, + "B2": { + "value": "X:${A|patata}:Y", + "type": "Calculated" + } + } + }, + "attrs": [ "B1", "B2" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" +echo "===========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create custom sub using payload: "${A|patata}" +================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create custom sub using payload: "X:${A|patata}:Y" +====================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom sub using json: "${A|patata}" and "X:${A|patata}:Y" +===================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create custom sub with using ngsi: "${A|patata}" and "X:${A|patata}:Y" +========================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test new file mode 100644 index 0000000000..0205830e13 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test @@ -0,0 +1,276 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +Expression in custom notification (null values in all cases with JEXL expression fail with covered subs) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create custom sub with covered using payload: "${A|patata}" +# 02. Create custom sub with covered using payload: "X:${A|patata}:Y" +# 03. Create custom sub with covered using json: "${A|patata}" and "X:${A|patata}:Y" +# 04. Create custom sub with covered using ngsi: "${A|patata}" and "X:${A|patata}:Y" +# 05. Create entity E1 with A=1 +# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# + + +echo '01. Create custom sub with covered using payload: "${A|patata}"' +echo "===============================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "${A|patata}" + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '02. Create custom sub with covered using payload: "X:${A|patata}:Y"' +echo "===================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "X:${A|patata}:Y" + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Create custom sub with covered using json: "${A|patata}" and "X:${A|patata}:Y"' +echo "==================================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "json": { + "B1": "${A|patata}", + "B2": "X:${A|patata}:Y" + } + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '04. Create custom sub with covered using ngsi: "${A|patata}" and "X:${A|patata}:Y"' +echo "==================================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B1": { + "value": "${A|patata}", + "type": "Calculated" + }, + "B2": { + "value": "X:${A|patata}:Y", + "type": "Calculated" + } + } + }, + "attrs": [ "B1", "B2" ], + "covered": true + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" +echo "===========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create custom sub with covered using payload: "${A|patata}" +=============================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create custom sub with covered using payload: "X:${A|patata}:Y" +=================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom sub with covered using json: "${A|patata}" and "X:${A|patata}:Y" +================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create custom sub with covered using ngsi: "${A|patata}" and "X:${A|patata}:Y" +================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test new file mode 100644 index 0000000000..85a6674b15 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test @@ -0,0 +1,340 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Expression in custom notification (null values in all cases) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create custom sub using payload: "${A}" +# 02. Create custom sub using payload: "X:${A}:Y" +# 03. Create custom sub using json: "${A}" and "X:${A}:Y" +# 04. Create custom sub using ngsi: "${A}" and "X:${A}:Y" +# 05. Create entity E1 with A=null +# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 07. Create entity E2 without A +# 08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# + + +echo '01. Create custom sub using payload: "${A}"' +echo "===========================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "${A}" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '02. Create custom sub using payload: "X:${A}:Y"' +echo "===============================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "X:${A}:Y" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Create custom sub using json: "${A}" and "X:${A}:Y"' +echo "=======================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "json": { + "B1": "${A}", + "B2": "X:${A}:Y" + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '04. Create custom sub with using ngsi: "${A}" and "X:${A}:Y"' +echo "============================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B1": { + "value": "${A}", + "type": "Calculated" + }, + "B2": { + "value": "X:${A}:Y", + "type": "Calculated" + } + } + }, + "attrs": [ "B1", "B2" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create entity E1 with A=null" +echo "================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" +echo "===========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +echo "07. Create entity E2 without A" +echo "==============================" +payload='{ + "id": "E2", + "type": "T", + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" +echo "===========================================================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create custom sub using payload: "${A}" +=========================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create custom sub using payload: "X:${A}:Y" +=============================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom sub using json: "${A}" and "X:${A}:Y" +======================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create custom sub with using ngsi: "${A}" and "X:${A}:Y" +============================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create entity E1 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +07. Create entity E2 without A +============================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +=========================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E2","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test new file mode 100644 index 0000000000..97d28912a4 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test @@ -0,0 +1,347 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Expression in custom notification (null values in all cases with covered subs) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart + +--SHELL-- + +# +# 01. Create custom sub with covered using payload: "${A}" +# 02. Create custom sub with covered using payload: "X:${A}:Y" +# 03. Create custom sub with covered using json: "${A}" and "X:${A}:Y" +# 04. Create custom sub with covered using ngsi: "${A}" and "X:${A}:Y" +# 05. Create entity E1 with A=null +# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 07. Create entity E2 without A +# 08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# + + +echo '01. Create custom sub with covered using payload: "${A}"' +echo "========================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "${A}" + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '02. Create custom sub with covered using payload: "X:${A}:Y"' +echo "============================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "headers": { + "content-type": "text/plain" + }, + "payload": "X:${A}:Y" + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '03. Create custom sub with covered using json: "${A}" and "X:${A}:Y"' +echo "====================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "json": { + "B1": "${A}", + "B2": "X:${A}:Y" + } + }, + "covered": true, + "attrs": [ "A" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo '04. Create custom sub with covered using json: "${A}" and "X:${A}:Y"' +echo "====================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern" : ".*", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "B1": { + "value": "${A}", + "type": "Calculated" + }, + "B2": { + "value": "X:${A}:Y", + "type": "Calculated" + } + } + }, + "attrs": [ "B1", "B2" ], + "covered": true + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create entity E1 with A=null" +echo "================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": null, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "06. Dump & reset accumulator and see notifications" +echo "==================================================" +accumulatorDump +accumulatorReset +echo +echo + + +echo "07. Create entity E2 without A" +echo "==============================" +payload='{ + "id": "E2", + "type": "T", + "B": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "08. Dump & reset accumulator and see notifications" +echo "==================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Create custom sub with covered using payload: "${A}" +======================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create custom sub with covered using payload: "X:${A}:Y" +============================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create custom sub with covered using json: "${A}" and "X:${A}:Y" +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create custom sub with covered using json: "${A}" and "X:${A}:Y" +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create entity E1 with A=null +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +06. Dump & reset accumulator and see notifications +================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +07. Create entity E2 without A +============================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E2?type=T +Content-Length: 0 + + + +08. Dump & reset accumulator and see notifications +================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 4 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X::Y======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 27 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"B1":null,"B2":"X:null:Y"}======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 191 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +{"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E2","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From fcb9304217302e720a4a3b295c03b562fba22deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 09:45:28 +0200 Subject: [PATCH 265/390] FIX null behaviour in custom notification payload field --- src/lib/ngsiNotify/Notifier.cpp | 2 +- ...fication_templates_many_notifications.test | 20 ++++---- .../covered_custom_notification.test | 18 +++---- .../jexl_null_values_all_cases.test | 25 +++++++--- ...exl_null_values_all_cases_covered_sub.test | 25 +++++++--- .../null_values_all_cases.test | 50 ++++++++++++++----- .../null_values_all_cases_covered_sub.test | 50 ++++++++++++++----- 7 files changed, 134 insertions(+), 56 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index df16cbb610..e26fc31e37 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -170,7 +170,7 @@ static bool setPayload } else { - if (!macroSubstitute(payloadP, notifPayload, exprContextObjectP, "", true)) + if (!macroSubstitute(payloadP, notifPayload, exprContextObjectP, "null", true)) { return false; } diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test index 341c55b7b9..4ffa8c252e 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates_many_notifications.test @@ -280,7 +280,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) PUT http://127.0.0.1:REGEX(\d+)/notify?a1=true&id=E1&type=T1 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 44 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -292,8 +292,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { "A1": "true", - "A2": "", - "A3": "" + "A2": "null", + "A3": "null" } #SORT_END #SORT_START @@ -351,7 +351,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) PUT http://127.0.0.1:REGEX(\d+)/notify?id=E1&type=T2 Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 32 +Content-Length: 44 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -361,9 +361,9 @@ Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "", - "A2": "", - "A3": "" + "A1": "null", + "A2": "null", + "A3": "null" } #SORT_END #SORT_START @@ -422,7 +422,7 @@ PUT http://127.0.0.1:REGEX(\d+)/notify?a3=3&id=E1&type=T3 Fiware-Servicepath: / A3: 3 Entity-Id: E1 -Content-Length: 33 +Content-Length: 41 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -432,8 +432,8 @@ Content-Type: text/plain; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[123]) { - "A1": "", - "A2": "", + "A1": "null", + "A2": "null", "A3": "3" } ======================================= diff --git a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test index 6d40e8e28b..14fffa6e0f 100644 --- a/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test +++ b/test/functionalTest/cases/3693_covered_notifications_in_subscriptions/covered_custom_notification.test @@ -36,7 +36,7 @@ accumulatorStart # 03. Create E1 with attribute A1=1 # 04. Dump & reset, see notifications A1=1 and A2=null in custom payload # 05. Create E2 with attribute A1=1 -# 06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) +# 06. Dump & reset, see notifications A1=1 and A2=null in custom payload # echo "01. Create covered custom subscriptions for E1 covering attributes A1 and A2" @@ -131,8 +131,8 @@ echo echo -echo "06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON)" -echo "========================================================================================" +echo "06. Dump & reset, see notifications A1=1 and A2=null in custom payload" +echo "======================================================================" accumulatorDump accumulatorReset echo @@ -174,7 +174,7 @@ Content-Length: 0 ====================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 31 +Content-Length: 35 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -182,7 +182,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": }======================================= +{ "A1-value": 1, "A2-value": null }======================================= 05. Create E2 with attribute A1=1 @@ -195,11 +195,11 @@ Content-Length: 0 -06. Dump & reset, see notifications A1=1 and A2= in custom payload (invalid JSON) -======================================================================================== +06. Dump & reset, see notifications A1=1 and A2=null in custom payload +====================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 31 +Content-Length: 35 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: custom Host: 127.0.0.1:REGEX(\d+) @@ -207,7 +207,7 @@ Accept: application/json Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 -{ "A1-value": 1, "A2-value": }======================================= +{ "A1-value": 1, "A2-value": null }======================================= --TEARDOWN-- diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test index f40c7e825a..aba513b32e 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases.test @@ -37,7 +37,7 @@ accumulatorStart # 03. Create custom sub using json: "${A|patata}" and "X:${A|patata}:Y" # 04. Create custom sub using ngsi: "${A|patata}" and "X:${A|patata}:Y" # 05. Create entity E1 with A=1 -# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 06. Dump & reset accumulator and see 4 notifications # @@ -167,8 +167,8 @@ echo echo -echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" -echo "===========================================================================================" +echo "06. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -226,8 +226,9 @@ Content-Length: 0 -06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) -=========================================================================================== +06. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -238,7 +239,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -261,6 +273,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END --TEARDOWN-- diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test index 0205830e13..6cf375ceae 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_null_values_all_cases_covered_sub.test @@ -37,7 +37,7 @@ accumulatorStart # 03. Create custom sub with covered using json: "${A|patata}" and "X:${A|patata}:Y" # 04. Create custom sub with covered using ngsi: "${A|patata}" and "X:${A|patata}:Y" # 05. Create entity E1 with A=1 -# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 06. Dump & reset accumulator and see 4 notifications # @@ -174,8 +174,8 @@ echo echo -echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" -echo "===========================================================================================" +echo "06. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -233,8 +233,9 @@ Content-Length: 0 -06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) -=========================================================================================== +06. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -245,7 +246,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -268,6 +280,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END --TEARDOWN-- diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test index 85a6674b15..3c7fe0ad03 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases.test @@ -36,9 +36,9 @@ accumulatorStart # 03. Create custom sub using json: "${A}" and "X:${A}:Y" # 04. Create custom sub using ngsi: "${A}" and "X:${A}:Y" # 05. Create entity E1 with A=null -# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 06. Dump & reset accumulator and see 4 notifications # 07. Create entity E2 without A -# 08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 08. Dump & reset accumulator and see 4 notifications # @@ -168,8 +168,8 @@ echo echo -echo "06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" -echo "===========================================================================================" +echo "06. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -191,8 +191,8 @@ echo echo -echo "08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification)" -echo "===========================================================================================" +echo "08. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -250,8 +250,9 @@ Content-Length: 0 -06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) -=========================================================================================== +06. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -262,7 +263,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -285,6 +297,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END 07. Create entity E2 without A @@ -297,8 +310,9 @@ Content-Length: 0 -08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) -=========================================================================================== +08. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -309,7 +323,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -332,6 +357,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E2","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END --TEARDOWN-- diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test index 97d28912a4..562fb86085 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/null_values_all_cases_covered_sub.test @@ -36,9 +36,9 @@ accumulatorStart # 03. Create custom sub with covered using json: "${A}" and "X:${A}:Y" # 04. Create custom sub with covered using ngsi: "${A}" and "X:${A}:Y" # 05. Create entity E1 with A=null -# 06. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 06. Dump & reset accumulator and see 4 notifications # 07. Create entity E2 without A -# 08. Dump & reset accumulator and see 3 notifications (first sub lead to empty notification) +# 08. Dump & reset accumulator and see 4 notifications # @@ -175,8 +175,8 @@ echo echo -echo "06. Dump & reset accumulator and see notifications" -echo "==================================================" +echo "06. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -198,8 +198,8 @@ echo echo -echo "08. Dump & reset accumulator and see notifications" -echo "==================================================" +echo "08. Dump & reset accumulator and see 4 notifications" +echo "====================================================" accumulatorDump accumulatorReset echo @@ -257,8 +257,9 @@ Content-Length: 0 -06. Dump & reset accumulator and see notifications -================================================== +06. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -269,7 +270,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -292,6 +304,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E1","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END 07. Create entity E2 without A @@ -304,8 +317,9 @@ Content-Length: 0 -08. Dump & reset accumulator and see notifications -================================================== +08. Dump & reset accumulator and see 4 notifications +==================================================== +#SORT_START POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 4 @@ -316,7 +330,18 @@ Accept: application/json Content-Type: text/plain Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) -X::Y======================================= +null======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 8 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: custom +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: text/plain +Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) + +X:null:Y======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 27 @@ -339,6 +364,7 @@ Content-Type: application/json; charset=utf-8 Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234]) {"subscriptionId":"REGEX([0-9a-f]{24})","data":[{"id":"E2","type":"T","B1":{"type":"Calculated","value":null,"metadata":{}},"B2":{"type":"Calculated","value":"X:null:Y","metadata":{}}}]}======================================= +#SORT_END --TEARDOWN-- From ac2f915253ac5be4ef9638aad7ce1914a0e3ae7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 10:14:25 +0200 Subject: [PATCH 266/390] FIX ftest --- .../2015_notification_templates/notification_templates.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functionalTest/cases/2015_notification_templates/notification_templates.test b/test/functionalTest/cases/2015_notification_templates/notification_templates.test index 23df76a2fe..705c99c8bc 100644 --- a/test/functionalTest/cases/2015_notification_templates/notification_templates.test +++ b/test/functionalTest/cases/2015_notification_templates/notification_templates.test @@ -539,7 +539,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) POST http://127.0.0.1:REGEX(\d+)/notify?a1=a1&a2=a2&id=E1&step=07&type=Thing Fiware-Servicepath: / Entity-Id: E1 -Content-Length: 36 +Content-Length: 40 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Step: 07 Ngsiv2-Attrsformat: custom @@ -554,7 +554,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[1234567]) { "A1": "a1", "A2": "a2", - "A4": "" + "A4": "null" } ======================================= #SORT_END From d039ce1e7f7c5b0f9bffc33f81526274172f08bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 10:24:23 +0200 Subject: [PATCH 267/390] ADD log, log10, log2 and sqrt tests --- .../jexl_transformation_full.test | 60 ++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index decc289255..b69cf370ba 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -67,7 +67,11 @@ payload='{ "M", "N", "O", - "P" + "P", + "Q", + "R", + "S", + "T" ], "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", @@ -135,6 +139,22 @@ payload='{ "P": { "type": "Number", "value": "${P|len}" + }, + "Q": { + "type": "Number", + "value": "${Q|log}" + }, + "R": { + "type": "Number", + "value": "${R|log10}" + }, + "S": { + "type": "Number", + "value": "${S|log2}" + }, + "T": { + "type": "Number", + "value": "${T|sqrt}" } } } @@ -218,6 +238,22 @@ payload='{ "P": { "type": "Text", "value": "NA" + }, + "Q": { + "type": "Number", + "value": 2.80 + }, + "R": { + "type": "Number", + "value": 100 + }, + "S": { + "type": "Number", + "value": 32 + }, + "T": { + "type": "Number", + "value": 25 } }' orionCurl --url /v2/entities --payload "$payload" @@ -257,7 +293,7 @@ Content-Length: 0 ================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 882 +Content-Length: 1076 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -351,6 +387,26 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Number", "value": 2 }, + "Q": { + "metadata": {}, + "type": "Number", + "value": 1.029619417 + }, + "R": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "S": { + "metadata": {}, + "type": "Number", + "value": 5 + }, + "T": { + "metadata": {}, + "type": "Number", + "value": 5 + }, "id": "E1", "type": "T" } From 13e20349565208c1d36ef6347c85b09bfb3a3ca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 11:00:24 +0200 Subject: [PATCH 268/390] FIX use cjexl 0.1.0 --- ci/deb/build.sh | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.alpine | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index 5a5393f877..4dffcd865b 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -175,7 +175,7 @@ rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - bash /opt/fiware-orion/get_cjexl.sh 0.0.0 $REPO_ACCESS_TOKEN + bash /opt/fiware-orion/get_cjexl.sh 0.1.0 $REPO_ACCESS_TOKEN fi if [ -n "${branch}" ]; then diff --git a/docker/Dockerfile b/docker/Dockerfile index 13033a9343..04394da083 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.0.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.1.0 ${REPO_ACCESS_TOKEN} && \ make && \ make install && \ # reduce size of installed binaries diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 2c4ed4a02e..5c302db67d 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -110,7 +110,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.0.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.1.0 ${REPO_ACCESS_TOKEN} && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ From 9cdcb5262a20ccba788b2fbe59f533e0de81bf0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 15:26:18 +0200 Subject: [PATCH 269/390] FIX memory leak --- src/lib/expressions/ExprManager.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index c56587cdf3..4a92d91e34 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -123,6 +123,10 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st // The ExprResult::fill() method allocates dynamic memory. So, the callers to evaluate() are supposed to invoke ExprResult::release() // method to free it r.fill(result); + + // cjexl_eval() allocated memory for us. We have to release it in order to avoid a leak + free((char*)result); + TIME_EXPR_JEXL_EVAL_STOP(); } From da9c9b51805a542ab2e6e590408420b0b78ac226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 16:41:04 +0200 Subject: [PATCH 270/390] ADD documentation --- CHANGES_NEXT_RELEASE | 9 +- doc/manuals/admin/build_source.md | 2 + doc/manuals/admin/statistics.md | 45 ++- doc/manuals/devel/sourceCode.md | 6 + doc/manuals/orion-api.md | 510 +++++++++++++++++++++++++++++- 5 files changed, 527 insertions(+), 45 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index c44ac67e3f..21a630772a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,7 +1,6 @@ -<<<<<<< HEAD -- Add: expression context build and evaluation counter in timing section in GET /statistics (#4004) -- Fix: lighter operation to get databases list from MongoDB (#4517) -======= +- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.1.0) (#4004) +- Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) +- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviur more consistent - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) ->>>>>>> master + diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index 64064030e4..3292e71d32 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -4,6 +4,8 @@ Orion Context Broker reference distribution is Debian 12. This doesn't mean that You can also have a look to [3.1 Building in not official distributions](../../../docker/README.md#31-building-in-not-official-distributions) section in the Docker documentation to check how to build Docker containers images in distributions other than the official one. +*NOTE:* the build process described in this document does not include the cjexl library, as it is considered optional from the point of view of the basic building process. + ## Debian 12 (officially supported) The Orion Context Broker uses the following libraries as build dependencies: diff --git a/doc/manuals/admin/statistics.md b/doc/manuals/admin/statistics.md index 7a0eb4f487..a44bafcb74 100644 --- a/doc/manuals/admin/statistics.md +++ b/doc/manuals/admin/statistics.md @@ -129,28 +129,22 @@ Provides timing information, i.e. the time that CB passes executing in different "timing": { "accumulated": { "jsonV1Parse": 7.860908311, - "mongoBackend": 416.796091597, - "mongoReadWait": 4656.924425628, - "mongoWriteWait": 259.347915990, - "mongoCommandWait": 0.514811318, - "exprBasicCtxBld": FIXME PR, - "exprBasicEval": FIXME PR, - "exprJexlCtxBld": FIXME PR, - "exprJexlEval": FIXME PR, - "render": 108.162782114, - "total": 6476.593504743 - }, + "jsonV2Parse": 120.680244446, + "mongoBackend": 12778.52734375, + "mongoReadWait": 7532.301757812, + "mongoWriteWait": 3619.282226562, + "mongoCommandWait": 0.120559767, + "exprJexlCtxBld": 27.092681885, + "exprJexlEval": 124.217208862, + "render": 44.540554047, + "total": 25051.384765625 + }, "last": { - "mongoBackend": 0.014752309, - "mongoReadWait": 0.012018445, - "mongoWriteWait": 0.000574611, - "exprBasicCtxBld": FIXME PR, - "exprBasicEval": FIXME PR, - "exprJexlCtxBld": FIXME PR, - "exprJexlEval": FIXME PR, - "render": 0.000019136, - "total": 0.015148915 - } + "mongoBackend": 0.003775352, + "mongoReadWait": 0.0013743, + "render": 0.000286864, + "total": 0.00440685 + } } ... } @@ -164,8 +158,6 @@ The block includes two main sections: The particular counters are as follows: -FIXME PR: explain expr* fields - * `total`: processing time for the whole request, excluding the time that the HTTP library takes for request/response dispatching (pseudo end-to-end time) * `jsonV1Parse`: time passed in NGSIv1 JSON parsing module (pseudo self-time) @@ -177,10 +169,15 @@ FIXME PR: explain expr* fields `last` includes the accumulation for all of them. In the case of mongoReadWait, only the time used to get the results cursor is taken into account, but not the time to process cursors results (which is time that belongs to mongoBackend counters). +* `exprJexlCtxBld`: time passed building context for custom notification expression evaluation (see [macro substitution](../orion-api.md#macro-substitution) and [JEXL support](../orion-api.md#jexl-support)) +* `exprJexlEval`: time passed evaluating custom notification expressions (see [macro substitution](../orion-api.md#macro-substitution) and [JEXL support](../orion-api.md#jexl-support)) + +*NOTE*: if Orion binary is build without using cjexl and only basic replacement is available, then `exprBasicCtxtBld` and `exprBasicEval` +fields appear instead of `exprJexlCtxBld` and `exprJexlEval`. Times are measured from the point in time in which a particular thread request starts using a module until it finishes using it. Thus, if the thread is stopped for some reason (e.g. the kernel decides to give priority to another thread based on its -scheculing policy) the time that the thread was sleeping, waiting to execute again is included in the measurement and thus, the measurement is not accurate. That is why we say *pseudo* selt/end-to-end time. However, +scheculing policy) the time that the thread was sleeping, waiting to execute again is included in the measurement and thus, the measurement is not accurate. That is why we say *pseudo* self/end-to-end time. However, under low load conditions this situation is not expected to have a significant impact. ### NotifQueue block diff --git a/doc/manuals/devel/sourceCode.md b/doc/manuals/devel/sourceCode.md index c9227740b0..951e8777f8 100644 --- a/doc/manuals/devel/sourceCode.md +++ b/doc/manuals/devel/sourceCode.md @@ -24,6 +24,7 @@ * [src/lib/cache/](#srclibcache) (Subscription cache implementation) * [src/lib/logSummary/](#srcliblogsummary) (Log Summary implementation) * [src/lib/metricsMgr/](#srclibmetricsmgr) (Metrics Manager implementation) +* [src/lib/expressions/](#seribexpressions) (Custom notification expressions support) ## src/app/contextBroker/ The main program is found in `contextBroker.cpp` and its purpose it to: @@ -551,3 +552,8 @@ This Metrics Manager resides in the library **metricsMgr**. For information about the metrics, please refer to [this document](../admin/metrics_api.md). [Top](#top) + +## src/lib/expressions/ +Provides support to the [macro substition logic used by custom notifications](../orion-api.md#macro-substitution). This library provides an abstraction for expression evaluation, providing two implementations: JEXL based and basic replacement based (the implementation to use is choosen at building time, based on the availability of the cjex library). + +[Top](#top) \ No newline at end of file diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 5e8e6e9bef..ea23b5f764 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -73,6 +73,32 @@ - [NGSI payload patching](#ngsi-payload-patching) - [Omitting payload](#omitting-payload) - [Additional considerations](#additional-considerations) + - [JEXL Support](#jexl-support) + - [Available Transformations](#available-transformations) + - [`uppercase`](#uppercase) + - [`lowercase`](#lowercase) + - [`split`](#split) + - [`indexOf`](#indexOf) + - [`len`](#len) + - [`trim`](#trim) + - [`substring`](#substring) + - [`includes`](#includes) + - [`isNaN`](#isNaN) + - [`parseInt`](#parseInt) + - [`parseFloat`](#parseFloat) + - [`typeOf`](#typeOf) + - [`toString`](#toString) + - [`floor`](#floor) + - [`ceil`](#ceil) + - [`round`](#round) + - [`toFixed`](#toFixed) + - [`log`](#log) + - [`log10`](#log10) + - [`log2`](#log2) + - [`sqrt`](#sqrt) + - [`replaceStr`](#replaceStr) + - [Failsafe cases](#failsafe-cases) + - [Known limitations](#known-limitations) - [Oneshot Subscriptions](#oneshot-subscriptions) - [Covered Subscriptions](#covered-subscriptions) - [Subscriptions based in alteration type](#subscriptions-based-in-alteration-type) @@ -2022,23 +2048,22 @@ In case of `mqttCustom`: * `payload`, `json` and `ngsi` (all them payload related fields) * `topic` -Macro substitution for templates is based on the syntax `${..}`. In particular: +Macro substitution for templates is based on the syntax `${}`. The support to JEXL +is explained in [JEXL Support](#jexl-support) section. The following identifiers are included in +the context evaluated by the JEXL expression: -* `${id}` is replaced by the `id` of the entity -* `${type}` is replaced by the `type` of the entity -* `${service}` is replaced by the service (i.e. `fiware-service` header value) in the +* `id`: for the `id` of the entity +* `type`: for the `type` of the entity +* `service`: for the service (i.e. `fiware-service` header value) in the update request triggering the subscription. -* `${servicePath}` is replaced by the service path (i.e. `fiware-servicepath` header value) in the +* `servicePath`: for the service path (i.e. `fiware-servicepath` header value) in the update request triggering the subscription. -* `${authToken}` is replaced by the authorization token (i.e. `x-auth-token` header value) in the +* `authToken: for the authorization token (i.e. `x-auth-token` header value) in the update request triggering the subscription. -* Any other `${token}` is replaced by the value of the attribute whose name is `token` or with - an empty string if the attribute is not included in the notification. If the value is a number, - a bool or null then its string representation is used. If the value is a JSON array or object - then its JSON representation as string is used. +* All the attributes in the entity triggering the notification (included in the update triggering the notification or not) -In the rare case an attribute was named in the same way of the `${service}`, `${servicePath}` or -`${authToken}` (e.g. an attribute which name is `service`) then the attribute value takes precedence. +In the rare case an attribute was named in the same way of the `service`, `servicePath` or +`authToken` (e.g. an attribute which name is `service`) then the attribute value takes precedence. Example: @@ -2296,6 +2321,463 @@ Some considerations to take into account when using custom notifications: (i.e. `ngsi` field) then `Ngsiv2-AttrsFormat: normalized` is used, as in a regular notification (given that the notification format is actually the same). +## JEXL Support + +Orion Context Broker supports [JEXL expressions](https://github.com/TomFrost/Jexl) in custom notification [macro replacement](#macro-substitution). Thus, subscriptions like this can be defined: + +``` +"httpCustom": { + ... + "ngsi": { + "relativeHumidity": { + "value": "${humidity/100}", + "type": "Calculated" + } + } +} +``` + +So, if a given update sets entity `humidity` attribute to `84.4` then the notification will include a `relativeHumidity` attribute with value `0.844`. + +A particular case of expressions are the ones in which the expression is a given context identifier, without an actual expression using it. For instance: + +``` +"httpCustom": { + ... + "ngsi": { + "originalHumidity": { + "value": "${humidity}", + "type": "Calculated" + } + } +} +``` + +We also refers to this case as *basic replacement*. + +An useful resource to test JEXL expressions is the [JEXL playground](https://czosel.github.io/jexl-playground). However, take into account the differences between the original JEXL implementation in JavaScript and the one included in Orion, described in the [known limitations](#known-limitations) section. + +Orion relies on cjexl library to provide this functionality. If Orion binary is build without using cjexl, then only basic replacement functionality is available. + +### Available Transformations + +#### `uppercase` + +Convert a string into uppercase. + +Extra arguments: none + +Example (being context `{"c": "fooBAR"}`): + +``` +c|uppercase +``` + +results in + +``` +"FOOBAR" +``` + +#### lowercase + +Convert a string into lowercase. + +Extra arguments: none + +Example (being context `{"c": "fooBAR"}`): + +``` +c|lowercase +``` + +results in + +``` +"foobar" +``` + +#### split + +Split the input string into array items. + +Extra arguments: delimiter to use for the split. + +Example (being context `{"c": "foo,bar,zzz"}`): + +``` +c|split(',') +``` + +results in + +``` +[ "foo", "bar", "zzz" ] +``` + +#### indexOf + +Provides the position of a given string within the input string. In the string is not found, returns `null`. + +Extra arguments: the input string to search. + +Note this function doesn't work if the input is an array (it only works for strings). + +Example (being context `{"c": "fooxybar"}`): + +``` +c|indexOf('xy') +``` + +results in + +``` +3 +``` + +#### len + +Provides the length of a string. + +Extra arguments: none. + +Note this function doesn't work if the input is an array (it only works for strings). + +Example (being context `{"c": "foobar"}`): + +``` +c|len +``` + +results in + +``` +6 +``` + +#### trim + +Removes starting and trailing whitespaces. + +Extra arguments: none. + +Example (being context `{"c": " foo bar "}`): + +``` +c|trim +``` + +results in + +``` +foo bar +``` + +#### substring + +Returns a substring between two positions. + +Extra arguments: +* Initial position +* Final position + +Example (being context `{"c": "foobar"}`): + +``` +c|substring(3,5) +``` + +results in + +``` +ba +``` + +#### includes + +Returns `true` if a given string is contained in the input string, `false` otherwise. + +Extra arguments: the input string to search. + +Example (being context `{"c": "foobar"}`): + +``` +c|includes('ba') +``` + +results in + +``` +true +``` + +#### isNaN + +Returns `true` if the input is not a number, `false` otherwise. + +Extra arguments: none. + +Example (being context `{"c": "foobar"}`): + +``` +c|isNaN +``` + +results in + +``` +true +``` + +#### parseInt + +Parses a string and return the corresponding integer number. + +Extra arguments: none. + +Example (being context `{"c": "25"}`): + +``` +c|parseInt +``` + +results in + +``` +25 +``` + +#### parseFloat + +Parses a string and return the corresponding float number + +Extra arguments: none. + +Example (being context `{"c": "25.33"}`): + +``` +c|parseFloat +``` + +results in + +``` +25.33 +``` + +#### typeOf + +Return a string with the type of the input. + +Extra arguments: none. + +Example (being context `{"c": 23}`): + +``` +c|typeOf +``` + +results in + +``` +"Number" +``` + +#### toString + +Return a string representation of the input. + +Extra arguments: none. + +Example (being context `{"c": 23}`): + +``` +c|toString +``` + +results in + +``` +"23" +``` + +#### floor + +Return the closest lower integer of a given number. + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|floor +``` + +results in + +``` +3 +``` + +#### ceil + +Return the closest upper integer of a given number. + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|ceil +``` + +results in + +``` +4 +``` + +#### round + +Return the closest integer (either lower or upper) of a given number. + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|round +``` + +results in + +``` +3 +``` + +#### toFixed + +Rounds a number to a number of decimal digits. + +Extra arguments: number of decimal digits. + +Example (being context `{"c": 3.18}`): + +``` +c|toFixed(1) +``` + +results in + +``` +3.2 +``` + +#### log + +Return the natural logarithm of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|log +``` + +results in + +``` +1.144222799920162 +``` + +#### log10 + +Return the base 10 logarithm of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|log10 +``` + +results in + +``` +0.49692964807321494 +``` + +#### log2 + +Return the base 2 logarithm of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|log2 +``` + +results in + +``` +1.6507645591169025 +``` + +#### sqrt + +Return the square root of a given number + +Extra arguments: none. + +Example (being context `{"c": 3.14}`): + +``` +c|sqrt +``` + +results in + +``` +1.772004514666935 +``` + +#### replaceStr + +Replace occurrences of a string with another in the input string. + +Extra arguments: +* Source string to replace +* Destination string to replace + +Example (being context `{"c": "foobar"}`): + +``` +c|replaceStr('o','u') +``` + +results in + +``` +"fuubar" +``` + +### Failsafe cases + +As failsafe behaviour, evaluation returns `null` in the following cases: + +* Some of the transformation used in the expression is unknown +* Operations with identifiers that are not defined in the context are used. For instance, `(A==null)?-1:A` will result in `null` (and not `-1`) if `A` is not in the context, due to `==` is an operation that cannot be done on undefined identifiers. However, `A||-1` will work (i.e. `-1` will result if `A` is not in the context), as `||` is not considered an operation on `A`. +* Syntax error in the JEXL expression (e.g. `A[0|uppercase`) + +### Known limitations + +- The unitary minus operator is not working properly, e.g. the following expression doesn't work (it failsafes to `null`): `A||-1`. However, the following alternatives are working: `A||0-1` and `A||'-1'|parseInt)` +- Negation operator `!` (supported in original JavaScript JEXL) is not supported + ## Oneshot Subscriptions Oneshot subscription provides an option to subscribe an entity only for one time notification. When consumer creates a subscription @@ -2395,10 +2877,6 @@ in which case all attributes are included in the notification, no matter if they entity. For these attributes that don't exist (`brightness` in this example) the `null` value (of type `"None"`) is used. -In the case of custom notifications, if `covered` is set to `true` then `null` will be used to replace `${...}` -for non existing attributes (the default behavior when `covered` is not set to `true` is to replace by the -empty string the non existing attributes). - We use the term "covered" in the sense the notification "covers" completely all the attributes in the `notification.attrs` field. It can be useful for those notification endpoints that are not flexible enough for a variable set of attributes and needs always the same set of incoming attributes From 826ffdd4bcb917de04d14e06eb89d9cc9c4042f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 16:47:30 +0200 Subject: [PATCH 271/390] FIX basic replacement logic --- src/lib/expressions/ExprManager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index 4a92d91e34..cd1a3d5182 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -124,8 +124,11 @@ ExprResult ExprManager::evaluate(ExprContextObject* exprContextObjectP, const st // method to free it r.fill(result); +#ifndef EXPR_BASIC // cjexl_eval() allocated memory for us. We have to release it in order to avoid a leak + // (disbled with EXPR_BASIC because in that case result is static memory) free((char*)result); +#endif TIME_EXPR_JEXL_EVAL_STOP(); } From 9e04d8c5a658daed8530ae45ee9794e87cd36d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 17:03:44 +0200 Subject: [PATCH 272/390] FIX warning about special chars --- doc/manuals/orion-api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index ea23b5f764..985db83d48 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -455,6 +455,8 @@ In addition, the [General syntax restrictions](#general-syntax-restrictions) als In case a client attempts to use a field that is invalid from a syntax point of view, the client gets a "Bad Request" error response, explaining the cause. +Note that although `:` and `-` are allowed in identifiers, they are strongly discouraged, as they collide with the [JEXL syntax](#jexl-support). In particular, `-` is used for subtraction operation (e.g. `${A-B}`) and `:` is used in the ternary operator (eg. `A?'A is true':'A is false`). Thus, an attribute name `lower-temperature` in an expression `${lower-temperature}` would be interpreted as the value of `lower` attribute minus `temperature` attribute (and not as the value of an attribute named `lower-temperature`). + ## Error Responses If present, the error payload is a JSON object including the following fields: From 8587216a1f129a473a6318253b6653582fe32008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 27 May 2024 17:06:19 +0200 Subject: [PATCH 273/390] FIX enhance CNR --- CHANGES_NEXT_RELEASE | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 21a630772a..0f5194da98 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,6 +1,7 @@ - Add: JEXL expression support in custom notification macro replacement (using cjexl 0.1.0) (#4004) - Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) -- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviur more consistent +- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004) +- Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) From fbccd91322412b84fac047351bb3641a5c3c3411 Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Tue, 28 May 2024 15:30:42 +0530 Subject: [PATCH 274/390] fix for wrong date values --- CHANGES_NEXT_RELEASE | 1 + src/lib/common/globals.cpp | 19 + .../wrong_date_values_in_expires_field.test | 358 ++++++++++++++++++ 3 files changed, 378 insertions(+) create mode 100644 test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_date_values_in_expires_field.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e8ee65ca95..c1ae92d17c 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) +- Fix: wrong date values should not allowed in subscription's expires field (#4541) diff --git a/src/lib/common/globals.cpp b/src/lib/common/globals.cpp index 511dbd9d0d..0dd8b886d1 100644 --- a/src/lib/common/globals.cpp +++ b/src/lib/common/globals.cpp @@ -606,6 +606,25 @@ double parse8601Time(const std::string& ss) time.tm_min = m; // 0-59 time.tm_sec = (int) s; // 0-61 (0-60 in C++11) + const int minYear = 0; + const int minMonth = 0; + const int maxMonth = 11; + const int minDay = 1; + const int maxDay = 31; + const int minHour = 0; + const int maxHour = 23; + const int minMinute = 0; + const int maxMinute = 59; + const int minSecond = 0; + const int maxSecond = 59; + + if (time.tm_year < minYear || time.tm_mon < minMonth || time.tm_mon > maxMonth || time.tm_mday < minDay || + time.tm_mday > maxDay || time.tm_hour < minHour || time.tm_hour > maxHour || time.tm_min < minMinute || + time.tm_min > maxMinute || time.tm_sec < minSecond || time.tm_sec > maxSecond) + { + return -1; + } + int64_t totalSecs = timegm(&time) - offset; float millis = s - (int) s; double timestamp = totalSecs; diff --git a/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_date_values_in_expires_field.test b/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_date_values_in_expires_field.test new file mode 100644 index 0000000000..a90e17a916 --- /dev/null +++ b/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_date_values_in_expires_field.test @@ -0,0 +1,358 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Wrong date values are not supported in 'expires' field of subscription + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create a subscription with an wrong date value in the 'seconds' component of the 'expires' field, see error +# 02. Create a subscription with an wrong date value in the 'minutes' component of the 'expires' field, see error +# 03. Create a subscription with an wrong date value in the 'hours' component of the 'expires' field, see error +# 04. Create a subscription with an wrong date value in the 'days' component of the 'expires' field, see error +# 05. Create a subscription with an wrong date value in the 'months' component of the 'expires' field, see error +# 06. Create a subscription with an wrong date value in the 'years' component of the 'expires' field, see error +# 07. Create a subscription with an valid date value in 'expires' field, (success case) +# + + +echo "01. Create a subscription with an wrong date value in the 'seconds' component of the 'expires' field, see error" +echo "===============================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2022-12-21T17:16:63.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create a subscription with an wrong date value in the 'minutes' component of the 'expires' field, see error" +echo "===============================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2022-12-21T17:65:03.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create a subscription with an wrong date value in the 'hours' component of the 'expires' field, see error" +echo "=============================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2022-12-21T25:16:11.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "04. Create a subscription with an wrong date value in the 'days' component of the 'expires' field, see error" +echo "============================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2022-12-34T17:16:23.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create a subscription with an wrong date value in the 'months' component of the 'expires' field, see error" +echo "==============================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2022-13-21T17:16:43.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "06. Create a subscription with an wrong date value in the 'years' component of the 'expires' field, see error" +echo "=============================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "1822-12-21T17:16:23.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "07. Create a subscription with an valid date value in 'expires' field, (success case)" +echo "=====================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2025-12-31T23:59:59.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. Create a subscription with an wrong date value in the 'seconds' component of the 'expires' field, see error +=============================================================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +02. Create a subscription with an wrong date value in the 'minutes' component of the 'expires' field, see error +=============================================================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +03. Create a subscription with an wrong date value in the 'hours' component of the 'expires' field, see error +============================================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +04. Create a subscription with an wrong date value in the 'days' component of the 'expires' field, see error +============================================================================================================ +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +05. Create a subscription with an wrong date value in the 'months' component of the 'expires' field, see error +============================================================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +06. Create a subscription with an wrong date value in the 'years' component of the 'expires' field, see error +============================================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +07. Create a subscription with an valid date value in 'expires' field, (success case) +===================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB + From 90ca88eb934ad0d8c7c9233d36dbb638fc76fe63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 May 2024 12:20:31 +0200 Subject: [PATCH 275/390] ADD mapper and thMapper transformations tests --- doc/manuals/orion-api.md | 46 +++++++++++++++++++ .../jexl_transformation_full.test | 38 +++++++++++++-- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 985db83d48..57e3396481 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -97,6 +97,8 @@ - [`log2`](#log2) - [`sqrt`](#sqrt) - [`replaceStr`](#replaceStr) + - [`mapper`](#mapper) + - [`thMapper`](#thmapper) - [Failsafe cases](#failsafe-cases) - [Known limitations](#known-limitations) - [Oneshot Subscriptions](#oneshot-subscriptions) @@ -2767,6 +2769,50 @@ results in "fuubar" ``` +#### mapper + +Returns a value among several choices based in one to one mapping. This function is based in an array of *values* and an array of *choices* (which length is exactly the same). Thus, if the input value is equal to the *i*-th item of *values*, then *i*-th item of *choices* is returned. + +This transformation returns `null` if some problem with the arguments is found (i.e. input is not found among the values, choices length is not exacly the same as values, the input is not an string, etc.) + +Extra arguments: +* values array +* choices array + +Example (being context `{"c": "fr", "values": ["es", "fr", "de"], "choices": ["Spain", "France", "Germany"]}`): + +``` +c|mapper(values,choices) +``` + +results in + +``` +"France" +``` + +#### thMapper + +Returns a value among several choices based in threshold values. This function is based in an array of *values* and an array of *choices* (which length is exactly the same as values plus one). Thus, if the input value is between the *i*-th and the *i+1*-th item of *values*, then *i*+1-th item of *choices* is returned. + +This transformation returns `null` if some problem with the arguments is found (i.e. choices length is not exacly the same as values plus one, some of the items in the values array is not a number, etc.) + +Extra arguments: +* values array +* choices array + +Example (being context `{"c": 0.5, "values": [-1, 1], "choices": ["low", "medium", "high"]}`): + +``` +c|thMapper(values,choices) +``` + +results in + +``` +"medium" +``` + ### Failsafe cases As failsafe behaviour, evaluation returns `null` in the following cases: diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index b69cf370ba..d4b83278e1 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -33,7 +33,7 @@ accumulatorStart --pretty-print # # 01. Create custom sub using all transformations -# 02. Create entity E1 with A to P attributes +# 02. Create entity E1 with A to V attributes # 03. Dump accumulator and see expected notification # @@ -71,7 +71,9 @@ payload='{ "Q", "R", "S", - "T" + "T", + "U", + "V" ], "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", @@ -155,6 +157,14 @@ payload='{ "T": { "type": "Number", "value": "${T|sqrt}" + }, + "U": { + "type": "Text", + "value": "${U|mapper(['\''es'\'','\''fr'\''],['\''Spain'\'','\''France'\''])}" + }, + "V": { + "type": "Text", + "value": "${V|thMapper([1,2],['\''low'\'','\''medium'\'','\''high'\''])}" } } } @@ -165,7 +175,7 @@ echo echo -echo "02. Create entity E1 with A to P attributes" +echo "02. Create entity E1 with A to V attributes" echo "===========================================" payload='{ "id": "E1", @@ -254,6 +264,14 @@ payload='{ "T": { "type": "Number", "value": 25 + }, + "U": { + "type": "Text", + "value": "fr" + }, + "V": { + "type": "Number", + "value": 1.5 } }' orionCurl --url /v2/entities --payload "$payload" @@ -279,7 +297,7 @@ Content-Length: 0 -02. Create entity E1 with A to P attributes +02. Create entity E1 with A to V attributes =========================================== HTTP/1.1 201 Created Date: REGEX(.*) @@ -293,7 +311,7 @@ Content-Length: 0 ================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 1076 +Content-Length: 1178 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -407,6 +425,16 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Number", "value": 5 }, + "U": { + "metadata": {}, + "type": "Text", + "value": "France" + }, + "V": { + "metadata": {}, + "type": "Text", + "value": "medium" + }, "id": "E1", "type": "T" } From e3a3f4584dd548727006c3b771fc788795ac0375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 May 2024 12:36:18 +0200 Subject: [PATCH 276/390] FIB use cjexl 0.2.0 --- CHANGES_NEXT_RELEASE | 2 +- ci/deb/build.sh | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.alpine | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0f5194da98..ae44980113 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.1.0) (#4004) +- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.2.0) (#4004) - Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) - Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004) - Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index 4dffcd865b..b0e5027b8a 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -175,7 +175,7 @@ rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - bash /opt/fiware-orion/get_cjexl.sh 0.1.0 $REPO_ACCESS_TOKEN + bash /opt/fiware-orion/get_cjexl.sh 0.2.0 $REPO_ACCESS_TOKEN fi if [ -n "${branch}" ]; then diff --git a/docker/Dockerfile b/docker/Dockerfile index 04394da083..e6c7846b56 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.1.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.2.0 ${REPO_ACCESS_TOKEN} && \ make && \ make install && \ # reduce size of installed binaries diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 5c302db67d..9e35eb3aa9 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -110,7 +110,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.1.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.2.0 ${REPO_ACCESS_TOKEN} && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ From ba0826f3add3e57c6104717357dd6086df84e3b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 May 2024 13:01:31 +0200 Subject: [PATCH 277/390] FIX improve get_cjexl.sh --- get_cjexl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get_cjexl.sh b/get_cjexl.sh index c906af50b2..efd360756e 100644 --- a/get_cjexl.sh +++ b/get_cjexl.sh @@ -23,7 +23,7 @@ TOKEN=$2 res_code=$(curl -I -s -o /dev/null -w "%{http_code}" -H "Authorization: token $TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION) if [ "$res_code" -eq 200 ]; then - echo "get_cjexl: downloading cjexl lib" + echo "get_cjexl: downloading cjexl lib $CJEXL_VERSION" ASSET_ID=$(curl -s -S -H "Authorization: token $TOKEN" https://api.github.com/repos/telefonicasc/cjexl/releases/tags/$CJEXL_VERSION | grep -C3 libcjexl.a | grep '"id"' | awk -F ' ' '{print $2}' | awk -F ',' '{print $1}') curl -L -s -o /usr/local/lib/libcjexl.a -H "Authorization: token $TOKEN" -H "Accept: application/octet-stream" https://api.github.com/repos/telefonicasc/cjexl/releases/assets/$ASSET_ID MD5SUM=$(md5sum /usr/local/lib/libcjexl.a | awk -F ' ' '{print$1}') From 75fab951320476046ddd2ffd98aea11667695f7d Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Tue, 28 May 2024 17:09:05 +0530 Subject: [PATCH 278/390] corrected failed FT --- .../error_messages_for_invalid_tenant_in_v2.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionalTest/cases/1087_error_handling/error_messages_for_invalid_tenant_in_v2.test b/test/functionalTest/cases/1087_error_handling/error_messages_for_invalid_tenant_in_v2.test index 722655fa24..c9e78a26de 100644 --- a/test/functionalTest/cases/1087_error_handling/error_messages_for_invalid_tenant_in_v2.test +++ b/test/functionalTest/cases/1087_error_handling/error_messages_for_invalid_tenant_in_v2.test @@ -41,7 +41,7 @@ payload='{ "timestamp_0": { "type": "DateTime", - "value": "017-06-17T07:21:24.238Z", + "value": "2017-06-17T07:21:24.238Z", "metadata": { "very_hot_1": { From 41f046fd5fac94947c738fbe309bf0c78f8365cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 28 May 2024 18:17:06 +0200 Subject: [PATCH 279/390] ADD additional jexl expression test --- .../jexl_transformation_multiple.test | 294 ++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test new file mode 100644 index 0000000000..ec3f7f0fff --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test @@ -0,0 +1,294 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +JEXL basic expression in custom notification (multiple transformations) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with several attributes with transformations +# 02. Create entity E1 +# 03. Update entity E1 +# 05. Dump accumulator and see two expected transformations +# + + +echo "01. Create custom sub with several attributes with transformations" +echo "==================================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "speed": { + "value": "${(speed|split('\'' '\''))[0]|parseInt}", + "type": "Calculated" + }, + "ratio": { + "value": "${count.sum/count.count}", + "type": "Calculated" + }, + "code": { + "value": "${code||'\''invalid'\''}", + "type": "Calculated" + }, + "alert": { + "value": "${(value>max)?'\''nok'\'':'\''ok'\''}", + "type": "Calculated" + }, + "count": { + "value": "${{count:count.count+1, sum:count.sum+((speed|split('\'' '\''))[0]|parseInt)}}", + "type": "Calculated" + } + } + }, + "attrs": [ "speed", "ratio", "code", "alert", "count" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + +#"value": "${{count:count.count+1,sum:count.sum+speed|split('\'' '\'')[0]|parseInt}}", + +echo "02. Create entity E1" +echo "====================" +payload='{ + "id": "E1", + "type": "T", + "speed": { + "value": "10 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 100 + }, + "type": "StructuredValue" + }, + "code": { + "value": null, + "type": "Number" + }, + "value": { + "value": 14, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1" +echo "====================" +payload='{ + "speed": { + "value": "30 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 500 + }, + "type": "StructuredValue" + }, + "code": { + "value": 456, + "type": "Number" + }, + "value": { + "value": 75, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see two expected transformations" +echo "=========================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with several attributes with transformations +================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 +==================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 +==================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see two expected transformations +========================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 379 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "ok" + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": "invalid" + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 110 + } + }, + "id": "E1", + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 20 + }, + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 10 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 375 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "nok" + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": 456 + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 530 + } + }, + "id": "E1", + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 100 + }, + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 30 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 486e7df47353eb1489be2d153acadec1bd9e49ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 08:09:45 +0200 Subject: [PATCH 280/390] ADD JEXL usage example to doc --- doc/manuals/orion-api.md | 198 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 184 insertions(+), 14 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 57e3396481..01040ca9ab 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -74,6 +74,7 @@ - [Omitting payload](#omitting-payload) - [Additional considerations](#additional-considerations) - [JEXL Support](#jexl-support) + - [JEXL usage example](#jexl-usage-example) - [Available Transformations](#available-transformations) - [`uppercase`](#uppercase) - [`lowercase`](#lowercase) @@ -2331,13 +2332,13 @@ Orion Context Broker supports [JEXL expressions](https://github.com/TomFrost/Jex ``` "httpCustom": { - ... - "ngsi": { - "relativeHumidity": { - "value": "${humidity/100}", - "type": "Calculated" - } - } + ... + "ngsi": { + "relativeHumidity": { + "value": "${humidity/100}", + "type": "Calculated" + } + } } ``` @@ -2347,13 +2348,13 @@ A particular case of expressions are the ones in which the expression is a given ``` "httpCustom": { - ... - "ngsi": { - "originalHumidity": { - "value": "${humidity}", - "type": "Calculated" - } - } + ... + "ngsi": { + "originalHumidity": { + "value": "${humidity}", + "type": "Calculated" + } + } } ``` @@ -2363,6 +2364,175 @@ An useful resource to test JEXL expressions is the [JEXL playground](https://czo Orion relies on cjexl library to provide this functionality. If Orion binary is build without using cjexl, then only basic replacement functionality is available. +### JEXL usage example + +As example, let's consider a subscription like this: + +``` +"httpCustom": { + ... + "ngsi": { + "speed": { + "value": "${(speed|split('\'' '\''))[0]|parseInt}", + "type": "Calculated" + }, + "ratio": { + "value": "${count.sum/count.count}", + "type": "Calculated" + }, + "code": { + "value": "${code||'\''invalid'\''}", + "type": "Calculated" + }, + "alert": { + "value": "${(value>max)?'\''nok'\'':'\''ok'\''}", + "type": "Calculated" + }, + "count": { + "value": "${{count:count.count+1, sum:count.sum+((speed|split('\'' '\''))[0]|parseInt)}}", + "type": "Calculated" + } +} +``` + +A entity update like this: + +``` +{ + ... + "speed": { + "value": "10 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 100 + }, + "type": "StructuredValue" + }, + "code": { + "value": null, + "type": "Number" + }, + "value": { + "value": 14, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +} +``` + +will trigger a notification like this: + +``` +"data": [ + { + ... + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 10 + }, + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 20 + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": "invalid" + }, + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "ok" + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 110 + } + } + } +] +``` + +A new entity update like this: + +``` +{ + ... + "speed": { + "value": "30 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 500 + }, + "type": "StructuredValue" + }, + "code": { + "value": 456, + "type": "Number" + }, + "value": { + "value": 75, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +} +``` + +will trigger a notification like this: + +``` +"data": [ + { + ... + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 30 + }, + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 100 + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": 456 + }, + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "nok" + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 530 + } + } + } +] +``` + ### Available Transformations #### `uppercase` From b0df97eac75918eec50678a32b80f04e18dd9468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 08:23:25 +0200 Subject: [PATCH 281/390] FIX doc --- doc/manuals/orion-api.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 01040ca9ab..3d2ac6db18 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2373,7 +2373,7 @@ As example, let's consider a subscription like this: ... "ngsi": { "speed": { - "value": "${(speed|split('\'' '\''))[0]|parseInt}", + "value": "${(speed|split(' '))[0]|parseInt}", "type": "Calculated" }, "ratio": { @@ -2381,15 +2381,15 @@ As example, let's consider a subscription like this: "type": "Calculated" }, "code": { - "value": "${code||'\''invalid'\''}", + "value": "${code||'invalid'}", "type": "Calculated" }, "alert": { - "value": "${(value>max)?'\''nok'\'':'\''ok'\''}", + "value": "${(value>max)?'nok':'ok'}", "type": "Calculated" }, "count": { - "value": "${{count:count.count+1, sum:count.sum+((speed|split('\'' '\''))[0]|parseInt)}}", + "value": "${{count:count.count+1, sum:count.sum+((speed|split(' '))[0]|parseInt)}}", "type": "Calculated" } } From 047a63b32d50def2c821b4c397c5b963e1cb7db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 08:35:37 +0200 Subject: [PATCH 282/390] Apply suggestions from code review --- .github/workflows/publishimage-master.yml | 1 + .github/workflows/publishimage-tag.yml | 1 + CHANGES_NEXT_RELEASE | 1 - doc/manuals/devel/sourceCode.md | 2 +- doc/manuals/orion-api.md | 2 +- src/lib/expressions/ExprContext.cpp | 2 -- src/lib/expressions/ExprManager.cpp | 3 +-- src/lib/rest/rest.cpp | 4 ++-- src/lib/serviceRoutines/versionTreat.cpp | 2 +- 9 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index 2811dea64f..64ceb47bcf 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -1,6 +1,7 @@ name: Publish Docker image (master) # The workflow will push images for master on every merge +# Ideally, this should be done at dockerhub, but it doesn't support secrets (see https://stackoverflow.com/questions/78446824/use-environment-variables-with-sensible-information-in-docker-hub-autobuild) on: push: diff --git a/.github/workflows/publishimage-tag.yml b/.github/workflows/publishimage-tag.yml index 6cd6b94b40..5dde6e8d27 100644 --- a/.github/workflows/publishimage-tag.yml +++ b/.github/workflows/publishimage-tag.yml @@ -1,6 +1,7 @@ name: Publish Docker image (tag) # The workflow will push images on every tag in the format x.y.z +# Ideally, this should be done at dockerhub, but it doesn't support secrets (see https://stackoverflow.com/questions/78446824/use-environment-variables-with-sensible-information-in-docker-hub-autobuild) on: push: diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index ae44980113..c3acdc183d 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -4,4 +4,3 @@ - Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) - diff --git a/doc/manuals/devel/sourceCode.md b/doc/manuals/devel/sourceCode.md index 951e8777f8..a829aa1f41 100644 --- a/doc/manuals/devel/sourceCode.md +++ b/doc/manuals/devel/sourceCode.md @@ -24,7 +24,7 @@ * [src/lib/cache/](#srclibcache) (Subscription cache implementation) * [src/lib/logSummary/](#srcliblogsummary) (Log Summary implementation) * [src/lib/metricsMgr/](#srclibmetricsmgr) (Metrics Manager implementation) -* [src/lib/expressions/](#seribexpressions) (Custom notification expressions support) +* [src/lib/expressions/](#srclibexpressions) (Custom notification expressions support) ## src/app/contextBroker/ The main program is found in `contextBroker.cpp` and its purpose it to: diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 3d2ac6db18..a294ce2d46 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -432,7 +432,7 @@ There are some exception cases in which the above restrictions do not apply. In * URL parameter `q` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `mq` allows the special characters needed by the [Simple Query Language](#simple-query-language) * URL parameter `georel` and `coords` allow `;` -* Within `ngsi` (i.e. `id`, `type` and attribute values) in [NGSI Payload patching](#ngsi-payload-patching) (to support characters used in the JEXL expression syntax) +* Within `ngsi` (i.e. `id`, `type` and attribute values) in [NGSI Payload patching](#ngsi-payload-patching) (to support characters used in the [JEXL expression syntax](#jexl-support)) * Whichever attribute value which uses `TextUnrestricted` as attribute type (see [Special Attribute Types](#special-attribute-types) section) ## Identifiers syntax restrictions diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 927809b693..80148399b3 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -217,7 +217,6 @@ std::string ExprContextList::get(void) */ void ExprContextList::add(const std::string &_value) { - LM_T(LmtExpr, ("adding to JEXL expression context list (string): %s", _value.c_str())); jh.addString(_value); } @@ -279,7 +278,6 @@ void ExprContextList::add(ExprContextObject exprContextObject) */ void ExprContextList::add(ExprContextList exprContextList) { - std::string s = exprContextList.get(); LM_T(LmtExpr, ("adding to JEXL expression context list (list): %s", s.c_str())); jh.addRaw(s); diff --git a/src/lib/expressions/ExprManager.cpp b/src/lib/expressions/ExprManager.cpp index cd1a3d5182..c9bbf0299e 100644 --- a/src/lib/expressions/ExprManager.cpp +++ b/src/lib/expressions/ExprManager.cpp @@ -46,7 +46,7 @@ static const char* cjexl_eval(void* ptr, const char* script_ptr, const char* con return ""; } #else -// Interface to use libcjexl.a +// Interface to use libcjexl extern "C" { void* cjexl_new_engine(); } @@ -72,7 +72,6 @@ void ExprManager::init(void) - /* **************************************************************************** * * ExprManager::evaluate - diff --git a/src/lib/rest/rest.cpp b/src/lib/rest/rest.cpp index f09fd37529..f0ce3fc4c0 100644 --- a/src/lib/rest/rest.cpp +++ b/src/lib/rest/rest.cpp @@ -618,8 +618,8 @@ static void requestCompleted clock_addtime(&accTimeStat.mongoWriteWaitTime, &threadLastTimeStat.mongoWriteWaitTime); clock_addtime(&accTimeStat.mongoReadWaitTime, &threadLastTimeStat.mongoReadWaitTime); clock_addtime(&accTimeStat.mongoCommandWaitTime, &threadLastTimeStat.mongoCommandWaitTime); - clock_addtime(&accTimeStat.exprBasicCtxBldTime, &threadLastTimeStat.exprBasicCtxBldTime); - clock_addtime(&accTimeStat.exprBasicEvalTime, &threadLastTimeStat.exprBasicEvalTime); + clock_addtime(&accTimeStat.exprBasicCtxBldTime, &threadLastTimeStat.exprBasicCtxBldTime); + clock_addtime(&accTimeStat.exprBasicEvalTime, &threadLastTimeStat.exprBasicEvalTime); clock_addtime(&accTimeStat.exprJexlCtxBldTime, &threadLastTimeStat.exprJexlCtxBldTime); clock_addtime(&accTimeStat.exprJexlEvalTime, &threadLastTimeStat.exprJexlEvalTime); clock_addtime(&accTimeStat.renderTime, &threadLastTimeStat.renderTime); diff --git a/src/lib/serviceRoutines/versionTreat.cpp b/src/lib/serviceRoutines/versionTreat.cpp index ecd3740e1f..1b71e28125 100644 --- a/src/lib/serviceRoutines/versionTreat.cpp +++ b/src/lib/serviceRoutines/versionTreat.cpp @@ -48,7 +48,7 @@ #include #ifndef EXPR_BASIC -// Interface to use libcjexl.a +// Interface to use libcjexl extern "C" { const char* cjexl_version(); } From fbde34e73bc50a01e63a11b9473a295a7723262a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 09:17:17 +0200 Subject: [PATCH 283/390] Update doc/manuals/orion-api.md --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index a294ce2d46..a75523cf91 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2988,7 +2988,7 @@ results in As failsafe behaviour, evaluation returns `null` in the following cases: * Some of the transformation used in the expression is unknown -* Operations with identifiers that are not defined in the context are used. For instance, `(A==null)?-1:A` will result in `null` (and not `-1`) if `A` is not in the context, due to `==` is an operation that cannot be done on undefined identifiers. However, `A||-1` will work (i.e. `-1` will result if `A` is not in the context), as `||` is not considered an operation on `A`. +* Operations with identifiers that are not defined in the context are used. For instance, `(A==null)?0:A` will result in `null` (and not `0`) if `A` is not in the context, due to `==` is an operation that cannot be done on undefined identifiers. However, `A||0` will work (i.e. `0` will result if `A` is not in the context), as `||` is not considered an operation on `A`. * Syntax error in the JEXL expression (e.g. `A[0|uppercase`) ### Known limitations From cfa543a5408363e6be32895bd27fe4b5d21a8d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 10:35:27 +0200 Subject: [PATCH 284/390] Update test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test --- .../4004_jexl_expressions_in_subs/jexl_transformation_full.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index d4b83278e1..977301f192 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -128,7 +128,7 @@ payload='{ }, "M": { "type": "Number", - "value": "${M|round(2)}" + "value": "${M|round}" }, "N": { "type": "Text", From 06cbe4e3506c56a67c81207a86cd29555dd58fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 30 May 2024 11:31:52 +0200 Subject: [PATCH 285/390] Update doc/manuals/orion-api.md Co-authored-by: mapedraza <40356341+mapedraza@users.noreply.github.com> --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index a75523cf91..0d81391493 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2987,7 +2987,7 @@ results in As failsafe behaviour, evaluation returns `null` in the following cases: -* Some of the transformation used in the expression is unknown +* Some of the transformation used in the expression is unknown (e.g. `A|undefinedExpression`) * Operations with identifiers that are not defined in the context are used. For instance, `(A==null)?0:A` will result in `null` (and not `0`) if `A` is not in the context, due to `==` is an operation that cannot be done on undefined identifiers. However, `A||0` will work (i.e. `0` will result if `A` is not in the context), as `||` is not considered an operation on `A`. * Syntax error in the JEXL expression (e.g. `A[0|uppercase`) From f3523924d055c1fb164e54d8ea978f8ef1b4d67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 30 May 2024 12:21:13 +0200 Subject: [PATCH 286/390] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 1a01755071..134c80ddc9 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,3 @@ -- Fix: lighter operation to get databases list from MongoDB (#4517) - Add: JEXL expression support in custom notification macro replacement (using cjexl 0.2.0) (#4004) - Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) - Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004) From 606e5c662b583616c695b23e6b87b81588d49004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 30 May 2024 18:03:03 +0200 Subject: [PATCH 287/390] FIX optimize CMakeList.txt --- CMakeLists.txt | 18 ++++++++++++------ src/lib/serviceRoutinesV2/CMakeLists.txt | 12 ++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0118dbef19..b28663ea95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,18 +234,24 @@ find_package (mongoc-1.0 1.24.3 EXACT) find_library (HAVE_CJEXL cjexl PATHS /usr/lib /usr/lib64 /usr/local/lib64 /usr/local/lib) if (HAVE_CJEXL) message("Using cjexl") + SET (COMMON_STATIC_LIBS + microhttpd.a + mosquitto.a + ${HAVE_CJEXL} + mongo::mongoc_static + ) else (HAVE_CJEXL) message("Not using cjexl") add_definitions(-DEXPR_BASIC) + SET (COMMON_STATIC_LIBS + microhttpd.a + mosquitto.a + mongo::mongoc_static + ) endif (HAVE_CJEXL) # Static libs common to contextBroker and unitTest binaries -SET (COMMON_STATIC_LIBS - microhttpd.a - mosquitto.a - ${HAVE_CJEXL} - mongo::mongoc_static -) + SET (DYNAMIC_LIBS curl diff --git a/src/lib/serviceRoutinesV2/CMakeLists.txt b/src/lib/serviceRoutinesV2/CMakeLists.txt index a7b3c7c89d..e32032732f 100644 --- a/src/lib/serviceRoutinesV2/CMakeLists.txt +++ b/src/lib/serviceRoutinesV2/CMakeLists.txt @@ -55,12 +55,12 @@ getRegistrations.cpp optionsGetOnly.cpp optionsGetPostOnly.cpp postRegistration.cpp -optionsGetDeleteOnly -optionsAllNotDelete -optionsGetPutOnly -optionsGetPutDeleteOnly -optionsGetDeletePatchOnly -optionsPostOnly +optionsGetDeleteOnly.cpp +optionsAllNotDelete.cpp +optionsGetPutOnly.cpp +optionsGetPutDeleteOnly.cpp +optionsGetDeletePatchOnly.cpp +optionsPostOnly.cpp serviceRoutinesCommon.cpp ) From 2cf28d06671e84c4498249bcbf9b2e741d21e21b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 31 May 2024 09:39:34 +0200 Subject: [PATCH 288/390] REMOVE publishimage-fiware.yml --- .github/workflows/publishimage-fiware.yml | 72 ----------------------- 1 file changed, 72 deletions(-) delete mode 100644 .github/workflows/publishimage-fiware.yml diff --git a/.github/workflows/publishimage-fiware.yml b/.github/workflows/publishimage-fiware.yml deleted file mode 100644 index 07f10beb01..0000000000 --- a/.github/workflows/publishimage-fiware.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Publish pre Image from master in FIWARE dockerhub - -# The workflow will push PRE images from master on every merge to the fiware dockerhub organization. The images will be tagged with PRE and the next minor increase on the -# semver(based on the github releases) -# It will NOT produce releases and release images or the 'latest' tag from master. Both (releases and 'latest' tag) rely on the -# dockerhub autobuild feature - - -on: - push: - branches: - - master - -# will produce tags in format fiware/orion:-PRE-, f.e. fiware/orion:2.6.0-PRE-12 -env: - IMAGE_NAME: fiware/orion - IMAGE_TAG_PRE: -PRE-${{ github.run_number }} - -jobs: - deploy-release: - - runs-on: ubuntu-22.04 - if: github.event_name == 'push' - - steps: - - - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Get Latest Release - id: latest_version - uses: abatilo/release-info-action@v1.3.0 - with: - owner: telefonicaid - repo: fiware-orion - - - name: Echo version - run: | - echo "Version ${{ steps.latest_version.outputs.latest_tag }}" - - - name: Increment version - id: increment-semver-minor - env: - LATEST_RELEASE: ${{ steps.latest_version.outputs.latest_tag }} - run: | - version=$LATEST_RELEASE - a=( ${version//./ } ) - ((++a[1])) - a[2]=0 - echo "${a[0]}.${a[1]}.${a[2]}" - version=$(echo "${a[0]}.${a[1]}.${a[2]}") - echo "::set-output name=version::${version}" - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 - with: - load: true - tags: | - ${{ env.IMAGE_NAME }}:${{ steps.increment-semver-minor.outputs.version }}${{ env.IMAGE_TAG_PRE }} - file: docker/Dockerfile - - - name: Push - run: docker push ${{ env.IMAGE_NAME }}:${{ steps.increment-semver-minor.outputs.version }}${{ env.IMAGE_TAG_PRE }} From 891c097fed0074d6c215d0bc849c4d60ad6240be Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Fri, 31 May 2024 19:17:14 +0900 Subject: [PATCH 289/390] (JP) ADD documentation about JEXL expressions in custom notifications --- doc/manuals.jp/admin/build_source.md | 2 + doc/manuals.jp/admin/statistics.md | 35 +- doc/manuals.jp/devel/sourceCode.md | 7 + doc/manuals.jp/orion-api.md | 794 ++++++++++++++++++++++++++- 4 files changed, 805 insertions(+), 33 deletions(-) diff --git a/doc/manuals.jp/admin/build_source.md b/doc/manuals.jp/admin/build_source.md index 2797e3f5b3..b5464ae4ea 100644 --- a/doc/manuals.jp/admin/build_source.md +++ b/doc/manuals.jp/admin/build_source.md @@ -4,6 +4,8 @@ Orion Context Broker のリファレンス配布は Debian 12 です。これは 公式以外のディストリビューションで Docker コンテナ・イメージをビルドする方法は、Docker ドキュメントの [3.1 非公式ディストリビューションでのビルド](../../../docker/README.jp.md#31-building-in-not-official-distributions)・セクションで確認できます。 +*注:* このドキュメントで説明されているビルド プロセスには cjexl ライブラリは含まれていません。これは、基本的なビルド プロセスの観点からはオプションであると見なされているためです。 + ## Debian 12 (正式サポート) Orion Context Broker は、以下のライブラリをビルドの依存関係として使用します : diff --git a/doc/manuals.jp/admin/statistics.md b/doc/manuals.jp/admin/statistics.md index 2f25673ebb..bcf301f0be 100644 --- a/doc/manuals.jp/admin/statistics.md +++ b/doc/manuals.jp/admin/statistics.md @@ -115,20 +115,22 @@ SemWait ブロックは、メインの内部セマフォの累積待ち時間を "timing": { "accumulated": { "jsonV1Parse": 7.860908311, - "mongoBackend": 416.796091597, - "mongoReadWait": 4656.924425628, - "mongoWriteWait": 259.347915990, - "mongoCommandWait": 0.514811318, - "render": 108.162782114, - "total": 6476.593504743 - }, + "jsonV2Parse": 120.680244446, + "mongoBackend": 12778.52734375, + "mongoReadWait": 7532.301757812, + "mongoWriteWait": 3619.282226562, + "mongoCommandWait": 0.120559767, + "exprJexlCtxBld": 27.092681885, + "exprJexlEval": 124.217208862, + "render": 44.540554047, + "total": 25051.384765625 + }, "last": { - "mongoBackend": 0.014752309, - "mongoReadWait": 0.012018445, - "mongoWriteWait": 0.000574611, - "render": 0.000019136, - "total": 0.015148915 - } + "mongoBackend": 0.003775352, + "mongoReadWait": 0.0013743, + "render": 0.000286864, + "total": 0.00440685 + } } ... } @@ -147,8 +149,13 @@ SemWait ブロックは、メインの内部セマフォの累積待ち時間を * `mongoBackend` : mongoBackend モジュールで渡された時間です (疑似セルフタイム) * `render` : レンダリングモジュールに渡された時間です (擬似セルフタイム) * `mongo*Wait``Read`, `Write` または `Cmd` オペレーションのために MongoDB を待っている時間です。与えられた要求が MongoDB への複数の read/write/cmd の呼び出しを含む場合、`last` 下の `mongo*Wait` に示された時間は、それらすべてのための蓄積を含むことに注意してください。mongoReadWait の場合、結果カーソルを取得するために使用された時間のみが考慮されますが、カーソル結果を処理する時間 (mongoBackend カウンタに属する時間) は考慮されません +* `exprJexlCtxBld`: カスタム通知式の評価のためのコンテキストの構築にかかった時間 ([マクロ置換](../orion-api.md#macro-substitution) および [JEXL サポート](../orion-api.md#jexl-support) を参照) +* `exprJexlEval`: カスタム通知式の評価にかかった時間 ([マクロ置換](../orion-api.md#macro-substitution) および [JEXL サポート](../orion-api.md#jexl-support) を参照) + +*注*: Orion バイナリが cjexl を使用せずにビルドされ、基本的な置換のみが使用可能な場合、`exprJexlCtxBld` および `exprJexlEval` の代わりに +`exprBasicCtxtBld` フィールドと `exprBasicEval` フィールドが表示されます。 -時間は、特定のスレッド・リクエストがモジュールの使用を開始し、使用を終了するまでの時間から測定されます。したがって、何らかの理由でスレッドが停止した場合 (カーネルがそのスケジューリングポリシーに基づいて別のスレッドに優先順位を与えることを決定した場合)、スレッドがスリープしていた時間が再び実行を待っている時間が測定に含まれているため、正確ではありません。このため、擬似 selt/end-to-end 時間と言っています。しかし、低負荷条件下では、この状況は重大な影響を及ぼさないと予想されます +時間は、特定のスレッド・リクエストがモジュールの使用を開始し、使用を終了するまでの時間から測定されます。したがって、何らかの理由でスレッドが停止した場合 (カーネルがそのスケジューリングポリシーに基づいて別のスレッドに優先順位を与えることを決定した場合)、スレッドがスリープしていた時間が再び実行を待っている時間が測定に含まれているため、正確ではありません。このため、擬似 self/end-to-end 時間と言っています。しかし、低負荷条件下では、この状況は重大な影響を及ぼさないと予想されます ### NotifQueue ブロック diff --git a/doc/manuals.jp/devel/sourceCode.md b/doc/manuals.jp/devel/sourceCode.md index d2deec1157..0c8186a8b8 100644 --- a/doc/manuals.jp/devel/sourceCode.md +++ b/doc/manuals.jp/devel/sourceCode.md @@ -24,6 +24,7 @@ * [src/lib/cache/](#srclibcache) (サブスクリプション・キャッシュの実装) * [src/lib/logSummary/](#srcliblogsummary) (ログ・サマリの実装) * [src/lib/metricsMgr/](#srclibmetricsmgr) (メトリック・マネージャの実装) +* [src/lib/expressions/](#srclibexpressions) (カスタム通知式のサポート) ## src/app/contextBroker/ @@ -540,3 +541,9 @@ NGSIv2 GET サブスクリプションのリクエストはサブスクリプシ プラットフォーム全体で同様のメトリックを使用するには、一般的なメトリックが考案され、Orion の場合は、この目的のためにマネージャが実装されました。この メトリック・マネージャは、ライブラリ **metricsMgr** にあります。メトリックについては、[このドキュメント](../admin/metrics_api.md)を参照してください。 [トップ](#top) + +## src/lib/expressions/ + +[カスタム通知で使用されるマクロ置換ロジック](../orion-api.md#macro-substitution) のサポートを提供します。このライブラリは、式評価の抽象化を提供し、JEXL ベースと基本置換ベースの 2 つの実装を提供します (使用する実装は、cjex ライブラリの可用性に基づいて、ビルド時に選択されます)。 + +[トップ](#top) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index c7684fec8c..0221a4d105 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -73,6 +73,35 @@ - [NGSI ペイロードのパッチ適用 (NGSI payload patching)](#ngsi-payload-patching) - [ペイロードの省略 (Omitting payload)](#omitting-payload) - [その他の考慮事項](#additional-considerations) + - [JEXL サポート (JEXL Support)](#jexl-support) + - [JEXL 使用例 (JEXL usage example)](#jexl-usage-example) + - [利用可能な変換 (Available Transformations)](#available-transformations) + - [`uppercase`](#uppercase) + - [`lowercase`](#lowercase) + - [`split`](#split) + - [`indexOf`](#indexOf) + - [`len`](#len) + - [`trim`](#trim) + - [`substring`](#substring) + - [`includes`](#includes) + - [`isNaN`](#isNaN) + - [`parseInt`](#parseInt) + - [`parseFloat`](#parseFloat) + - [`typeOf`](#typeOf) + - [`toString`](#toString) + - [`floor`](#floor) + - [`ceil`](#ceil) + - [`round`](#round) + - [`toFixed`](#toFixed) + - [`log`](#log) + - [`log10`](#log10) + - [`log2`](#log2) + - [`sqrt`](#sqrt) + - [`replaceStr`](#replaceStr) + - [`mapper`](#mapper) + - [`thMapper`](#thmapper) + - [フェイルセーフ・ケース (Failsafe cases)](#failsafe-cases) + - [既知の制限 (Known limitations)](#known-limitations) - [Oneshot サブスクリプション (Oneshot Subscriptions)](#oneshot-subscriptions) - [カバード・サブスクリプション (Covered subscriptions)](#covered-subscriptions) - [変更タイプに基づくサブスクリプション (Subscriptions based in alteration type)](#subscriptions-based-in-alteration-type) @@ -391,11 +420,12 @@ Orion で使用されるメタデータ更新セマンティクス (および関 GET /v2/entities/E%253C01%253E ``` -上記の制限が適用されない例外的なケースがいくつかあります。 特に、次のフィールドで: +上記の制限が適用されない例外的なケースがいくつかあります。 特に、次のケースで: - URL パラメータ `q` は、[シンプル・クエリ言語](#simple-query-language) に必要な特殊文字を許可します - URL パラメータ `mq` は、[シンプル・クエリ言語](#simple-query-language) に必要な特殊文字を許可します - URL パラメータ `georel` と `coords` は `;` を許可します +- [NGSI ペイロード パッチ](#ngsi-payload-patching) の `ngsi` (つまり `id`、`type`、および属性値) 内 ([JEXL 式構文](#jexl-support) で使用される文字をサポートするため) - 属性タイプとして `TextUnrestricted` を使用する属性値 ([特別な属性タイプ](#special-attribute-types)のセクションを参照) @@ -422,6 +452,8 @@ GET /v2/entities/E%253C01%253E クライアントが構文の観点から無効なフィールドを使用しようとすると、クライアントは、原因を説明する "Bad Request" エラーの レスポンスを受け取ります。 +`:` と `-` は識別子で使用できますが、[JEXL 構文](#jexl-support) と衝突するため、使用は強く推奨されません。特に、`-` は減算演算 (例: `${A-B}`) に使用され、`:` は三項演算子 (例: `A?'A is true':'A is false`) に使用されます。したがって、式 `${lower-temperature}` 内の属性名 `lower-temperature` は、`lower` 属性の値から `temperature` 属性を引いた値として解釈されます (`lower-temperature` という名前の属性の値として解釈されるわけではありません)。 + ## エラー・レスポンス (Error Responses) @@ -2008,22 +2040,22 @@ NGSIv1 は非推奨であることに注意してください。したがって - `payload`, `json`, `ngsi` (すべてペイロード関連のフィールド) - `topic` -テンプレートのマクロ置換は、構文 `${..}` に基づいています。特に: +テンプレートのマクロ置換は、構文 `${}` に基づいています。JEXL のサポートについては、 +[JEXL サポート](#jexl-support) セクションで説明されています。JEXL 式によって評価されるコンテキストには、 +次の識別子が含まれます: -- `${id}` は、エンティティの `id` に置き換えられます -- `${type}` は、エンティティの `type` に置き換えられます -- `${service}` は、サブスクリプションをトリガーする更新リクエストのサービス (つまり、`Fiware-Service` ヘッダ値) - に置き換えられます -- `${servicePath}` は、サブスクリプションをトリガーする更新リクエストのサービス パス (つまり、`Fiware-Servicepath` - ヘッダ値) に置き換えられます -- `${authToken}` は、サブスクリプションをトリガーする更新リクエストで認証トークン (つまり、`x-auth-token` ヘッダ値) - に置き換えられます -- 他の `${token}` は、名前が `token` の属性の値に置き換えられます。属性が通知に含まれていない場合は空文字列に - 置き換えられます。値が数値、bool または null の場合、その文字列表現が使用されます。値が JSON 配列または - オブジェクトの場合、JSON 表現は文字列として使用されます +- `id`: エンティティの `id` +- `type`: エンティティの `type` +- `service`: サブスクリプションをトリガーする更新リクエスト内のサービス + (つまり、`fiware-service` ヘッダ値) +- `servicePath`: サブスクリプションをトリガーする更新リクエスト内のサービス・パス + (つまり、`fiware-servicepath` ヘッダ値) +- `authToken: サブスクリプションをトリガーする更新リクエスト内の認証トークン + (つまり、`x-auth-token` ヘッダ値) +- 通知をトリガーするエンティティ内のすべての属性（通知をトリガーする更新に含まれているかどうか） -まれに、属性が `${service}`, `${servicePath}` または `${authToken}` と同じ方法で命名された場合 (例: `service` -という名前の属性)、属性値が優先されます。 +まれに、属性の名前が `service`、`servicePath`、または `authToken` と同じ名前になっている場合 +(名前が `service` である属性など)、属性値が優先されます。 例: @@ -2293,6 +2325,734 @@ the value of the "temperature" attribute (of type Number) is 23.4 `Ngsiv2-AttrsFormat` ヘッダは `custom` に設定されます。ただし、NGSI patch が使用される場合 (つまり、`ngsi` フィールド)、 通常の通知と同様に、`Ngsiv2-AttrsFormat: normalized` が使用されることに注意してください (通知形式が実際には同じである場合) + + +## JEXL サポート (JEXL Support) + +Orion Context Brokerは、カスタム通知の[マクロ置換](#macro-substitution)で[JEXL式](https://github.com/TomFrost/Jexl)をサポートしています。したがって、次のようなサブスクリプションを定義できます: + +``` +"httpCustom": { + ... + "ngsi": { + "relativeHumidity": { + "value": "${humidity/100}", + "type": "Calculated" + } + } +} +``` + +したがって、特定の更新によってエンティティの `humidity` 属性が `84.4` に設定された場合、通知には値 `0.844` の `relativeHumidity` 属性が含まれます。 + +式の特殊なケースとしては、式が特定のコンテキスト識別子であり、実際の式でそれを使用しないケースがあります。たとえば、次のようになります: + +``` +"httpCustom": { + ... + "ngsi": { + "originalHumidity": { + "value": "${humidity}", + "type": "Calculated" + } + } +} +``` + +このケースを *基本置換* (basic replacement) とも呼びます。 + +JEXL 式をテストするための便利なリソースは [JEXL プレイグラウンド](https://czosel.github.io/jexl-playground) です。ただし、[既知の制限](#known-limitations) セクションで説明されているように、JavaScript での元の JEXL 実装と Orion に含まれる実装の違いを考慮してください。 + +Orion は、この機能を提供するために cjexl ライブラリに依存しています。Orion バイナリが cjexl を使用せずにビルドされている場合、基本置換機能のみが利用できます。 + + + +### JEXL 使用例 (JEXL usage example) + +例として、次のようなサブスクリプションを考えてみましょう: + +``` +"httpCustom": { + ... + "ngsi": { + "speed": { + "value": "${(speed|split(' '))[0]|parseInt}", + "type": "Calculated" + }, + "ratio": { + "value": "${count.sum/count.count}", + "type": "Calculated" + }, + "code": { + "value": "${code||'invalid'}", + "type": "Calculated" + }, + "alert": { + "value": "${(value>max)?'nok':'ok'}", + "type": "Calculated" + }, + "count": { + "value": "${{count:count.count+1, sum:count.sum+((speed|split(' '))[0]|parseInt)}}", + "type": "Calculated" + } +} +``` + +エンティティの更新は次のようになります: + +``` +{ + ... + "speed": { + "value": "10 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 100 + }, + "type": "StructuredValue" + }, + "code": { + "value": null, + "type": "Number" + }, + "value": { + "value": 14, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +} +``` + +次のような通知がトリガーされます: + +``` +"data": [ + { + ... + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 10 + }, + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 20 + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": "invalid" + }, + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "ok" + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 110 + } + } + } +] +``` + +新しいエンティティの更新は次のようになります: + +``` +{ + ... + "speed": { + "value": "30 m/s", + "type": "Text" + }, + "count": { + "value": { + "count": 5, + "sum": 500 + }, + "type": "StructuredValue" + }, + "code": { + "value": 456, + "type": "Number" + }, + "value": { + "value": 75, + "type": "Number" + }, + "max": { + "value": 50, + "type": "Number" + } +} +``` + +次のような通知が表示されます: + +``` +"data": [ + { + ... + "speed": { + "metadata": {}, + "type": "Calculated", + "value": 30 + }, + "ratio": { + "metadata": {}, + "type": "Calculated", + "value": 100 + }, + "code": { + "metadata": {}, + "type": "Calculated", + "value": 456 + }, + "alert": { + "metadata": {}, + "type": "Calculated", + "value": "nok" + }, + "count": { + "metadata": {}, + "type": "Calculated", + "value": { + "count": 6, + "sum": 530 + } + } + } +] +``` + + + +### 利用可能な変換 (Available Transformations) + + + +#### `uppercase` + +文字列を大文字に変換します。 + +追加引数: なし + +例 (コンテキスト `{"c": "fooBAR"}`): + +``` +c|uppercase +``` + +結果 + +``` +"FOOBAR" +``` + + + +#### lowercase + +文字列を小文字に変換します。 + +追加引数: なし + +例 (コンテキスト `{"c": "fooBAR"}`): + +``` +c|lowercase +``` + +結果 + +``` +"foobar" +``` + + + +#### split + +入力文字列を配列項目に分割します。 + +追加引数: 分割に使用する区切り文字 + +例 (コンテキスト `{"c": "foo,bar,zzz"}`): + +``` +c|split(',') +``` + +結果 + +``` +[ "foo", "bar", "zzz" ] +``` + + + +#### indexOf + +入力文字列内の指定された文字列の位置を提供します。文字列が見つからない場合は、`null` を返します。 + +追加引数: 検索する入力文字列 + +この関数は、入力が配列の場合は機能しないことに注意してください (文字列に対してのみ機能します)。 + +例 (コンテキスト `{"c": "fooxybar"}`): + +``` +c|indexOf('xy') +``` + +結果 + +``` +3 +``` + + + +#### len + +文字列の長さを提供します。 + +追加引数: なし + +この関数は、入力が配列の場合は機能しないことに注意してください (文字列に対してのみ機能します)。 + +例 (コンテキスト `{"c": "foobar"}`): + +``` +c|len +``` + +結果 + +``` +6 +``` + + + +#### trim + +先頭と末尾の空白を削除します。 + +追加引数: なし + +例 (コンテキスト `{"c": " foo bar "}`): + +``` +c|trim +``` + +結果 + +``` +foo bar +``` + + + +#### substring + +2つの位置の間の部分文字列を返します。 + +追加引数: +* 開始位置 +* 終了位置 + +例 (コンテキスト `{"c": "foobar"}`): + +``` +c|substring(3,5) +``` + +結果 + +``` +ba +``` + + + +#### includes + +指定された文字列が入力文字列に含まれている場合は `true` を返し、含まれていない場合は `false` を返します。 + +追加引数: 検索する入力文字列 + +例 (コンテキスト `{"c": "foobar"}`): + +``` +c|includes('ba') +``` + +結果 + +``` +true +``` + + + +#### isNaN + +入力が数値でない場合は `true` を返し、それ以外の場合は `false` を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": "foobar"}`): + +``` +c|isNaN +``` + +結果 + +``` +true +``` + + + +#### parseInt + +文字列を解析し、対応する整数を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": "25"}`): + +``` +c|parseInt +``` + +結果 + +``` +25 +``` + + + +#### parseFloat + +文字列を解析し、対応する浮動小数点数を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": "25.33"}`): + +``` +c|parseFloat +``` + +結果 + +``` +25.33 +``` + + + +#### typeOf + +入力の型を含む文字列を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": 23}`): + +``` +c|typeOf +``` + +結果 + +``` +"Number" +``` + + + +#### toString + +Return a string representation of the input. + +追加引数: なし + +例 (コンテキスト `{"c": 23}`): + +``` +c|toString +``` + +結果 + +``` +"23" +``` + + + +#### floor + +指定された数値に最も近い小さい整数を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": 3.14}`): + +``` +c|floor +``` + +結果 + +``` +3 +``` + + + +#### ceil + +指定された数値に最も近い上位の整数を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": 3.14}`): + +``` +c|ceil +``` + +結果 + +``` +4 +``` + + + +#### round + +指定された数値に最も近い整数（下限または上限）を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": 3.14}`): + +``` +c|round +``` + +結果 + +``` +3 +``` + + + +#### toFixed + +数値を小数点以下の桁数に丸めます。 + +追加引数: 小数点以下の桁数 + +例 (コンテキスト `{"c": 3.18}`): + +``` +c|toFixed(1) +``` + +結果 + +``` +3.2 +``` + + + +#### log + +与えられた数値の自然対数を返す。 + +追加引数: なし + +例 (コンテキスト `{"c": 3.14}`): + +``` +c|log +``` + +結果 + +``` +1.144222799920162 +``` + + + +#### log10 + +与えられた数値の10を底とする対数を返す。 + +追加引数: なし + +例 (コンテキスト `{"c": 3.14}`): + +``` +c|log10 +``` + +結果 + +``` +0.49692964807321494 +``` + + + +#### log2 + +指定された数値の 2 を底とする対数を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": 3.14}`): + +``` +c|log2 +``` + +結果 + +``` +1.6507645591169025 +``` + + + +#### sqrt + +指定された数値の平方根を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": 3.14}`): + +``` +c|sqrt +``` + +結果 + +``` +1.772004514666935 +``` + + + +#### replaceStr + +入力文字列内の文字列を別の文字列に置き換えます。 + +追加引数: +* 置換するソース文字列 +* 置換先の文字列 + +例 (コンテキスト `{"c": "foobar"}`): + +``` +c|replaceStr('o','u') +``` + +結果 + +``` +"fuubar" +``` + + + +#### mapper + +1 対 1 のマッピングに基づいて、複数の選択肢の中から値を返します。この関数は、値の配列 (*value*) と選択肢の配列 (*choices*) に基づいています (長さはまったく同じ) 。したがって、入力値が *values* の *i* 番目の項目と等しい場合は、*choices* の *i* 番目の項目が返されます。 + +この変換は、引数に何らかの問題が見つかった場合 (つまり、入力が値の中に見つからない、選択肢の長さが値とまったく同じではない、入力が文字列ではないなど)、`null` を返します。 + +追加引数: +* 値の配列 (values array) +* 選択肢の配列 (choices array) + +例 (コンテキスト `{"c": "fr", "values": ["es", "fr", "de"], "choices": ["Spain", "France", "Germany"]}`): + +``` +c|mapper(values,choices) +``` + +結果 + +``` +"France" +``` + + + +#### thMapper + +しきい値に基づいて、複数の選択肢の中から値を返します。この関数は、値の配列 (*values*) と選択肢の配列 (*choices*) に基づいています (長さは、値に 1 を加えた値とまったく同じです)。したがって、入力値が *values* の *i* 番目と *i+1* 番目の項目の間にある場合は、*choices* の *i*+1 番目の項目が返されます。 + +引数に問題が見つかった場合 (つまり、選択肢の長さが値に 1 を加えた値とまったく同じではない、値配列の一部の項目が数値ではないなど)、この変換は `null` を返します。 + +追加引数: +* 値の配列 (values array) +* 選択肢の配列 (choices array) + +例 (コンテキスト `{"c": 0.5, "values": [-1, 1], "choices": ["low", "medium", "high"]}`): + +``` +c|thMapper(values,choices) +``` + +結果 + +``` +"medium" +``` + + + +### フェイルセーフ・ケース (Failsafe cases) + +フェイルセーフ動作として、評価は次の場合に `null` を返します: + +* 式で使用されている変換の一部が不明です (例: `A|undefinedExpression`) +* コンテキストで定義されていない識別子を使用した演算が使用されています。たとえば、`(A==null)?0:A` は、`A` がコンテキストにない場合、`null` (`0` ではありません) になります。これは、`==` が未定義の識別子では実行できない演算であるためです。ただし、`||` は `A` に対する演算とは見なされないため、`A||0` は機能します (つまり、`A` がコンテキストにない場合は `0` になります) +* JEXL 式の構文エラーです (例: `A[0|uppercase`) + + + +### 既知の制限 (Known limitations) + +- 単位マイナス演算子が正しく動作しません。たとえば、次の式は動作しません (`null` にフェイルセーフされます): `A||-1`。ただし、次の代替式は動作します: `A||0-1` および `A||'-1'|parseInt)` +- 否定演算子 `!` (元の JavaScript JEXL でサポート) はサポートされていません + ## Oneshot サブスクリプション (Oneshot Subscriptions) @@ -2397,10 +3157,6 @@ EOF この場合、エンティティに存在するかどうかに関係なく、すべての属性が通知に含まれます。存在しないこれらの属性 (この例では `brightness`) には、`null` 値 (タイプ `"None"`) が使用されます。 -カスタム通知の場合、`covered` が `true` に設定されていると、`null` は、存在しない属性の `${...}` を置き換えるために -使用されます (`covered` が `true` に設定されていない場合のデフォルトの動作は、存在しない属性を空の文字列に -置き換えることです)。 - 通知が `notification.attrs` フィールドのすべての属性を完全に "カバーする" (covers) という意味で "カバーされる" (covered) という用語を使用します。これは、可変の属性セットに対して十分な柔軟性がなく、受信したすべての通知で常に同じ着信属性の セットを必要とする通知エンドポイントに役立ちます。 From a3400df82f2809b57741daf2400350dcb0dd3f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 3 Jun 2024 13:29:22 +0200 Subject: [PATCH 290/390] ADD new transformation --- .../jexl_transformation_full.test | 146 ++++++++++++++++-- 1 file changed, 137 insertions(+), 9 deletions(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index 977301f192..90e1078595 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -33,7 +33,7 @@ accumulatorStart --pretty-print # # 01. Create custom sub using all transformations -# 02. Create entity E1 with A to V attributes +# 02. Create entity E1 with A to AD attributes # 03. Dump accumulator and see expected notification # @@ -73,7 +73,15 @@ payload='{ "S", "T", "U", - "V" + "V", + "W", + "X", + "Y", + "Z", + "AA", + "AB", + "AC", + "AD" ], "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", @@ -136,7 +144,7 @@ payload='{ }, "O": { "type": "Text", - "value": "${Z||'\''Is null'\''}" + "value": "${ZZ||'\''Is null'\''}" }, "P": { "type": "Number", @@ -165,6 +173,38 @@ payload='{ "V": { "type": "Text", "value": "${V|thMapper([1,2],['\''low'\'','\''medium'\'','\''high'\''])}" + }, + "W": { + "type": "StructuredValue", + "value": "${W|toJson}" + }, + "X": { + "type": "Text", + "value": "${X|replaceRegex('\''[0-9]+'\'','\''x'\'')}" + }, + "Y": { + "type": "Text", + "value": "${Y|matchRegex('\''[0-9]+'\'')}" + }, + "Z": { + "type": "StructuredValue", + "value": "${Z|values}" + }, + "AA": { + "type": "StructuredValue", + "value": "${AA|keys}" + }, + "AB": { + "type": "Number", + "value": "${AB|arrSum}" + }, + "AC": { + "type": "Number", + "value": "${AC|arrAvg}" + }, + "AD": { + "type": "Number", + "value": "${AD|len}" } } } @@ -175,8 +215,8 @@ echo echo -echo "02. Create entity E1 with A to V attributes" -echo "===========================================" +echo "02. Create entity E1 with A to AD attributes" +echo "============================================" payload='{ "id": "E1", "type": "T", @@ -272,6 +312,38 @@ payload='{ "V": { "type": "Number", "value": 1.5 + }, + "W": { + "type": "Text", + "value": "[1,2,3]" + }, + "X": { + "type": "Text", + "value": "aba1234aba782" + }, + "Y": { + "type": "Text", + "value": "aba1234aba782" + }, + "Z": { + "type": "StructuredValue", + "value": {"x": 1, "y": {"y1": 2, "y2": 3}} + }, + "AA": { + "type": "StructuredValue", + "value": {"x": 1, "y": {"y1": 2, "y2": 3}} + }, + "AB": { + "type": "StructuredValue", + "value": [1, 2, 3] + }, + "AC": { + "type": "StructuredValue", + "value": [1, 2, 3] + }, + "AD": { + "type": "StructuredValue", + "value": [1, 2, 3] } }' orionCurl --url /v2/entities --payload "$payload" @@ -297,8 +369,8 @@ Content-Length: 0 -02. Create entity E1 with A to V attributes -=========================================== +02. Create entity E1 with A to AD attributes +============================================ HTTP/1.1 201 Created Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) @@ -311,7 +383,7 @@ Content-Length: 0 ================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 1178 +Content-Length: 1627 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -327,6 +399,29 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Text", "value": "NA3" }, + "AA": { + "metadata": {}, + "type": "StructuredValue", + "value": [ + "x", + "y" + ] + }, + "AB": { + "metadata": {}, + "type": "Number", + "value": 6 + }, + "AC": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "AD": { + "metadata": {}, + "type": "Number", + "value": 3 + }, "B": { "metadata": {}, "type": "Text", @@ -435,6 +530,39 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Text", "value": "medium" }, + "W": { + "metadata": {}, + "type": "StructuredValue", + "value": [ + 1, + 2, + 3 + ] + }, + "X": { + "metadata": {}, + "type": "Text", + "value": "abaxabax" + }, + "Y": { + "metadata": {}, + "type": "Text", + "value": [ + "1234", + "782" + ] + }, + "Z": { + "metadata": {}, + "type": "StructuredValue", + "value": [ + 1, + { + "y1": 2, + "y2": 3 + } + ] + }, "id": "E1", "type": "T" } @@ -446,5 +574,5 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 --TEARDOWN-- brokerStop CB -dbDrop CB +#dbDrop CB accumulatorStop From 199ada0ed0a4d47b12ddcb3771663ff127e3860b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 3 Jun 2024 13:53:51 +0200 Subject: [PATCH 291/390] ADD new ftest --- docker/Dockerfile | 2 +- .../jexl_transformation_binary_map_accum.test | 220 ++++++++++++++++++ .../jexl_transformation_full.test | 2 +- 3 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test diff --git a/docker/Dockerfile b/docker/Dockerfile index e6c7846b56..463a4e3034 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.2.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.3.0 ${REPO_ACCESS_TOKEN} && \ make && \ make install && \ # reduce size of installed binaries diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test new file mode 100644 index 0000000000..baa991c929 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test @@ -0,0 +1,220 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +JEXL basic expression in custom notification (binary map accumulation) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with several attributes with transformations +# 02. Create entity E1 +# 03. Update entity E1 +# 05. Dump accumulator and see two expected transformations +# + + +echo "01. Create custom sub with several attributes with transformations" +echo "==================================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "occupied": { + "value": "${slots|values|arrSum}", + "type": "Calculated" + }, + "free": { + "value": "${(slots|values|len)-(slots|values|arrSum)}", + "type": "Calculated" + } + } + }, + "attrs": [ "free", "occupied" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + +#"value": "${{count:count.count+1,sum:count.sum+speed|split('\'' '\'')[0]|parseInt}}", + +echo "02. Create entity E1" +echo "====================" +payload='{ + "id": "E1", + "type": "T", + "slots": { + "value": { + "s1": 0, + "s2": 1, + "s3": 0, + "s4": 1, + "s5": 0 + }, + "type": "Text" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1" +echo "====================" +payload='{ + "slots": { + "value": { + "s1": 0, + "s2": 1, + "s3": 1, + "s4": 1, + "s5": 0 + }, + "type": "Text" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see two expected transformations" +echo "=========================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with several attributes with transformations +================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 +==================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 +==================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see two expected transformations +========================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 187 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "free": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "id": "E1", + "occupied": { + "metadata": {}, + "type": "Calculated", + "value": 2 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 187 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "free": { + "metadata": {}, + "type": "Calculated", + "value": 2 + }, + "id": "E1", + "occupied": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index 90e1078595..12646463e7 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -574,5 +574,5 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 --TEARDOWN-- brokerStop CB -#dbDrop CB +dbDrop CB accumulatorStop From d62069fa3ebef69de37114949774bfa95f91bffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 3 Jun 2024 14:09:02 +0200 Subject: [PATCH 292/390] FIX cjexl 0.3.0 --- CHANGES_NEXT_RELEASE | 2 +- ci/deb/build.sh | 2 +- docker/Dockerfile.alpine | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 134c80ddc9..2728600db4 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.2.0) (#4004) +- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.3.0) (#4004) - Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) - Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004) - Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index b0e5027b8a..6202177dbb 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -175,7 +175,7 @@ rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - bash /opt/fiware-orion/get_cjexl.sh 0.2.0 $REPO_ACCESS_TOKEN + bash /opt/fiware-orion/get_cjexl.sh 0.3.0 $REPO_ACCESS_TOKEN fi if [ -n "${branch}" ]; then diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 9e35eb3aa9..736f2dbf3e 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -110,7 +110,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.2.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.3.0 ${REPO_ACCESS_TOKEN} && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ From d2f8cafb1f6b7b059a4406073b9c95b0c023e013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 3 Jun 2024 16:09:57 +0200 Subject: [PATCH 293/390] ADD docu --- doc/manuals/orion-api.md | 147 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 6 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 0d81391493..6cceed1a4a 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -79,27 +79,34 @@ - [`uppercase`](#uppercase) - [`lowercase`](#lowercase) - [`split`](#split) - - [`indexOf`](#indexOf) + - [`indexOf`](#indexof) - [`len`](#len) - [`trim`](#trim) - [`substring`](#substring) - [`includes`](#includes) - [`isNaN`](#isNaN) - - [`parseInt`](#parseInt) - - [`parseFloat`](#parseFloat) + - [`parseInt`](#parseint) + - [`parseFloat`](#parsefloat) - [`typeOf`](#typeOf) - - [`toString`](#toString) + - [`toString`](#tostring) + - [`toJson`](#tojson) - [`floor`](#floor) - [`ceil`](#ceil) - [`round`](#round) - - [`toFixed`](#toFixed) + - [`toFixed`](#tofixed) - [`log`](#log) - [`log10`](#log10) - [`log2`](#log2) - [`sqrt`](#sqrt) - - [`replaceStr`](#replaceStr) + - [`replaceStr`](#replacestr) + - [`replaceRegex`](#replaceregex) + - [`matchRegex`](#matchregex) - [`mapper`](#mapper) - [`thMapper`](#thmapper) + - [`values`](#values) + - [`keys`](#keys) + - [`arrSum`](#arrsum) + - [`arrAvg`](#arravg) - [Failsafe cases](#failsafe-cases) - [Known limitations](#known-limitations) - [Oneshot Subscriptions](#oneshot-subscriptions) @@ -2775,6 +2782,24 @@ results in "23" ``` +#### toJson + +Convert the input string to a JSON document. If the string is not a valid JSON, it returns `null`. + +Extra arguments: none. + +Example (being context `{"c": "[1,2]"}`): + +``` +c|toJson +``` + +results in + +``` +[1, 2] +``` + #### floor Return the closest lower integer of a given number. @@ -2939,6 +2964,44 @@ results in "fuubar" ``` +#### replaceRegex + +Replace tokens matching a given regex in an input string by another string. + +Extra arguments: +* Regex to match tokens to replace +* Destination string to replace + +Example (being context `{"c": "aba1234aba786aba"}`): + +``` +c|replaceRegex('\d+','X') +``` + +results in + +``` +"abaXabaXaba" +``` + +#### matchRegex + +Returns an array of tokens matching a given regular expression in input string. In the case of invalid regex, it returns `null`. If no matches are found, it returns the empty array (`[]`). + +Extra arguments: regular expression. + +Example (being context `{"c": "abc1234fgh897hyt"}`): + +``` +c|matchRegex('\d+`) +``` + +results in + +``` +["1234, "897"]] +``` + #### mapper Returns a value among several choices based in one to one mapping. This function is based in an array of *values* and an array of *choices* (which length is exactly the same). Thus, if the input value is equal to the *i*-th item of *values*, then *i*-th item of *choices* is returned. @@ -2983,6 +3046,78 @@ results in "medium" ``` +#### values + +Returns an array with the values of the keys of a given object (or `null` if input is not an object). + +Extra arguments: none + +Example (being context `{"c": {"x": 1, "y": "foo"}}`): + +``` +c|values +``` + +results in + +``` +[1,"foo"] +``` + +#### keys + +Returns an array with the keys of a given object (or `null` if input is not an object). + +Extra arguments: none + +Example (being context `{"c": {"x": 1, "y": "foo"}}`): + +``` +c|keys +``` + +results in + +``` +["x","y"] +``` + +#### arrSum + +Returns the sum of the elements of an array (or `null` if the input in an array or the array containts some not numberic item). + +Extra arguments: none + +Example (being context `{"c": [1, 5]}`): + +``` +c|arrSum +``` + +results in + +``` +6 +``` + +#### arrAvg + +Returns the average of the elements of an array (or `null` if the input in an array or the array containts some not numberic item). + +Extra arguments: none + +Example (being context `{"c": [1, 5]}`): + +``` +c|arrAvg +``` + +results in + +``` +3 +``` + ### Failsafe cases As failsafe behaviour, evaluation returns `null` in the following cases: From 2d71fb79cac3c4a8c36e2c69b8d7b8c6b4ddcb45 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 4 Jun 2024 12:19:36 +0900 Subject: [PATCH 294/390] Fix typos --- doc/manuals/orion-api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 6cceed1a4a..294fdd7dce 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -3084,7 +3084,7 @@ results in #### arrSum -Returns the sum of the elements of an array (or `null` if the input in an array or the array containts some not numberic item). +Returns the sum of the elements of an array (or `null` if the input in an array or the array contains some not numberic item). Extra arguments: none @@ -3102,7 +3102,7 @@ results in #### arrAvg -Returns the average of the elements of an array (or `null` if the input in an array or the array containts some not numberic item). +Returns the average of the elements of an array (or `null` if the input in an array or the array contains some not numberic item). Extra arguments: none From e9a0e773deaffc23d948b161f46b9777b8f6f2ff Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 4 Jun 2024 12:20:57 +0900 Subject: [PATCH 295/390] (JP) ADD documentation about new transformations (#4561) --- doc/manuals.jp/orion-api.md | 175 +++++++++++++++++++++++++++++++++--- 1 file changed, 162 insertions(+), 13 deletions(-) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index 0221a4d105..d815003198 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -79,27 +79,34 @@ - [`uppercase`](#uppercase) - [`lowercase`](#lowercase) - [`split`](#split) - - [`indexOf`](#indexOf) + - [`indexOf`](#indexof) - [`len`](#len) - [`trim`](#trim) - [`substring`](#substring) - [`includes`](#includes) - [`isNaN`](#isNaN) - - [`parseInt`](#parseInt) - - [`parseFloat`](#parseFloat) + - [`parseInt`](#parseint) + - [`parseFloat`](#parsefloat) - [`typeOf`](#typeOf) - - [`toString`](#toString) + - [`toString`](#tostring) + - [`toJson`](#tojson) - [`floor`](#floor) - [`ceil`](#ceil) - [`round`](#round) - - [`toFixed`](#toFixed) + - [`toFixed`](#tofixed) - [`log`](#log) - [`log10`](#log10) - [`log2`](#log2) - [`sqrt`](#sqrt) - - [`replaceStr`](#replaceStr) + - [`replaceStr`](#replacestr) + - [`replaceRegex`](#replaceregex) + - [`matchRegex`](#matchregex) - [`mapper`](#mapper) - [`thMapper`](#thmapper) + - [`values`](#values) + - [`keys`](#keys) + - [`arrSum`](#arrsum) + - [`arrAvg`](#arravg) - [フェイルセーフ・ケース (Failsafe cases)](#failsafe-cases) - [既知の制限 (Known limitations)](#known-limitations) - [Oneshot サブスクリプション (Oneshot Subscriptions)](#oneshot-subscriptions) @@ -2600,7 +2607,7 @@ c|split(',') [ "foo", "bar", "zzz" ] ``` - + #### indexOf @@ -2726,7 +2733,7 @@ c|isNaN true ``` - + #### parseInt @@ -2746,7 +2753,7 @@ c|parseInt 25 ``` - + #### parseFloat @@ -2786,11 +2793,11 @@ c|typeOf "Number" ``` - + #### toString -Return a string representation of the input. +入力の文字列表現を返します。 追加引数: なし @@ -2806,6 +2813,26 @@ c|toString "23" ``` + + +#### toJson + +入力文字列を JSON ドキュメントに変換します。文字列が有効な JSON でない場合は、`null` を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": "[1,2]"}`): + +``` +c|toJson +``` + +結果 + +``` +[1, 2] +``` + #### floor @@ -2866,7 +2893,7 @@ c|round 3 ``` - + #### toFixed @@ -2966,7 +2993,7 @@ c|sqrt 1.772004514666935 ``` - + #### replaceStr @@ -2988,6 +3015,48 @@ c|replaceStr('o','u') "fuubar" ``` + + +#### replaceRegex + +入力文字列内の指定された正規表現に一致するトークンを別の文字列に置き換えます。 + +追加引数: +* 置換するトークンに一致する正規表現 +* 置換先の文字列 + +例 (コンテキスト `{"c": "aba1234aba786aba"}`): + +``` +c|replaceRegex('\d+','X') +``` + +結果 + +``` +"abaXabaXaba" +``` + + + +#### matchRegex + +入力文字列内の指定された正規表現に一致するトークンの配列を返します。無効な正規表現の場合は、`null` を返します。一致するものが見つからない場合は、空の配列 (`[]`) を返します。 + +追加引数: 正規表現 + +例 (コンテキスト `{"c": "abc1234fgh897hyt"}`): + +``` +c|matchRegex('\d+`) +``` + +結果 + +``` +["1234, "897"]] +``` + #### mapper @@ -3036,6 +3105,86 @@ c|thMapper(values,choices) "medium" ``` + + +#### values + +指定されたオブジェクトのキーの値を含む配列を返します (入力がオブジェクトでない場合は `null`)。 + +追加引数: なし + +例 (コンテキスト `{"c": {"x": 1, "y": "foo"}}`): + +``` +c|values +``` + +結果 + +``` +[1,"foo"] +``` + + + +#### keys + +指定されたオブジェクトのキーを含む配列を返します (入力がオブジェクトでない場合は `null`)。 + +追加引数: なし + +例 (コンテキスト `{"c": {"x": 1, "y": "foo"}}`): + +``` +c|keys +``` + +結果 + +``` +["x","y"] +``` + + + +#### arrSum + +配列の要素の合計を返します (配列内の入力または配列に数値以外の項目が含まれている場合は `null` を返します)。 + +追加引数: なし + +例 (コンテキスト `{"c": [1, 5]}`): + +``` +c|arrSum +``` + +結果 + +``` +6 +``` + + + +#### arrAvg + +配列の要素の平均を返します (配列内の入力または配列に数値以外の項目が含まれている場合は `null` を返します)。 + +追加引数: なし + +例 (コンテキスト `{"c": [1, 5]}`): + +``` +c|arrAvg +``` + +結果 + +``` +3 +``` + ### フェイルセーフ・ケース (Failsafe cases) From c5482d9067e49a49ade84febd33739f414de5ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 May 2024 16:39:16 +0200 Subject: [PATCH 296/390] FIX remove NGSIv1 lasting API ops --- CMakeLists.txt | 2 +- src/app/contextBroker/orionRestServices.cpp | 121 +++++++++++--------- 2 files changed, 68 insertions(+), 55 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c93634bd5..f7632d895d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,8 +187,8 @@ ENDIF() SET (ORION_LIBS common - serviceRoutines serviceRoutinesV2 + serviceRoutines ngsiNotify rest # verbName(Verb) from setExtendedHttpInfo@MongoCommonSubscription.cpp.o; jsonRequestTreat from payloadParse@RestService.cpp.o; jsonParse diff --git a/src/app/contextBroker/orionRestServices.cpp b/src/app/contextBroker/orionRestServices.cpp index f29b9a897a..cf4eafc048 100644 --- a/src/app/contextBroker/orionRestServices.cpp +++ b/src/app/contextBroker/orionRestServices.cpp @@ -32,10 +32,12 @@ #include "serviceRoutines/leakTreat.h" #include "serviceRoutines/postDiscoverContextAvailability.h" -#include "serviceRoutines/postQueryContext.h" +/// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future +///#include "serviceRoutines/postQueryContext.h" #include "serviceRoutines/postRegisterContext.h" #include "serviceRoutines/postSubscribeContext.h" -#include "serviceRoutines/postUpdateContext.h" +/// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future +///#include "serviceRoutines/postUpdateContext.h" #include "serviceRoutines/postUpdateContextSubscription.h" #include "serviceRoutines/postUnsubscribeContext.h" #include "serviceRoutines/postNotifyContext.h" @@ -90,7 +92,8 @@ // FIXME: disable NGSI9 API routes in Orion 3.8.0, to be definetively removed at some point of the future // (along with badNgsi9Request.h|cpp files themselves) //#include "serviceRoutines/badNgsi9Request.h" -#include "serviceRoutines/badNgsi10Request.h" +/// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future +///#include "serviceRoutines/badNgsi10Request.h" #include "serviceRoutines/badRequest.h" #include "serviceRoutinesV2/badVerbAllNotDelete.h" @@ -212,7 +215,8 @@ static RestService getServiceVlegacy[] = //{ Ngsi10ContextEntityTypesAttribute, 5, { "ngsi10", "contextEntityTypes", "*", "attributes", "*" }, getNgsi10ContextEntityTypesAttribute }, //{ IndividualContextEntity, 3, { "v1", "contextEntities", "*" }, getIndividualContextEntity }, //{ IndividualContextEntityAttributes, 4, { "v1", "contextEntities", "*", "attributes" }, getIndividualContextEntity }, - { IndividualContextEntityAttribute, 5, { "v1", "contextEntities", "*", "attributes", "*" }, getIndividualContextEntityAttribute }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ IndividualContextEntityAttribute, 5, { "v1", "contextEntities", "*", "attributes", "*" }, getIndividualContextEntityAttribute }, //{ Ngsi10ContextEntityTypes, 3, { "v1", "contextEntityTypes", "*" }, getNgsi10ContextEntityTypes }, //{ Ngsi10ContextEntityTypesAttributeContainer, 4, { "v1", "contextEntityTypes", "*", "attributes" }, getNgsi10ContextEntityTypes }, //{ Ngsi10ContextEntityTypesAttribute, 5, { "v1", "contextEntityTypes", "*", "attributes", "*" }, getNgsi10ContextEntityTypesAttribute }, @@ -285,8 +289,9 @@ static RestService postServiceVlegacy[] = // FIXME: disable NGSI9 API routes in Orion 3.8.0, to be definetively removed at some point of the future //{ RegisterContext, 2, { "ngsi9", "registerContext" }, postRegisterContext }, //{ DiscoverContextAvailability, 2, { "ngsi9", "discoverContextAvailability" }, postDiscoverContextAvailability }, - { UpdateContext, 2, { "v1", "updateContext" }, (RestTreat) postUpdateContext }, - { QueryContext, 2, { "v1", "queryContext" }, postQueryContext }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ UpdateContext, 2, { "v1", "updateContext" }, (RestTreat) postUpdateContext }, + ///{ QueryContext, 2, { "v1", "queryContext" }, postQueryContext }, // FIXME: disable unused NGSv1 API routes in Orion 3.9.0, to be definetively removed at some point of the future //{ SubscribeContext, 2, { "v1", "subscribeContext" }, postSubscribeContext }, //{ UpdateContextSubscription, 2, { "v1", "updateContextSubscription" }, postUpdateContextSubscription }, @@ -319,8 +324,9 @@ static RestService postServiceVlegacy[] = //{ IndividualContextEntityAttributeWithTypeAndId, 8, { "v1", "contextEntities", "type", "*", "id", "*", "attributes", "*" }, postIndividualContextEntityAttributeWithTypeAndId }, //{ ContextEntitiesByEntityIdAndType, 7, { "v1", "registry", "contextEntities", "type", "*", "id", "*" }, postContextEntitiesByEntityIdAndType }, //{ EntityByIdAttributeByNameIdAndType, 9, { "v1", "registry", "contextEntities", "type", "*", "id", "*", "attributes", "*" }, postEntityByIdAttributeByNameWithTypeAndId }, - { UpdateContext, 2, { "ngsi10", "updateContext" }, (RestTreat) postUpdateContext }, - { QueryContext, 2, { "ngsi10", "queryContext" }, postQueryContext }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ UpdateContext, 2, { "ngsi10", "updateContext" }, (RestTreat) postUpdateContext }, + ///{ QueryContext, 2, { "ngsi10", "queryContext" }, postQueryContext }, //{ SubscribeContext, 2, { "ngsi10", "subscribeContext" }, postSubscribeContext }, //{ UpdateContextSubscription, 2, { "ngsi10", "updateContextSubscription" }, postUpdateContextSubscription }, //{ UnsubscribeContext, 2, { "ngsi10", "unsubscribeContext" }, postUnsubscribeContext }, @@ -355,7 +361,8 @@ static RestService putServiceVlegacy[] = //{ IndividualContextEntityAttributes, 4, { "ngsi10", "contextEntities", "*", "attributes" }, putIndividualContextEntity }, //{ IndividualContextEntityAttribute, 5, { "ngsi10", "contextEntities", "*", "attributes", "*" }, putIndividualContextEntityAttribute }, //{ Ngsi10SubscriptionsConvOp, 3, { "ngsi10", "contextSubscriptions", "*" }, putSubscriptionConvOp }, - { IndividualContextEntity, 3, { "v1", "contextEntities", "*" }, putIndividualContextEntity }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ IndividualContextEntity, 3, { "v1", "contextEntities", "*" }, putIndividualContextEntity }, //{ IndividualContextEntityAttributes, 4, { "v1", "contextEntities", "*", "attributes" }, putIndividualContextEntity }, //{ IndividualContextEntityAttribute, 5, { "v1", "contextEntities", "*", "attributes", "*" }, putIndividualContextEntityAttribute }, //{ Ngsi10SubscriptionsConvOp, 3, { "v1", "contextSubscriptions", "*" }, putSubscriptionConvOp }, @@ -416,7 +423,8 @@ static RestService deleteServiceVlegacy[] = //{ IndividualContextEntityAttributes, 4, { "ngsi10", "contextEntities", "*", "attributes" }, deleteIndividualContextEntity }, //{ IndividualContextEntityAttribute, 5, { "ngsi10", "contextEntities", "*", "attributes", "*" }, deleteIndividualContextEntityAttribute }, //{ Ngsi10SubscriptionsConvOp, 3, { "ngsi10", "contextSubscriptions", "*" }, deleteSubscriptionConvOp }, - { IndividualContextEntity, 3, { "v1", "contextEntities", "*" }, deleteIndividualContextEntity }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ IndividualContextEntity, 3, { "v1", "contextEntities", "*" }, deleteIndividualContextEntity }, //{ IndividualContextEntityAttributes, 4, { "v1", "contextEntities", "*", "attributes" }, deleteIndividualContextEntity }, //{ IndividualContextEntityAttribute, 5, { "v1", "contextEntities", "*", "attributes", "*" }, deleteIndividualContextEntityAttribute }, //{ Ngsi10SubscriptionsConvOp, 3, { "v1", "contextSubscriptions", "*" }, deleteSubscriptionConvOp }, @@ -496,17 +504,19 @@ static RestService badVerbVlegacy[] = // FIXME: disable NGSI9 API routes in Orion 3.8.0, to be definetively removed at some point of the future //{ RegisterContext, 2, { "ngsi9", "registerContext" }, badVerbPostOnly }, //{ DiscoverContextAvailability, 2, { "ngsi9", "discoverContextAvailability" }, badVerbPostOnly }, - { RegisterContext, 3, { "v1", "registry", "registerContext" }, badVerbPostOnly }, - { DiscoverContextAvailability, 3, { "v1", "registry", "discoverContextAvailability" }, badVerbPostOnly }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ RegisterContext, 3, { "v1", "registry", "registerContext" }, badVerbPostOnly }, + ///{ DiscoverContextAvailability, 3, { "v1", "registry", "discoverContextAvailability" }, badVerbPostOnly }, // FIXME: disable NGSI9 API routes in Orion 3.8.0, to be definetively removed at some point of the future //{ RegisterContext, 2, { "ngsi9", "registerContext" }, badVerbPostOnly }, //{ DiscoverContextAvailability, 2, { "ngsi9", "discoverContextAvailability" }, badVerbPostOnly }, - { UpdateContext, 2, { "v1", "updateContext" }, badVerbPostOnly }, - { QueryContext, 2, { "v1", "queryContext" }, badVerbPostOnly }, - { SubscribeContext, 2, { "v1", "subscribeContext" }, badVerbPostOnly }, - { UpdateContextSubscription, 2, { "v1", "updateContextSubscription" }, badVerbPostOnly }, - { UnsubscribeContext, 2, { "v1", "unsubscribeContext" }, badVerbPostOnly }, - { NotifyContext, 2, { "v1", "notifyContext" }, badVerbPostOnly }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ UpdateContext, 2, { "v1", "updateContext" }, badVerbPostOnly }, + ///{ QueryContext, 2, { "v1", "queryContext" }, badVerbPostOnly }, + ///{ SubscribeContext, 2, { "v1", "subscribeContext" }, badVerbPostOnly }, + ///{ UpdateContextSubscription, 2, { "v1", "updateContextSubscription" }, badVerbPostOnly }, + ///{ UnsubscribeContext, 2, { "v1", "unsubscribeContext" }, badVerbPostOnly }, + ///{ NotifyContext, 2, { "v1", "notifyContext" }, badVerbPostOnly }, // FIXME: disable NGSI9 API routes in Orion 3.8.0, to be definetively removed at some point of the future //{ ContextEntitiesByEntityId, 3, { "ngsi9", "contextEntities", "*" }, badVerbGetPostOnly }, //{ ContextEntityAttributes, 4, { "ngsi9", "contextEntities", "*", "attributes" }, badVerbGetPostOnly }, @@ -514,35 +524,36 @@ static RestService badVerbVlegacy[] = //{ ContextEntityTypes, 3, { "ngsi9", "contextEntityTypes", "*" }, badVerbGetPostOnly }, //{ ContextEntityTypeAttributeContainer, 4, { "ngsi9", "contextEntityTypes", "*", "attributes" }, badVerbGetPostOnly }, //{ ContextEntityTypeAttribute, 5, { "ngsi9", "contextEntityTypes", "*", "attributes", "*" }, badVerbGetPostOnly }, - { ContextEntitiesByEntityId, 4, { "v1", "registry", "contextEntities", "*" }, badVerbGetPostOnly }, - { ContextEntityAttributes, 5, { "v1", "registry", "contextEntities", "*", "attributes" }, badVerbGetPostOnly }, - { EntityByIdAttributeByName, 6, { "v1", "registry", "contextEntities", "*", "attributes", "*" }, badVerbGetPostOnly }, - { ContextEntityTypes, 4, { "v1", "registry", "contextEntityTypes", "*" }, badVerbGetPostOnly }, - { ContextEntityTypeAttributeContainer, 5, { "v1", "registry", "contextEntityTypes", "*", "attributes" }, badVerbGetPostOnly }, - { ContextEntityTypeAttribute, 6, { "v1", "registry", "contextEntityTypes", "*", "attributes", "*" }, badVerbGetPostOnly }, - { IndividualContextEntity, 3, { "ngsi10", "contextEntities", "*" }, badVerbAllFour }, - { IndividualContextEntityAttributes, 4, { "ngsi10", "contextEntities", "*", "attributes" }, badVerbAllFour }, - { IndividualContextEntityAttribute, 5, { "ngsi10", "contextEntities", "*", "attributes", "*" }, badVerbAllFour }, - { Ngsi10ContextEntityTypes, 3, { "ngsi10", "contextEntityTypes", "*" }, badVerbGetOnly }, - { Ngsi10ContextEntityTypesAttributeContainer, 4, { "ngsi10", "contextEntityTypes", "*", "attributes" }, badVerbGetOnly }, - { Ngsi10ContextEntityTypesAttribute, 5, { "ngsi10", "contextEntityTypes", "*", "attributes", "*" }, badVerbGetOnly }, - { SubscribeContext, 2, { "ngsi10", "contextSubscriptions" }, badVerbPostOnly }, - { Ngsi10SubscriptionsConvOp, 3, { "ngsi10", "contextSubscriptions", "*" }, badVerbPutDeleteOnly }, - { IndividualContextEntity, 3, { "v1", "contextEntities", "*" }, badVerbAllFour }, - { IndividualContextEntityAttributes, 4, { "v1", "contextEntities", "*", "attributes" }, badVerbAllFour }, - { IndividualContextEntityAttribute, 5, { "v1", "contextEntities", "*", "attributes", "*" }, badVerbAllFour }, - { Ngsi10ContextEntityTypes, 3, { "v1", "contextEntityTypes", "*" }, badVerbGetOnly }, - { Ngsi10ContextEntityTypesAttributeContainer, 4, { "v1", "contextEntityTypes", "*", "attributes" }, badVerbGetOnly }, - { Ngsi10ContextEntityTypesAttribute, 5, { "v1", "contextEntityTypes", "*", "attributes", "*" }, badVerbGetOnly }, - { SubscribeContext, 2, { "v1", "contextSubscriptions" }, badVerbPostOnly }, - { Ngsi10SubscriptionsConvOp, 3, { "v1", "contextSubscriptions", "*" }, badVerbPutDeleteOnly }, - { EntityTypes, 2, { "v1", "contextTypes" }, badVerbGetOnly }, - { AttributesForEntityType, 3, { "v1", "contextTypes", "*" }, badVerbGetOnly }, - { AllContextEntities, 2, { "v1", "contextEntities" }, badVerbGetPostOnly }, - { AllEntitiesWithTypeAndId, 6, { "v1", "contextEntities", "type", "*", "id", "*" }, badVerbAllFour }, - { IndividualContextEntityAttributeWithTypeAndId, 8, { "v1", "contextEntities", "type", "*", "id", "*", "attributes", "*" }, badVerbAllFour }, - { ContextEntitiesByEntityIdAndType, 7, { "v1", "registry", "contextEntities", "type", "*", "id", "*" }, badVerbGetPostOnly }, - { EntityByIdAttributeByNameIdAndType, 9, { "v1", "registry", "contextEntities", "type", "*", "id", "*", "attributes", "*" }, badVerbGetPostOnly }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ ContextEntitiesByEntityId, 4, { "v1", "registry", "contextEntities", "*" }, badVerbGetPostOnly }, + ///{ ContextEntityAttributes, 5, { "v1", "registry", "contextEntities", "*", "attributes" }, badVerbGetPostOnly }, + ///{ EntityByIdAttributeByName, 6, { "v1", "registry", "contextEntities", "*", "attributes", "*" }, badVerbGetPostOnly }, + ///{ ContextEntityTypes, 4, { "v1", "registry", "contextEntityTypes", "*" }, badVerbGetPostOnly }, + ///{ ContextEntityTypeAttributeContainer, 5, { "v1", "registry", "contextEntityTypes", "*", "attributes" }, badVerbGetPostOnly }, + ///{ ContextEntityTypeAttribute, 6, { "v1", "registry", "contextEntityTypes", "*", "attributes", "*" }, badVerbGetPostOnly }, + ///{ IndividualContextEntity, 3, { "ngsi10", "contextEntities", "*" }, badVerbAllFour }, + ///{ IndividualContextEntityAttributes, 4, { "ngsi10", "contextEntities", "*", "attributes" }, badVerbAllFour }, + ///{ IndividualContextEntityAttribute, 5, { "ngsi10", "contextEntities", "*", "attributes", "*" }, badVerbAllFour }, + ///{ Ngsi10ContextEntityTypes, 3, { "ngsi10", "contextEntityTypes", "*" }, badVerbGetOnly }, + ///{ Ngsi10ContextEntityTypesAttributeContainer, 4, { "ngsi10", "contextEntityTypes", "*", "attributes" }, badVerbGetOnly }, + ///{ Ngsi10ContextEntityTypesAttribute, 5, { "ngsi10", "contextEntityTypes", "*", "attributes", "*" }, badVerbGetOnly }, + ///{ SubscribeContext, 2, { "ngsi10", "contextSubscriptions" }, badVerbPostOnly }, + ///{ Ngsi10SubscriptionsConvOp, 3, { "ngsi10", "contextSubscriptions", "*" }, badVerbPutDeleteOnly }, + ///{ IndividualContextEntity, 3, { "v1", "contextEntities", "*" }, badVerbAllFour }, + ///{ IndividualContextEntityAttributes, 4, { "v1", "contextEntities", "*", "attributes" }, badVerbAllFour }, + ///{ IndividualContextEntityAttribute, 5, { "v1", "contextEntities", "*", "attributes", "*" }, badVerbAllFour }, + ///{ Ngsi10ContextEntityTypes, 3, { "v1", "contextEntityTypes", "*" }, badVerbGetOnly }, + ///{ Ngsi10ContextEntityTypesAttributeContainer, 4, { "v1", "contextEntityTypes", "*", "attributes" }, badVerbGetOnly }, + ///{ Ngsi10ContextEntityTypesAttribute, 5, { "v1", "contextEntityTypes", "*", "attributes", "*" }, badVerbGetOnly }, + ///{ SubscribeContext, 2, { "v1", "contextSubscriptions" }, badVerbPostOnly }, + ///{ Ngsi10SubscriptionsConvOp, 3, { "v1", "contextSubscriptions", "*" }, badVerbPutDeleteOnly }, + ///{ EntityTypes, 2, { "v1", "contextTypes" }, badVerbGetOnly }, + ///{ AttributesForEntityType, 3, { "v1", "contextTypes", "*" }, badVerbGetOnly }, + ///{ AllContextEntities, 2, { "v1", "contextEntities" }, badVerbGetPostOnly }, + ///{ AllEntitiesWithTypeAndId, 6, { "v1", "contextEntities", "type", "*", "id", "*" }, badVerbAllFour }, + ///{ IndividualContextEntityAttributeWithTypeAndId, 8, { "v1", "contextEntities", "type", "*", "id", "*", "attributes", "*" }, badVerbAllFour }, + ///{ ContextEntitiesByEntityIdAndType, 7, { "v1", "registry", "contextEntities", "type", "*", "id", "*" }, badVerbGetPostOnly }, + ///{ EntityByIdAttributeByNameIdAndType, 9, { "v1", "registry", "contextEntities", "type", "*", "id", "*", "attributes", "*" }, badVerbGetPostOnly }, // FIXME: disable administrative API routes not aligned with documentation in Orion 3.8.0, // to be definetively removed at some point of the future { LogTraceRequest, 2, { "log", "trace" }, badVerbGetDeleteOnly }, @@ -561,15 +572,17 @@ static RestService badVerbVlegacy[] = { LogLevelRequest, 2, { "admin", "log" }, badVerbPutOnly }, { SemStateRequest, 2, { "admin", "sem" }, badVerbGetOnly }, { MetricsRequest, 2, { "admin", "metrics" }, badVerbGetDeleteOnly }, - { UpdateContext, 2, { "ngsi10", "updateContext" }, badVerbPostOnly }, - { QueryContext, 2, { "ngsi10", "queryContext" }, badVerbPostOnly }, - { SubscribeContext, 2, { "ngsi10", "subscribeContext" }, badVerbPostOnly }, - { UpdateContextSubscription, 2, { "ngsi10", "updateContextSubscription" }, badVerbPostOnly }, - { UnsubscribeContext, 2, { "ngsi10", "unsubscribeContext" }, badVerbPostOnly }, - { NotifyContext, 2, { "ngsi10", "notifyContext" }, badVerbPostOnly }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ UpdateContext, 2, { "ngsi10", "updateContext" }, badVerbPostOnly }, + ///{ QueryContext, 2, { "ngsi10", "queryContext" }, badVerbPostOnly }, + ///{ SubscribeContext, 2, { "ngsi10", "subscribeContext" }, badVerbPostOnly }, + ///{ UpdateContextSubscription, 2, { "ngsi10", "updateContextSubscription" }, badVerbPostOnly }, + ///{ UnsubscribeContext, 2, { "ngsi10", "unsubscribeContext" }, badVerbPostOnly }, + ///{ NotifyContext, 2, { "ngsi10", "notifyContext" }, badVerbPostOnly }, // FIXME: disable NGSI9 API routes in Orion 3.8.0, to be definetively removed at some point of the future //{ InvalidRequest, 2, { "ngsi9", "*" }, badNgsi9Request }, - { InvalidRequest, 2, { "ngsi10", "*" }, badNgsi10Request }, + /// FIXME: disable unused NGSIv1 API (last batch) in Orion 4.0.0, to be definetively removed at some point of the future + ///{ InvalidRequest, 2, { "ngsi10", "*" }, badNgsi10Request }, { InvalidRequest, 0, { "*", "*", "*", "*", "*", "*" }, badRequest }, { InvalidRequest, 0, { }, NULL }, From 2fb12939f22cf580a1c3de497ca456a1038bcb19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 10:50:40 +0200 Subject: [PATCH 297/390] ADD entry to CNR --- CHANGES_NEXT_RELEASE | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 2728600db4..e82bdaff98 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -5,3 +5,11 @@ - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) - Hardening: compile code using C++14 standard +- Remove: all the remaining NGSIv1 operations + - PUT /v1/contextEntities/{id} + - DELETE /v1/contextEntities/{id} + - GET /v1/contextEntities/{id}/attributes/{name} + - POST /v1/updateContext + - POST /NGSI10/updateContext + - POST /v1/queryContext + - POST /NGSI10/queryContext From 131bac2add7889aa1c22c44e8427c2844840ede7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 11:20:13 +0200 Subject: [PATCH 298/390] REMOVE subs legacy format --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/admin/database_model.md | 3 +- doc/manuals/admin/logs.md | 9 -- doc/manuals/admin/statistics.md | 1 - doc/manuals/orion-api.md | 37 +------- src/lib/common/RenderFormat.cpp | 3 - src/lib/common/RenderFormat.h | 15 ++-- src/lib/common/statistics.cpp | 1 - .../mongoBackend/mongoCreateSubscription.cpp | 9 -- .../mongoBackend/mongoGetSubscriptions.cpp | 12 +-- .../mongoBackend/mongoUpdateSubscription.cpp | 9 -- src/lib/ngsi10/SubscribeContextRequest.cpp | 2 +- src/lib/ngsiNotify/Notifier.cpp | 32 +------ src/lib/serviceRoutines/statisticsTreat.cpp | 2 - .../log_deprecate_warning.test | 1 - .../statistics_with_full_counters.test | 1 - .../mongoSubscribeContext_test.cpp | 48 +++++----- .../mongoUpdateContextSubscription_test.cpp | 48 +++++----- ..._withOnchangeSubscriptionsNoCache_test.cpp | 76 ++++++++-------- ...Context_withOnchangeSubscriptions_test.cpp | 88 +++++++++---------- 20 files changed, 145 insertions(+), 253 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 2728600db4..c8e938dfd6 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -5,3 +5,4 @@ - Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) - Fix: lighter operation to get databases list from MongoDB (#4517) - Hardening: compile code using C++14 standard +- Remove: legacy subscription format diff --git a/doc/manuals/admin/database_model.md b/doc/manuals/admin/database_model.md index ae438dbef6..f11c762b4c 100644 --- a/doc/manuals/admin/database_model.md +++ b/doc/manuals/admin/database_model.md @@ -304,8 +304,7 @@ Fields: fields: q, mq, georel, geometry and/or coords (optional) - **count**: the number of notifications sent associated to the subscription. -- **format**: the format to use to send notification, possible values are **JSON** - (meaning JSON notifications in NGSIv1 legacy format), **normalized**, **keyValues**, **simplifiedNormalized**, **simplifiedKeyValues** and **values** (the last five used in NGSIv2 format). +- **format**: the format to use to send notification, possible values are **normalized**, **keyValues**, **simplifiedNormalized**, **simplifiedKeyValues** and **values**. - **status**: either `active` (for active subscriptions), `inactive` (for inactive subscriptions) or `oneshot` (for [oneshot subscriptions](../orion-api.md#oneshot-subscriptions)). Note that Orion API consider additional states (e.g. `expired`) but they never hit the DB (they are managed by Orion). diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index f88223e73b..bf711aba94 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -422,15 +422,6 @@ time=2024-01-11T13:57:13.595Z | lvl=WARN | corr=527c0912-b089-11ee-bb8c-080027cd time=2024-01-11T13:57:13.624Z | lvl=WARN | corr=52808938-b089-11ee-9835-080027cd35f1 | trans=1704981432-655-00000000010 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=postUpdateContext.cpp[163]:updateForward | msg=Deprecated usage of legacyForwarding mode in update forwarding operation (regId: 659ff3b9691855f16d00ec5a) ``` -* Usages of [`"attrsFormat": "legacy"`](../orion-api.md#subscriptionnotification). For instance: - -``` -time=2024-01-11T16:23:24.646Z | lvl=WARN | corr=be709034-b09d-11ee-b5d1-080027cd35f1 | trans=1704990203-652-00000000012 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=MongoCommonSubscription.cpp[598]:setFormat | msg=Deprecated usage of notification legacy format in subscription creation (subId: 65a015fcda947708d30425eb) -time=2024-01-11T16:23:24.675Z | lvl=WARN | corr=be74dc98-b09d-11ee-b2d6-080027cd35f1 | trans=1704990203-652-00000000013 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=mongoGetSubscriptions.cpp[217]:setNotification | msg=Deprecated usage of notification legacy format detected in existing subscription (subId: 65a015fcda947708d30425eb) -time=2024-01-11T16:23:24.701Z | lvl=WARN | corr=be709034-b09d-11ee-b5d1-080027cd35f1 | trans=1704990203-652-00000000012 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=MongoCommonSubscription.cpp[598]:setFormat | msg=Deprecated usage of notification legacy format in subscription modification (subId: 65a015fcda947708d30425eb) -time=2024-01-11T16:23:24.716Z | lvl=WARN | corr=be7ae5ac-b09d-11ee-98c8-080027cd35f1 | trans=1704990203-652-00000000015 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=Notifier.cpp[680]:buildSenderParams | msg=Deprecated usage of notification legacy format in notification (subId: 65a015fcda947708d30425eb) -``` - * Usages of `geo:point`, `geo:line`, `geo:box` or `geo:line`. ``` diff --git a/doc/manuals/admin/statistics.md b/doc/manuals/admin/statistics.md index a44bafcb74..2f8bf60f17 100644 --- a/doc/manuals/admin/statistics.md +++ b/doc/manuals/admin/statistics.md @@ -57,7 +57,6 @@ The counter block provides information about counters for the times a particular "deprecatedFeatures": { "geoFormat": 2, "ngsiv1Forwarding": 4, - "ngsiv1NotifFormat": 4, "ngsiv1Requests": 4 }, "invalidRequests": 2, diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 294fdd7dce..3b2951b08c 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -1993,40 +1993,7 @@ If `attrsFormat` is `values` then values partial entity representation mode is u } ``` -If `attrsFormat` is `legacy` then subscription representation follows NGSIv1 format. This way, users -can benefit from the enhancements of Orion subscriptions (e.g. filtering) with NGSIv1 legacy notification receivers. - -Note that NGSIv1 is deprecated. Thus, we don't recommend to use `legacy` notification format any longer. - -```json -{ - "subscriptionId": "56e2ad4e8001ff5e0a5260ec", - "originator": "localhost", - "contextResponses": [{ - "contextElement": { - "type": "Car", - "isPattern": "false", - "id": "Car1", - "attributes": [{ - "name": "temperature", - "type": "centigrade", - "value": "26.5", - "metadatas": [{ - "name": "TimeInstant", - "type": "recvTime", - "value": "2015-12-12 11:11:11.123" - }] - }] - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }] -} -``` - -Notifications must include the `Ngsiv2-AttrsFormat` (expect when `attrsFormat` is `legacy`) +Notifications must include the `Ngsiv2-AttrsFormat` HTTP header with the value of the format of the associated subscription, so that notification receivers are aware of the format without needing to process the notification payload. @@ -4712,7 +4679,7 @@ A `notification` object contains the following subfields: |--------------------|-------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `attrs` or `exceptAttrs` | | array | Both cannot be used at the same time.
  • attrs: List of attributes to be included in notification messages. It also defines the order in which attributes must appear in notifications when attrsFormat value is used (see [Notification Messages](#notification-messages) section). An empty list means that all attributes are to be included in notifications. See [Filtering out attributes and metadata](#filtering-out-attributes-and-metadata) section for more detail.
  • exceptAttrs: List of attributes to be excluded from the notification message, i.e. a notification message includes all entity attributes except the ones listed in this field. It must be a non-empty list.
  • If neither attrs nor exceptAttrs is specified, all attributes are included in notifications.
| | [`http`](#subscriptionnotificationhttp), [`httpCustom`](#subscriptionnotificationhttpcustom), [`mqtt`](#subscriptionnotificationmqtt) or [`mqttCustom`](#subscriptionnotificationmqttcustom)| ✓ | object | One of them must be present, but not more than one at the same time. It is used to convey parameters for notifications delivered through the transport protocol. | -| `attrsFormat` | ✓ | string | Specifies how the entities are represented in notifications. Accepted values are `normalized` (default), `simplifiedNormalized`, `keyValues`, `simplifiedKeyValues`, `values` or `legacy`.
If `attrsFormat` takes any value different than those, an error is raised. See detail in [Notification Messages](#notification-messages) section. | +| `attrsFormat` | ✓ | string | Specifies how the entities are represented in notifications. Accepted values are `normalized` (default), `simplifiedNormalized`, `keyValues`, `simplifiedKeyValues`, or `values`.
If `attrsFormat` takes any value different than those, an error is raised. See detail in [Notification Messages](#notification-messages) section. | | `metadata` | ✓ | string | List of metadata to be included in notification messages. See [Filtering out attributes and metadata](#filtering-out-attributes-and-metadata) section for more detail. | | `onlyChangedAttrs` | ✓ | boolean | If `true` then notifications will include only attributes that changed in the triggering update request, in combination with the `attrs` or `exceptAttrs` field. (default is `false` if the field is omitted)) | | `covered` | ✓ | boolean | If `true` then notifications will include all the attributes defined in `attrs` field, even if they are not present in the entity (in this, case, with `null` value). (default value is false). For further information see [Covered subscriptions](#covered-subscriptions) section | diff --git a/src/lib/common/RenderFormat.cpp b/src/lib/common/RenderFormat.cpp index e65c009203..9c29536216 100644 --- a/src/lib/common/RenderFormat.cpp +++ b/src/lib/common/RenderFormat.cpp @@ -41,7 +41,6 @@ const char* renderFormatToString(RenderFormat format, bool noDefault, bool useLe { switch (format) { - case NGSI_V1_LEGACY: return useLegacyWord ? "legacy" : "JSON"; case NGSI_V2_NORMALIZED: return "normalized"; case NGSI_V2_KEYVALUES: return "keyValues"; case NGSI_V2_VALUES: return "values"; @@ -71,8 +70,6 @@ const char* renderFormatToString(RenderFormat format, bool noDefault, bool useLe */ RenderFormat stringToRenderFormat(const std::string& s, bool noDefault) { - if (s == "JSON") { return NGSI_V1_LEGACY; } // DB content for NGSIv1 rendering due to legacy reasons - if (s == "legacy") { return NGSI_V1_LEGACY; } if (s == "normalized") { return NGSI_V2_NORMALIZED; } if (s == "keyValues") { return NGSI_V2_KEYVALUES; } if (s == "values") { return NGSI_V2_VALUES; } diff --git a/src/lib/common/RenderFormat.h b/src/lib/common/RenderFormat.h index 8d6d8f1660..8307896052 100644 --- a/src/lib/common/RenderFormat.h +++ b/src/lib/common/RenderFormat.h @@ -44,14 +44,13 @@ typedef enum RenderFormat { NO_FORMAT = 0, - NGSI_V1_LEGACY = 1, - NGSI_V2_NORMALIZED = 2, - NGSI_V2_KEYVALUES = 3, - NGSI_V2_VALUES = 4, - NGSI_V2_UNIQUE_VALUES = 5, - NGSI_V2_CUSTOM = 6, - NGSI_V2_SIMPLIFIEDNORMALIZED = 7, - NGSI_V2_SIMPLIFIEDKEYVALUES = 8 + NGSI_V2_NORMALIZED = 1, + NGSI_V2_KEYVALUES = 2, + NGSI_V2_VALUES = 3, + NGSI_V2_UNIQUE_VALUES = 4, + NGSI_V2_CUSTOM = 5, + NGSI_V2_SIMPLIFIEDNORMALIZED = 6, + NGSI_V2_SIMPLIFIEDKEYVALUES = 7 } RenderFormat; diff --git a/src/lib/common/statistics.cpp b/src/lib/common/statistics.cpp index 10dd8b1631..1a6dd710ff 100644 --- a/src/lib/common/statistics.cpp +++ b/src/lib/common/statistics.cpp @@ -152,7 +152,6 @@ int noOfSimulatedNotifications = -1; // Deprecated features int noOfDprNgsiv1Request = -1; int noOfDprLegacyForwarding = -1; -int noOfDprLegacyNotif = -1; int noOfDprGeoformat = -1; diff --git a/src/lib/mongoBackend/mongoCreateSubscription.cpp b/src/lib/mongoBackend/mongoCreateSubscription.cpp index 91192349f5..d5579884bb 100644 --- a/src/lib/mongoBackend/mongoCreateSubscription.cpp +++ b/src/lib/mongoBackend/mongoCreateSubscription.cpp @@ -198,15 +198,6 @@ std::string mongoCreateSubscription return ""; } - if (sub.attrsFormat == NGSI_V1_LEGACY) - { - __sync_fetch_and_add(&noOfDprLegacyNotif, 1); - if (logDeprecate) - { - LM_W(("Deprecated usage of notification legacy format in subscription creation (subId: %s)", subId.c_str())); - } - } - reqSemGive(__FUNCTION__, "ngsiv2 create subscription request", reqSemTaken); return subId; diff --git a/src/lib/mongoBackend/mongoGetSubscriptions.cpp b/src/lib/mongoBackend/mongoGetSubscriptions.cpp index c682955d0b..c14866017c 100644 --- a/src/lib/mongoBackend/mongoGetSubscriptions.cpp +++ b/src/lib/mongoBackend/mongoGetSubscriptions.cpp @@ -210,17 +210,7 @@ static void setNotification(Subscription* subP, const orion::BSONObj& r, const s nP->lastSuccessCode = r.hasField(CSUB_LASTSUCCESSCODE)? getIntOrLongFieldAsLongF(r, CSUB_LASTSUCCESSCODE) : -1; // Attributes format - subP->attrsFormat = r.hasField(CSUB_FORMAT)? stringToRenderFormat(getStringFieldF(r, CSUB_FORMAT)) : NGSI_V1_LEGACY; - - if (subP->attrsFormat == NGSI_V1_LEGACY) - { - __sync_fetch_and_add(&noOfDprLegacyNotif, 1); - if (logDeprecate) - { - LM_W(("Deprecated usage of notification legacy format detected in existing subscription (subId: %s)", subP->id.c_str())); - } - } - + subP->attrsFormat = r.hasField(CSUB_FORMAT)? stringToRenderFormat(getStringFieldF(r, CSUB_FORMAT)) : NGSI_V2_NORMALIZED; // // Check values from subscription cache, update object from cache-values if necessary diff --git a/src/lib/mongoBackend/mongoUpdateSubscription.cpp b/src/lib/mongoBackend/mongoUpdateSubscription.cpp index 93d71ccb2b..134a9910e4 100644 --- a/src/lib/mongoBackend/mongoUpdateSubscription.cpp +++ b/src/lib/mongoBackend/mongoUpdateSubscription.cpp @@ -370,15 +370,6 @@ std::string mongoUpdateSubscription if (subUp.notifyOnMetadataChangeProvided) setNotifyOnMetadataChange(subUp, &setB); if (subUp.attrsFormatProvided) setFormat(subUp, &setB); - if (subUp.attrsFormat == NGSI_V1_LEGACY) - { - __sync_fetch_and_add(&noOfDprLegacyNotif, 1); - if (logDeprecate) - { - LM_W(("Deprecated usage of notification legacy format in subscription modification (subId: %s)", subUp.id.c_str())); - } - } - // Description is special, as "" value removes the field if (subUp.descriptionProvided) { diff --git a/src/lib/ngsi10/SubscribeContextRequest.cpp b/src/lib/ngsi10/SubscribeContextRequest.cpp index 5038ef83ec..a0e2b19022 100644 --- a/src/lib/ngsi10/SubscribeContextRequest.cpp +++ b/src/lib/ngsi10/SubscribeContextRequest.cpp @@ -180,7 +180,7 @@ void SubscribeContextRequest::toNgsiv2Subscription(Subscription* sub) // description and expression are not touched, so default empty string provided by constructor will be used sub->status = STATUS_ACTIVE; sub->descriptionProvided = false; - sub->attrsFormat = NGSI_V1_LEGACY; + sub->attrsFormat = NGSI_V2_NORMALIZED; sub->notification.blacklist = false; sub->notification.httpInfo.custom = false; diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index e26fc31e37..14724c9a1e 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -151,20 +151,7 @@ static bool setPayload ncr.subscriptionId = subscriptionId; ncr.contextElementResponseVector.push_back(&cer); - if (*renderFormatP == NGSI_V1_LEGACY) - { - __sync_fetch_and_add(&noOfDprLegacyNotif, 1); - if (logDeprecate) - { - LM_W(("Deprecated usage of notification legacy format in notification (subId: %s)", subscriptionId.c_str())); - } - - *payloadP = ncr.toJsonV1(false, attrsFilter, blacklist, metadataFilter); - } - else - { - *payloadP = ncr.toJson(*renderFormatP, attrsFilter, blacklist, metadataFilter); - } + *payloadP = ncr.toJson(*renderFormatP, attrsFilter, blacklist, metadataFilter); *mimeTypeP = "application/json"; } @@ -681,22 +668,7 @@ SenderThreadParams* Notifier::buildSenderParams ci.outMimeType = JSON; - std::string payloadString; - if (renderFormat == NGSI_V1_LEGACY) - { - __sync_fetch_and_add(&noOfDprLegacyNotif, 1); - if (logDeprecate) - { - LM_W(("Deprecated usage of notification legacy format in notification (subId: %s)", subId.c_str())); - } - - bool asJsonObject = (ci.uriParam[URI_PARAM_ATTRIBUTE_FORMAT] == "object" && ci.outMimeType == JSON); - payloadString = ncr.toJsonV1(asJsonObject, attrsFilter, blacklist, metadataFilter); - } - else - { - payloadString = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter); - } + std::string payloadString = ncr.toJson(renderFormat, attrsFilter, blacklist, metadataFilter); /* Parse URL */ std::string host; diff --git a/src/lib/serviceRoutines/statisticsTreat.cpp b/src/lib/serviceRoutines/statisticsTreat.cpp index a0a003a874..ec734f51a9 100644 --- a/src/lib/serviceRoutines/statisticsTreat.cpp +++ b/src/lib/serviceRoutines/statisticsTreat.cpp @@ -93,7 +93,6 @@ static void resetStatistics(void) noOfDprNgsiv1Request = -1; noOfDprLegacyForwarding = -1; - noOfDprLegacyNotif = -1; noOfDprGeoformat = -1; statisticsTime = getCurrentTime(); @@ -221,7 +220,6 @@ std::string renderCounterStats(bool fullCounters) JsonObjectHelper jsDeprecated; renderUsedCounter(&jsDeprecated, "ngsiv1Requests", noOfDprNgsiv1Request, fullCounters); renderUsedCounter(&jsDeprecated, "ngsiv1Forwarding", noOfDprLegacyForwarding, fullCounters); - renderUsedCounter(&jsDeprecated, "ngsiv1NotifFormat", noOfDprLegacyNotif, fullCounters); renderUsedCounter(&jsDeprecated, "geoFormat", noOfDprGeoformat, fullCounters); std::string deprecation = jsDeprecated.str(); diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index 31092ef852..fe57f14647 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -488,7 +488,6 @@ Content-Length: 566 "deprecatedFeatures": { "geoFormat": 2, "ngsiv1Forwarding": 4, - "ngsiv1NotifFormat": 4, "ngsiv1Requests": 3 }, "jsonRequests": 8, diff --git a/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test b/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test index 403721e557..14dda84090 100644 --- a/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test +++ b/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test @@ -53,7 +53,6 @@ Content-Length: 1624 "deprecatedFeatures": { "geoFormat": 0, "ngsiv1Forwarding": 0, - "ngsiv1NotifFormat": 0, "ngsiv1Requests": 0 }, "discoveryErrors": 0, diff --git a/test/unittests/mongoBackend/mongoSubscribeContext_test.cpp b/test/unittests/mongoBackend/mongoSubscribeContext_test.cpp index d3e00359c3..314969b4d3 100644 --- a/test/unittests/mongoBackend/mongoSubscribeContext_test.cpp +++ b/test/unittests/mongoBackend/mongoSubscribeContext_test.cpp @@ -1366,7 +1366,7 @@ TEST(mongoSubscribeContext, matchEnt1_Attr0_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -1470,7 +1470,7 @@ TEST(mongoSubscribeContext, matchEnt1_Attr0_C1_JSON) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -1575,7 +1575,7 @@ TEST(mongoSubscribeContext, matchEnt1_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1684,7 +1684,7 @@ TEST(mongoSubscribeContext, matchEnt1_AttrN_C1_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1809,7 +1809,7 @@ TEST(mongoSubscribeContext, matchEnt1NoType_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1934,7 +1934,7 @@ TEST(mongoSubscribeContext, matchEnt1NoType_AttrN_C1_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2051,7 +2051,7 @@ TEST(mongoSubscribeContext, matchEnt1Pattern_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2168,7 +2168,7 @@ TEST(mongoSubscribeContext, matchEnt1Pattern_AttrN_C1_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2299,7 +2299,7 @@ TEST(mongoSubscribeContext, matchEnt1PatternNoType_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2430,7 +2430,7 @@ TEST(mongoSubscribeContext, matchEnt1PatternNoType_AttrN_C1_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2537,7 +2537,7 @@ TEST(mongoSubscribeContext, matchEnt1_Attr0_CN) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2644,7 +2644,7 @@ TEST(mongoSubscribeContext, matchEnt1_Attr0_CN_partial) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2752,7 +2752,7 @@ TEST(mongoSubscribeContext, matchEnt1_Attr0_CNbis) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2858,7 +2858,7 @@ TEST(mongoSubscribeContext, matchEnt1_AttrN_CN_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2971,7 +2971,7 @@ TEST(mongoSubscribeContext, matchEnt1_AttrN_CN_partial) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3084,7 +3084,7 @@ TEST(mongoSubscribeContext, matchEnt1_AttrN_CN_partial_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3197,7 +3197,7 @@ TEST(mongoSubscribeContext, matchEnt1_AttrN_CNbis) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3310,7 +3310,7 @@ TEST(mongoSubscribeContext, matchEnt1_AttrN_CN) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3433,7 +3433,7 @@ TEST(mongoSubscribeContext, matchEntN_Attr0_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3551,7 +3551,7 @@ TEST(mongoSubscribeContext, matchEntN_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3673,7 +3673,7 @@ TEST(mongoSubscribeContext, matchEntN_Attr0_CN) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3796,7 +3796,7 @@ TEST(mongoSubscribeContext, matchEntN_Attr0_CNbis) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3916,7 +3916,7 @@ TEST(mongoSubscribeContext, matchEntN_AttrN_CN) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -4040,7 +4040,7 @@ TEST(mongoSubscribeContext, matchEntN_AttrN_CNbis) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); diff --git a/test/unittests/mongoBackend/mongoUpdateContextSubscription_test.cpp b/test/unittests/mongoBackend/mongoUpdateContextSubscription_test.cpp index 0559c70094..6e26762847 100644 --- a/test/unittests/mongoBackend/mongoUpdateContextSubscription_test.cpp +++ b/test/unittests/mongoBackend/mongoUpdateContextSubscription_test.cpp @@ -1737,7 +1737,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_Attr0_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -1836,7 +1836,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_Attr0_C1_JSON) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -1933,7 +1933,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2033,7 +2033,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_AttrN_C1_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2148,7 +2148,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1NoType_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2263,7 +2263,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1NoType_AttrN_C1_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2370,7 +2370,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1Pattern_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2477,7 +2477,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1Pattern_AttrN_C1_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2598,7 +2598,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1PatternNoType_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2719,7 +2719,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1PatternNoType_AttrN_C1_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2821,7 +2821,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_Attr0_CN) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -2925,7 +2925,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_Attr0_CN_partial) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3030,7 +3030,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_Attr0_CNbis) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3130,7 +3130,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_AttrN_CN_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3234,7 +3234,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_AttrN_CN_partial) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3338,7 +3338,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_AttrN_CN_partial_disjoint) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3442,7 +3442,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_AttrN_CNbis) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3546,7 +3546,7 @@ TEST(mongoUpdateContextSubscription, matchEnt1_AttrN_CN) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3663,7 +3663,7 @@ TEST(mongoUpdateContextSubscription, matchEntN_Attr0_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3772,7 +3772,7 @@ TEST(mongoUpdateContextSubscription, matchEntN_AttrN_C1) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -3887,7 +3887,7 @@ TEST(mongoUpdateContextSubscription, matchEntN_Attr0_CN) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -4004,7 +4004,7 @@ TEST(mongoUpdateContextSubscription, matchEntN_Attr0_CNbis) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -4115,7 +4115,7 @@ TEST(mongoUpdateContextSubscription, matchEntN_AttrN_CN) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); @@ -4228,7 +4228,7 @@ TEST(mongoUpdateContextSubscription, matchEntN_AttrN_CNbis) "", "no correlator", 0, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(1); diff --git a/test/unittests/mongoBackend/mongoUpdateContext_withOnchangeSubscriptionsNoCache_test.cpp b/test/unittests/mongoBackend/mongoUpdateContext_withOnchangeSubscriptionsNoCache_test.cpp index 88d1d0ed18..65e84375cd 100644 --- a/test/unittests/mongoBackend/mongoUpdateContext_withOnchangeSubscriptionsNoCache_test.cpp +++ b/test/unittests/mongoBackend/mongoUpdateContext_withOnchangeSubscriptionsNoCache_test.cpp @@ -447,7 +447,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_updateMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -521,7 +521,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_appendMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -591,7 +591,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_appendMatch_type "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -660,7 +660,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_appendMatch_idAn "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -729,7 +729,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_deleteMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -805,7 +805,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_updateMatch_noTy "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -816,7 +816,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_updateMatch_noTy "", "", 2, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -894,7 +894,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_appendMatch_noTy "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -905,7 +905,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_appendMatch_noTy "", "", 2, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -978,7 +978,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_deleteMatch_noTy "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -989,7 +989,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_deleteMatch_noTy "", "", 2, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1060,7 +1060,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_updateMatch_patt "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1132,7 +1132,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_appendMatch_patt "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1200,7 +1200,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_deleteMatch_patt "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1270,7 +1270,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_updateMatch_patt "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1342,7 +1342,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_appendMatch_patt "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1410,7 +1410,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_deleteMatch_patt "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1480,7 +1480,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_updateMatchDisjo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1550,7 +1550,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_appendMatchDisjo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1620,7 +1620,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_deleteMatchDisjo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1858,7 +1858,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_updateMixMatchNo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1932,7 +1932,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_appendMixMatchNo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2002,7 +2002,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_deleteMixMatchNo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2074,7 +2074,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_update2Matches1N "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2148,7 +2148,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_append2Matches1N "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2218,7 +2218,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, Cond1_delete2Matches1N "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2288,7 +2288,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_updateMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2361,7 +2361,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_appendMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2429,7 +2429,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_deleteMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2499,7 +2499,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_updateMatchDisjo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2569,7 +2569,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_appendMatchDisjo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2639,7 +2639,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_deleteMatchDisjo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2877,7 +2877,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_updateMixMatchNo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2951,7 +2951,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_appendMixMatchNo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3021,7 +3021,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_deleteMixMatchNo "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3093,7 +3093,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_update2Matches1N "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3167,7 +3167,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_append2Matches1N "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3237,7 +3237,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptionsNoCache, CondN_delete2Matches1N "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); diff --git a/test/unittests/mongoBackend/mongoUpdateContext_withOnchangeSubscriptions_test.cpp b/test/unittests/mongoBackend/mongoUpdateContext_withOnchangeSubscriptions_test.cpp index aec62730c8..402b725eac 100644 --- a/test/unittests/mongoBackend/mongoUpdateContext_withOnchangeSubscriptions_test.cpp +++ b/test/unittests/mongoBackend/mongoUpdateContext_withOnchangeSubscriptions_test.cpp @@ -402,7 +402,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_updateMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -477,7 +477,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_appendMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -547,7 +547,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_deleteMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -624,7 +624,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_updateMatch_noType) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -635,7 +635,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_updateMatch_noType) "", "", 2, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -720,7 +720,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_appendMatch_noType) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -731,7 +731,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_appendMatch_noType) "", "", 2, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -812,7 +812,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_deleteMatch_noType) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -823,7 +823,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_deleteMatch_noType) "", "", 2, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -902,7 +902,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_updateMatch_pattern) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -980,7 +980,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_appendMatch_pattern) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1052,7 +1052,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_deleteMatch_pattern) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1126,7 +1126,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_updateMatch_pattern_noT "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1202,7 +1202,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_appendMatch_pattern_noT "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1274,7 +1274,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_deleteMatch_pattern_noT "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1348,7 +1348,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_updateMatchDisjoint) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1422,7 +1422,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_appendMatchDisjoint) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1496,7 +1496,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_deleteMatchDisjoint) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1553,7 +1553,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_updateNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, _, _, _)).Times(0); @@ -1610,7 +1610,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_appendNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, _, _, _)).Times(0); @@ -1666,7 +1666,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_deleteNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, _, _, _)).Times(0); @@ -1723,7 +1723,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_updateMatchWithoutChang "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(0); @@ -1796,7 +1796,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_updateMixMatchNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -1868,7 +1868,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_appendMixMatchNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, _, _, _)).Times(1); @@ -1936,7 +1936,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_deleteMixMatchNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, _, _, _)).Times(1); @@ -2013,7 +2013,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_update2Matches1Notifica "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2093,7 +2093,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_append2Matches1Notifica "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2168,7 +2168,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, Cond1_delete2Matches1Notifica "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2242,7 +2242,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_updateMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2319,7 +2319,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_appendMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2391,7 +2391,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_deleteMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2465,7 +2465,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_updateMatchDisjoint) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2540,7 +2540,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_appendMatchDisjoint) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2614,7 +2614,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_deleteMatchDisjoint) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2671,7 +2671,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_updateNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, _, _, _)).Times(0); @@ -2727,7 +2727,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_appendNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, _, _, _)).Times(0); @@ -2783,7 +2783,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_deleteNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, _, _, _)).Times(0); @@ -2840,7 +2840,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_updateMatchWithoutChang "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, emptyV, false, emptyV)).Times(0); @@ -2913,7 +2913,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_updateMixMatchNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -2991,7 +2991,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_appendMixMatchNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3065,7 +3065,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_deleteMixMatchNoMatch) "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3141,7 +3141,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_update2Matches1Notifica "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3220,7 +3220,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_append2Matches1Notifica "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); @@ -3295,7 +3295,7 @@ TEST(mongoUpdateContext_withOnchangeSubscriptions, CondN_delete2Matches1Notifica "", "", 1, - NGSI_V1_LEGACY, + NGSI_V2_NORMALIZED, attrsFilter, false, emptyV)).Times(1); From 139b8a21369514e7da34b74eefec4458ca9d7117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 11:40:20 +0200 Subject: [PATCH 299/390] FIX DISABLE remaining pure NGSIv1 tests --- .../{accept_fail.test => accept_fail.test.DISABLED} | 0 .../{disable_ngsi_cli.test => disable_ngsi_cli.test.DISABLED} | 0 ...g_dynamic.test => log_deprecate_warning_dynamic.test.DISABLED} | 0 ...icate_error.test => metadata_id_duplicate_error.test.DISABLED} | 0 ...large_query_context.test => large_query_context.test.DISABLED} | 0 ...rge_update_context.test => large_update_context.test.DISABLED} | 0 ....test => individual_context_entity_delete_fails.test.DISABLED} | 0 .../{update_sequence.test => update_sequence.test.DISABLED} | 0 ...th_http_header.test => service_path_http_header.test.DISABLED} | 0 ...ontextRequest.test => POST_UpdateContextRequest.test.DISABLED} | 0 ...lygon_geoscope.test => invalid_polygon_geoscope.test.DISABLED} | 0 ...lid_service_paths.test => invalid_service_paths.test.DISABLED} | 0 ...xist.test => append_attribute_that_doesnt_exist.test.DISABLED} | 0 ..._exist.test => delete_attribute_that_does_exist.test.DISABLED} | 0 ...xist.test => delete_attribute_that_doesnt_exist.test.DISABLED} | 0 ...cal_query_found.test => query_local_query_found.test.DISABLED} | 0 ...y_not_found.test => query_local_query_not_found.test.DISABLED} | 0 .../{query_simple.test => query_simple.test.DISABLED} | 0 ...xist.test => update_attribute_that_doesnt_exist.test.DISABLED} | 0 .../{update_local_found.test => update_local_found.test.DISABLED} | 0 ..._local_not_found.test => update_local_not_found.test.DISABLED} | 0 .../{basic_dot_attr.test => basic_dot_attr.test.DISABLED} | 0 ...h_service_path.test => delete_with_service_path.test.DISABLED} | 0 ...d.test => update_with_writeConcern_acknowledged.test.DISABLED} | 0 ...test => update_with_writeConcern_unacknowledged.test.DISABLED} | 0 ...priate_payload.test => qc_inappropriate_payload.test.DISABLED} | 0 ...r_payload.test => 3119_qc_nullcharacter_payload.test.DISABLED} | 0 ...=> ngsiv1_geobox_create_does_not_break_location.test.DISABLED} | 0 ...> ngsiv1_geojson_create_does_not_break_location.test.DISABLED} | 0 ...> ngsiv1_geoline_create_does_not_break_location.test.DISABLED} | 0 ... ngsiv1_geopoint_create_does_not_break_location.test.DISABLED} | 0 ...gsiv1_geopolygon_create_does_not_break_location.test.DISABLED} | 0 ...ases.test => killer_request_apiv1_parsing_cases.test.DISABLED} | 0 ...test => killer_request_apiv1_deeply_nested_attr.test.DISABLED} | 0 ....test => killer_request_apiv1_parsing_cases_bis.test.DISABLED} | 0 35 files changed, 0 insertions(+), 0 deletions(-) rename test/functionalTest/cases/0000_content_related_headers/{accept_fail.test => accept_fail.test.DISABLED} (100%) rename test/functionalTest/cases/0000_deprecated_checkings/{disable_ngsi_cli.test => disable_ngsi_cli.test.DISABLED} (100%) rename test/functionalTest/cases/0000_deprecated_checkings/{log_deprecate_warning_dynamic.test => log_deprecate_warning_dynamic.test.DISABLED} (100%) rename test/functionalTest/cases/0000_deprecated_checkings/{metadata_id_duplicate_error.test => metadata_id_duplicate_error.test.DISABLED} (100%) rename test/functionalTest/cases/0000_large_requests/{large_query_context.test => large_query_context.test.DISABLED} (100%) rename test/functionalTest/cases/0000_large_requests/{large_update_context.test => large_update_context.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_convenience/{individual_context_entity_delete_fails.test => individual_context_entity_delete_fails.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_update/{update_sequence.test => update_sequence.test.DISABLED} (100%) rename test/functionalTest/cases/0392_service_path_update_and_query/{service_path_http_header.test => service_path_http_header.test.DISABLED} (100%) rename test/functionalTest/cases/0647_crash_with_compounds/{POST_UpdateContextRequest.test => POST_UpdateContextRequest.test.DISABLED} (100%) rename test/functionalTest/cases/0690_geoquery_crash/{invalid_polygon_geoscope.test => invalid_polygon_geoscope.test.DISABLED} (100%) rename test/functionalTest/cases/0746_valid_service_path_for_updates/{invalid_service_paths.test => invalid_service_paths.test.DISABLED} (100%) rename test/functionalTest/cases/0787_cprs_full_functional/{append_attribute_that_doesnt_exist.test => append_attribute_that_doesnt_exist.test.DISABLED} (100%) rename test/functionalTest/cases/0787_cprs_full_functional/{delete_attribute_that_does_exist.test => delete_attribute_that_does_exist.test.DISABLED} (100%) rename test/functionalTest/cases/0787_cprs_full_functional/{delete_attribute_that_doesnt_exist.test => delete_attribute_that_doesnt_exist.test.DISABLED} (100%) rename test/functionalTest/cases/0787_cprs_full_functional/{query_local_query_found.test => query_local_query_found.test.DISABLED} (100%) rename test/functionalTest/cases/0787_cprs_full_functional/{query_local_query_not_found.test => query_local_query_not_found.test.DISABLED} (100%) rename test/functionalTest/cases/0787_cprs_full_functional/{query_simple.test => query_simple.test.DISABLED} (100%) rename test/functionalTest/cases/0787_cprs_full_functional/{update_attribute_that_doesnt_exist.test => update_attribute_that_doesnt_exist.test.DISABLED} (100%) rename test/functionalTest/cases/0787_cprs_full_functional/{update_local_found.test => update_local_found.test.DISABLED} (100%) rename test/functionalTest/cases/0787_cprs_full_functional/{update_local_not_found.test => update_local_not_found.test.DISABLED} (100%) rename test/functionalTest/cases/0878_attributes_with_dot/{basic_dot_attr.test => basic_dot_attr.test.DISABLED} (100%) rename test/functionalTest/cases/0885_service_path_and_delete/{delete_with_service_path.test => delete_with_service_path.test.DISABLED} (100%) rename test/functionalTest/cases/0889_writeConcern_configurable/{update_with_writeConcern_acknowledged.test => update_with_writeConcern_acknowledged.test.DISABLED} (100%) rename test/functionalTest/cases/0889_writeConcern_configurable/{update_with_writeConcern_unacknowledged.test => update_with_writeConcern_unacknowledged.test.DISABLED} (100%) rename test/functionalTest/cases/3055_qc_inappropriate_payload/{qc_inappropriate_payload.test => qc_inappropriate_payload.test.DISABLED} (100%) rename test/functionalTest/cases/3119_qc_nullcharacter_payload/{3119_qc_nullcharacter_payload.test => 3119_qc_nullcharacter_payload.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geobox_create_does_not_break_location.test => ngsiv1_geobox_create_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geojson_create_does_not_break_location.test => ngsiv1_geojson_create_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geoline_create_does_not_break_location.test => ngsiv1_geoline_create_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geopoint_create_does_not_break_location.test => ngsiv1_geopoint_create_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geopolygon_create_does_not_break_location.test => ngsiv1_geopolygon_create_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3607_killer_request_apiv1/{killer_request_apiv1_parsing_cases.test => killer_request_apiv1_parsing_cases.test.DISABLED} (100%) rename test/functionalTest/cases/3608_killer_request_apiv1/{killer_request_apiv1_deeply_nested_attr.test => killer_request_apiv1_deeply_nested_attr.test.DISABLED} (100%) rename test/functionalTest/cases/3608_killer_request_apiv1/{killer_request_apiv1_parsing_cases_bis.test => killer_request_apiv1_parsing_cases_bis.test.DISABLED} (100%) diff --git a/test/functionalTest/cases/0000_content_related_headers/accept_fail.test b/test/functionalTest/cases/0000_content_related_headers/accept_fail.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_content_related_headers/accept_fail.test rename to test/functionalTest/cases/0000_content_related_headers/accept_fail.test.DISABLED diff --git a/test/functionalTest/cases/0000_deprecated_checkings/disable_ngsi_cli.test b/test/functionalTest/cases/0000_deprecated_checkings/disable_ngsi_cli.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_deprecated_checkings/disable_ngsi_cli.test rename to test/functionalTest/cases/0000_deprecated_checkings/disable_ngsi_cli.test.DISABLED diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test rename to test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning_dynamic.test.DISABLED diff --git a/test/functionalTest/cases/0000_deprecated_checkings/metadata_id_duplicate_error.test b/test/functionalTest/cases/0000_deprecated_checkings/metadata_id_duplicate_error.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_deprecated_checkings/metadata_id_duplicate_error.test rename to test/functionalTest/cases/0000_deprecated_checkings/metadata_id_duplicate_error.test.DISABLED diff --git a/test/functionalTest/cases/0000_large_requests/large_query_context.test b/test/functionalTest/cases/0000_large_requests/large_query_context.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_large_requests/large_query_context.test rename to test/functionalTest/cases/0000_large_requests/large_query_context.test.DISABLED diff --git a/test/functionalTest/cases/0000_large_requests/large_update_context.test b/test/functionalTest/cases/0000_large_requests/large_update_context.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_large_requests/large_update_context.test rename to test/functionalTest/cases/0000_large_requests/large_update_context.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_convenience/individual_context_entity_delete_fails.test b/test/functionalTest/cases/0000_ngsi10_convenience/individual_context_entity_delete_fails.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_convenience/individual_context_entity_delete_fails.test rename to test/functionalTest/cases/0000_ngsi10_convenience/individual_context_entity_delete_fails.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_update/update_sequence.test b/test/functionalTest/cases/0000_ngsi10_update/update_sequence.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_update/update_sequence.test rename to test/functionalTest/cases/0000_ngsi10_update/update_sequence.test.DISABLED diff --git a/test/functionalTest/cases/0392_service_path_update_and_query/service_path_http_header.test b/test/functionalTest/cases/0392_service_path_update_and_query/service_path_http_header.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0392_service_path_update_and_query/service_path_http_header.test rename to test/functionalTest/cases/0392_service_path_update_and_query/service_path_http_header.test.DISABLED diff --git a/test/functionalTest/cases/0647_crash_with_compounds/POST_UpdateContextRequest.test b/test/functionalTest/cases/0647_crash_with_compounds/POST_UpdateContextRequest.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0647_crash_with_compounds/POST_UpdateContextRequest.test rename to test/functionalTest/cases/0647_crash_with_compounds/POST_UpdateContextRequest.test.DISABLED diff --git a/test/functionalTest/cases/0690_geoquery_crash/invalid_polygon_geoscope.test b/test/functionalTest/cases/0690_geoquery_crash/invalid_polygon_geoscope.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0690_geoquery_crash/invalid_polygon_geoscope.test rename to test/functionalTest/cases/0690_geoquery_crash/invalid_polygon_geoscope.test.DISABLED diff --git a/test/functionalTest/cases/0746_valid_service_path_for_updates/invalid_service_paths.test b/test/functionalTest/cases/0746_valid_service_path_for_updates/invalid_service_paths.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0746_valid_service_path_for_updates/invalid_service_paths.test rename to test/functionalTest/cases/0746_valid_service_path_for_updates/invalid_service_paths.test.DISABLED diff --git a/test/functionalTest/cases/0787_cprs_full_functional/append_attribute_that_doesnt_exist.test b/test/functionalTest/cases/0787_cprs_full_functional/append_attribute_that_doesnt_exist.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0787_cprs_full_functional/append_attribute_that_doesnt_exist.test rename to test/functionalTest/cases/0787_cprs_full_functional/append_attribute_that_doesnt_exist.test.DISABLED diff --git a/test/functionalTest/cases/0787_cprs_full_functional/delete_attribute_that_does_exist.test b/test/functionalTest/cases/0787_cprs_full_functional/delete_attribute_that_does_exist.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0787_cprs_full_functional/delete_attribute_that_does_exist.test rename to test/functionalTest/cases/0787_cprs_full_functional/delete_attribute_that_does_exist.test.DISABLED diff --git a/test/functionalTest/cases/0787_cprs_full_functional/delete_attribute_that_doesnt_exist.test b/test/functionalTest/cases/0787_cprs_full_functional/delete_attribute_that_doesnt_exist.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0787_cprs_full_functional/delete_attribute_that_doesnt_exist.test rename to test/functionalTest/cases/0787_cprs_full_functional/delete_attribute_that_doesnt_exist.test.DISABLED diff --git a/test/functionalTest/cases/0787_cprs_full_functional/query_local_query_found.test b/test/functionalTest/cases/0787_cprs_full_functional/query_local_query_found.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0787_cprs_full_functional/query_local_query_found.test rename to test/functionalTest/cases/0787_cprs_full_functional/query_local_query_found.test.DISABLED diff --git a/test/functionalTest/cases/0787_cprs_full_functional/query_local_query_not_found.test b/test/functionalTest/cases/0787_cprs_full_functional/query_local_query_not_found.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0787_cprs_full_functional/query_local_query_not_found.test rename to test/functionalTest/cases/0787_cprs_full_functional/query_local_query_not_found.test.DISABLED diff --git a/test/functionalTest/cases/0787_cprs_full_functional/query_simple.test b/test/functionalTest/cases/0787_cprs_full_functional/query_simple.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0787_cprs_full_functional/query_simple.test rename to test/functionalTest/cases/0787_cprs_full_functional/query_simple.test.DISABLED diff --git a/test/functionalTest/cases/0787_cprs_full_functional/update_attribute_that_doesnt_exist.test b/test/functionalTest/cases/0787_cprs_full_functional/update_attribute_that_doesnt_exist.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0787_cprs_full_functional/update_attribute_that_doesnt_exist.test rename to test/functionalTest/cases/0787_cprs_full_functional/update_attribute_that_doesnt_exist.test.DISABLED diff --git a/test/functionalTest/cases/0787_cprs_full_functional/update_local_found.test b/test/functionalTest/cases/0787_cprs_full_functional/update_local_found.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0787_cprs_full_functional/update_local_found.test rename to test/functionalTest/cases/0787_cprs_full_functional/update_local_found.test.DISABLED diff --git a/test/functionalTest/cases/0787_cprs_full_functional/update_local_not_found.test b/test/functionalTest/cases/0787_cprs_full_functional/update_local_not_found.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0787_cprs_full_functional/update_local_not_found.test rename to test/functionalTest/cases/0787_cprs_full_functional/update_local_not_found.test.DISABLED diff --git a/test/functionalTest/cases/0878_attributes_with_dot/basic_dot_attr.test b/test/functionalTest/cases/0878_attributes_with_dot/basic_dot_attr.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0878_attributes_with_dot/basic_dot_attr.test rename to test/functionalTest/cases/0878_attributes_with_dot/basic_dot_attr.test.DISABLED diff --git a/test/functionalTest/cases/0885_service_path_and_delete/delete_with_service_path.test b/test/functionalTest/cases/0885_service_path_and_delete/delete_with_service_path.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0885_service_path_and_delete/delete_with_service_path.test rename to test/functionalTest/cases/0885_service_path_and_delete/delete_with_service_path.test.DISABLED diff --git a/test/functionalTest/cases/0889_writeConcern_configurable/update_with_writeConcern_acknowledged.test b/test/functionalTest/cases/0889_writeConcern_configurable/update_with_writeConcern_acknowledged.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0889_writeConcern_configurable/update_with_writeConcern_acknowledged.test rename to test/functionalTest/cases/0889_writeConcern_configurable/update_with_writeConcern_acknowledged.test.DISABLED diff --git a/test/functionalTest/cases/0889_writeConcern_configurable/update_with_writeConcern_unacknowledged.test b/test/functionalTest/cases/0889_writeConcern_configurable/update_with_writeConcern_unacknowledged.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0889_writeConcern_configurable/update_with_writeConcern_unacknowledged.test rename to test/functionalTest/cases/0889_writeConcern_configurable/update_with_writeConcern_unacknowledged.test.DISABLED diff --git a/test/functionalTest/cases/3055_qc_inappropriate_payload/qc_inappropriate_payload.test b/test/functionalTest/cases/3055_qc_inappropriate_payload/qc_inappropriate_payload.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3055_qc_inappropriate_payload/qc_inappropriate_payload.test rename to test/functionalTest/cases/3055_qc_inappropriate_payload/qc_inappropriate_payload.test.DISABLED diff --git a/test/functionalTest/cases/3119_qc_nullcharacter_payload/3119_qc_nullcharacter_payload.test b/test/functionalTest/cases/3119_qc_nullcharacter_payload/3119_qc_nullcharacter_payload.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3119_qc_nullcharacter_payload/3119_qc_nullcharacter_payload.test rename to test/functionalTest/cases/3119_qc_nullcharacter_payload/3119_qc_nullcharacter_payload.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geobox_create_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geobox_create_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geobox_create_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geobox_create_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geojson_create_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geojson_create_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geojson_create_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geojson_create_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geoline_create_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geoline_create_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geoline_create_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geoline_create_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopoint_create_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopoint_create_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopoint_create_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopoint_create_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopolygon_create_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopolygon_create_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopolygon_create_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopolygon_create_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3607_killer_request_apiv1/killer_request_apiv1_parsing_cases.test b/test/functionalTest/cases/3607_killer_request_apiv1/killer_request_apiv1_parsing_cases.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3607_killer_request_apiv1/killer_request_apiv1_parsing_cases.test rename to test/functionalTest/cases/3607_killer_request_apiv1/killer_request_apiv1_parsing_cases.test.DISABLED diff --git a/test/functionalTest/cases/3608_killer_request_apiv1/killer_request_apiv1_deeply_nested_attr.test b/test/functionalTest/cases/3608_killer_request_apiv1/killer_request_apiv1_deeply_nested_attr.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3608_killer_request_apiv1/killer_request_apiv1_deeply_nested_attr.test rename to test/functionalTest/cases/3608_killer_request_apiv1/killer_request_apiv1_deeply_nested_attr.test.DISABLED diff --git a/test/functionalTest/cases/3608_killer_request_apiv1/killer_request_apiv1_parsing_cases_bis.test b/test/functionalTest/cases/3608_killer_request_apiv1/killer_request_apiv1_parsing_cases_bis.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3608_killer_request_apiv1/killer_request_apiv1_parsing_cases_bis.test rename to test/functionalTest/cases/3608_killer_request_apiv1/killer_request_apiv1_parsing_cases_bis.test.DISABLED From 987b83006853ef4b068cd384367ef2c1516c1fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 12:09:05 +0200 Subject: [PATCH 300/390] FIX tests --- .../log_deprecate_warning.test | 205 ++++-------------- ...lse_positive_in_log_deprecation_logic.test | 98 --------- 2 files changed, 38 insertions(+), 265 deletions(-) delete mode 100644 test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index 31092ef852..f390bf58f4 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -33,69 +33,21 @@ brokerStart CB 0 IPV4 -logDeprecate -statCounters # Note there is not a "Update registration using legacyForwarding mode" # as the registrations update operation is not implemented (issue #3007) # -# 01. Query E1-T1 -# 02. GET /v1/contextEntities/E/attributes/A -# 03. Create entity using NGSIv1 and geo:point -# 04. Create entity using NGSIv2 and geo:point -# 05. Create registration using legacyForwarding mode -# 06. Retrieve registrations (which uses legacyForwarding mode) -# 07. Forwarded query using legacyForwarding mode -# 08. Update query using legacyForwarding mode -# 09. Create subscription using attrsFormat legacy -# 10. Retrieve subscriptions (which uses attrsFormat legacy) -# 11. Update subscription using attrsFormat legacy -# 12. Trigger notification using attrsFormat legacy -# 13. Get WARNING trace in logs -# 14. Get statistics and see deprecatedFeatures counters +# 01. Create entity using NGSIv2 and geo:point +# 02. Create registration using legacyForwarding mode +# 03. Retrieve registrations (which uses legacyForwarding mode) +# 04. Forwarded query using legacyForwarding mode +# 05. Update query using legacyForwarding mode +# 06. Create subscription using attrsFormat legacy +# 07. Retrieve subscriptions (which uses attrsFormat legacy) +# 08. Update subscription using attrsFormat legacy +# 09. Trigger notification using attrsFormat legacy +# 10. Get WARNING trace in logs +# 11. Get statistics and see deprecatedFeatures counters # -echo "01. Query E1-T1" -echo "===============" -payload='{ - "entities": [ - { - "type": "T1", - "id": "E1" - } - ] -}' -orionCurl --url /v1/queryContext --payload "${payload}" -echo -echo - -echo "02. GET /v1/contextEntities/E/attributes/A" -echo "==========================================" -orionCurl --url /v1/contextEntities/E/attributes/A -echo -echo - - -echo "03. Create entity using NGSIv1 and geo:point" -echo "============================================" -payload='{ - "contextElements": [ - { - "type": "City", - "isPattern": "false", - "id": "Barcelona", - "attributes": [ - { - "name": "location", - "type": "geo:point", - "value": "40.418889, -3.691944" - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url /v1/updateContext --payload "$payload" -echo -echo - - -echo "04. Create entity using NGSIv2 and geo:point" +echo "01. Create entity using NGSIv2 and geo:point" echo "============================================" payload='{ "id": "Sevilla", @@ -110,7 +62,7 @@ echo echo -echo "05. Create registration using legacyForwarding mode" +echo "02. Create registration using legacyForwarding mode" echo "===================================================" payload='{ "dataProvided": { @@ -138,21 +90,21 @@ echo REG_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") -echo "06. Retrieve registrations (which uses legacyForwarding mode)" +echo "03. Retrieve registrations (which uses legacyForwarding mode)" echo "=============================================================" orionCurl --url /v2/registrations echo echo -echo "07. Forwarded query using legacyForwarding mode" +echo "04. Forwarded query using legacyForwarding mode" echo "===============================================" orionCurl --url /v2/entities/E1/attrs/A1?type=T1 echo echo -echo "08. Update query using legacyForwarding mode" +echo "05. Update query using legacyForwarding mode" echo "============================================" payload='{ "value": 1, @@ -163,7 +115,7 @@ echo echo -echo "09. Create subscription using attrsFormat legacy" +echo "06. Create subscription using attrsFormat legacy" echo "================================================" payload='{ "subject": { @@ -189,14 +141,14 @@ echo SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") -echo "10. Retrieve subscriptions (which uses attrsFormat legacy)" +echo "07. Retrieve subscriptions (which uses attrsFormat legacy)" echo "==========================================================" orionCurl --url /v2/subscriptions echo echo -echo "11. Update subscription using attrsFormat legacy" +echo "08. Update subscription using attrsFormat legacy" echo "================================================" payload='{ "notification": { @@ -211,7 +163,7 @@ echo echo -echo "12. Trigger notification using attrsFormat legacy" +echo "09. Trigger notification using attrsFormat legacy" echo "=================================================" payload='{ "id": "E2", @@ -226,14 +178,14 @@ echo echo -echo "13. Get WARNING trace in logs" +echo "10. Get WARNING trace in logs" echo "=============================" cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' | sed -e "s/$REG_ID/REG_ID/g" | sed -e "s/$SUB_ID/SUB_ID/g" echo echo -echo "14. Get statistics and see deprecatedFeatures counters" +echo "11. Get statistics and see deprecatedFeatures counters" echo "======================================================" orionCurl --url /statistics echo @@ -241,72 +193,7 @@ echo --REGEXPECT-- -01. Query E1-T1 -=============== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 70 - -{ - "errorCode": { - "code": "404", - "reasonPhrase": "No context element found" - } -} - - -02. GET /v1/contextEntities/E/attributes/A -========================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 98 - -{ - "statusCode": { - "code": "404", - "details": "Entity id: /E/", - "reasonPhrase": "No context element found" - } -} - - -03. Create entity using NGSIv1 and geo:point -============================================ -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 207 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "name": "location", - "type": "geo:point", - "value": "" - } - ], - "id": "Barcelona", - "isPattern": "false", - "type": "City" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -04. Create entity using NGSIv2 and geo:point +01. Create entity using NGSIv2 and geo:point ============================================ HTTP/1.1 201 Created Date: REGEX(.*) @@ -316,7 +203,7 @@ Content-Length: 0 -05. Create registration using legacyForwarding mode +02. Create registration using legacyForwarding mode =================================================== HTTP/1.1 201 Created Date: REGEX(.*) @@ -326,7 +213,7 @@ Content-Length: 0 -06. Retrieve registrations (which uses legacyForwarding mode) +03. Retrieve registrations (which uses legacyForwarding mode) ============================================================= HTTP/1.1 200 OK Date: REGEX(.*) @@ -360,7 +247,7 @@ Content-Length: 233 ] -07. Forwarded query using legacyForwarding mode +04. Forwarded query using legacyForwarding mode =============================================== HTTP/1.1 404 Not Found Date: REGEX(.*) @@ -374,7 +261,7 @@ Content-Length: 95 } -08. Update query using legacyForwarding mode +05. Update query using legacyForwarding mode ============================================ HTTP/1.1 404 Not Found Date: REGEX(.*) @@ -388,7 +275,7 @@ Content-Length: 95 } -09. Create subscription using attrsFormat legacy +06. Create subscription using attrsFormat legacy ================================================ HTTP/1.1 201 Created Date: REGEX(.*) @@ -398,7 +285,7 @@ Content-Length: 0 -10. Retrieve subscriptions (which uses attrsFormat legacy) +07. Retrieve subscriptions (which uses attrsFormat legacy) ========================================================== HTTP/1.1 200 OK Date: REGEX(.*) @@ -435,7 +322,7 @@ Content-Length: 288 ] -11. Update subscription using attrsFormat legacy +08. Update subscription using attrsFormat legacy ================================================ HTTP/1.1 204 No Content Date: REGEX(.*) @@ -443,7 +330,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -12. Trigger notification using attrsFormat legacy +09. Trigger notification using attrsFormat legacy ================================================= HTTP/1.1 201 Created Date: REGEX(.*) @@ -453,12 +340,8 @@ Content-Length: 0 -13. Get WARNING trace in logs +10. Get WARNING trace in logs ============================= -Deprecated NGSIv1 request received: POST /v1/queryContext, request payload (48 bytes): { "entities": [ { "type": "T1", "id": "E1" } ] }, response code: 200 -Deprecated NGSIv1 request received: GET /v1/contextEntities/E/attributes/A, response code: 200 -Deprecated usage of geo:point detected in attribute location at entity update, please use geo:json instead -Deprecated NGSIv1 request received: POST /v1/updateContext, request payload (208 bytes): { "contextElements": [ { "type": "City", "isPattern": "false", "id": "Barcelona", "attributes": [ { "name": "location", "type": "geo:point", "value": "40.418889, -3.691944" } ] } ], "updateAction": "APPEND" }, response code: 200 Deprecated usage of geo:point detected in attribute location at entity update, please use geo:json instead Deprecated usage of legacyForwarding mode in registration creation (regId: REG_ID) Deprecated usage of legacyForwarding mode detected in existing registration (regId: REG_ID) @@ -475,24 +358,23 @@ Deprecated usage of notification legacy format in notification (subId: SUB_ID) Raising alarm NotificationError localhost:1234/: notification failure for queue worker: Couldn't connect to server -14. Get statistics and see deprecatedFeatures counters +11. Get statistics and see deprecatedFeatures counters ====================================================== HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 566 +Content-Length: 412 { "counters": { "deprecatedFeatures": { - "geoFormat": 2, + "geoFormat": 1, "ngsiv1Forwarding": 4, - "ngsiv1NotifFormat": 4, - "ngsiv1Requests": 3 + "ngsiv1NotifFormat": 4 }, - "jsonRequests": 8, - "noPayloadRequests": 5, + "jsonRequests": 6, + "noPayloadRequests": 4, "requests": { "/statistics": { "GET": 1 @@ -515,17 +397,6 @@ Content-Length: 566 "/v2/subscriptions/{id}": { "PATCH": 1 } - }, - "requestsLegacy": { - "/v1/contextEntities/{id}/attributes/{name}": { - "GET": 1 - }, - "/v1/queryContext": { - "POST": 1 - }, - "/v1/updateContext": { - "POST": 1 - } } }, "measuring_interval_in_secs": REGEX(\d+), diff --git a/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test b/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test deleted file mode 100644 index 1ee335d350..0000000000 --- a/test/functionalTest/cases/4454_false_positive_in_log_deprecation_logic/false_positive_in_log_deprecation_logic.test +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -False positive in log deprecation - ---SHELL-INIT-- -dbInit CB -brokerStart CB 0 IPV4 -logDeprecate - ---SHELL-- - -# -# 01. GET /v2/entities/glory_ev1 -# 02. GET /v1/contextEntities/glory_ev2/attributes/A -# 03. Get WARNING trace in logs corresponding only to the second case -# - -echo "01. GET /v2/entities/glory_ev1" -echo "==============================" -orionCurl --url /v2/entities/glory_ev1 -echo -echo - - -echo "02. GET /v1/contextEntities/glory_ev2/attributes/A" -echo "==================================================" -orionCurl --url /v1/contextEntities/glory_ev2/attributes/A -echo -echo - - -echo "03. Get WARNING trace in logs corresponding only to the second case" -echo "===================================================================" -cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' -echo -echo - - ---REGEXPECT-- -01. GET /v2/entities/glory_ev1 -============================== -HTTP/1.1 404 Not Found -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 95 - -{ - "description": "The requested entity has not been found. Check type and id", - "error": "NotFound" -} - - -02. GET /v1/contextEntities/glory_ev2/attributes/A -================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 106 - -{ - "statusCode": { - "code": "404", - "details": "Entity id: /glory_ev2/", - "reasonPhrase": "No context element found" - } -} - - -03. Get WARNING trace in logs corresponding only to the second case -=================================================================== -Deprecated NGSIv1 request received: GET /v1/contextEntities/glory_ev2/attributes/A, response code: 200 - - ---TEARDOWN-- -brokerStop CB -dbDrop CB From b8ca953d00ba383caf681e1ae513a78fa69c2728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 12:14:51 +0200 Subject: [PATCH 301/390] DISABLE tests --- ...etadata.test => metadata_id_as_regular_metadata.test.DISABLED} | 0 ..._with_counters.test => statistics_with_counters.test.DISABLED} | 0 ...filter_string_scope.test => filter_string_scope.test.DISABLED} | 0 ...ntity_all_attrs.test => GET_v2_entity_all_attrs.test.DISABLED} | 0 ..._entity_one_attr.test => GET_v2_entity_one_attr.test.DISABLED} | 0 ...ue.test => GET_v2_entity_single_attribute_value.test.DISABLED} | 0 ... GET_v2_entity_single_attribute_value_not_found.test.DISABLED} | 0 ...> GET_v2_entity_single_attribute_value_too_many.test.DISABLED} | 0 ....test => vector_compound_in_metadata_for_api_v1.test.DISABLED} | 0 ...ribute_ngsiv1.test => order_by_attribute_ngsiv1.test.DISABLED} | 0 .../{v1_append_strict.test => v1_append_strict.test.DISABLED} | 0 .../1127_v1_replace/{v1_replace.test => v1_replace.test.DISABLED} | 0 ...> mq_as_scope_for_compound_values_in_v1_queries.test.DISABLED} | 0 ...in_v1_queries.test => mq_as_scope_in_v1_queries.test.DISABLED} | 0 ...param_options_map.test => uri_param_options_map.test.DISABLED} | 0 ...t_types_values_option_with_null_and_empty_types.test.DISABLED} | 0 ...get_types_values_option_with_null_type_entities.test.DISABLED} | 0 ..._basic.test => GET_v2_ent_attr_val_accept_basic.test.DISABLED} | 0 ...ation_in_ngsv1.test => ngsiv2_location_in_ngsv1.test.DISABLED} | 0 ...gsv1_near.test => ngsiv2_location_in_ngsv1_near.test.DISABLED} | 0 ...bug_cross_sp_update.test => bug_cross_sp_update.test.DISABLED} | 0 ...with_same_name.test => json_tags_with_same_name.test.DISABLED} | 0 ...s_pattern.test => enity_type_wrongly_as_pattern.test.DISABLED} | 0 ...o_point.test => v1_append_not_updates_geo_point.test.DISABLED} | 0 ...se.test => fwd_update_ok_but_not_found_response.test.DISABLED} | 0 ...ble_forward.test => fwd_v2_query_double_forward.test.DISABLED} | 0 ...tration.test => fwd_v2_query_false_registration.test.DISABLED} | 0 ..._v2_query_false_registration_and_one_more_value.test.DISABLED} | 0 ...lse_registration_and_one_more_value_in_provider.test.DISABLED} | 0 ...> fwd_v2_query_five_different_cprs_in_one_query.test.DISABLED} | 0 ...t => fwd_v2_query_forward_depends_on_attr_field.test.DISABLED} | 0 ...nd.test => fwd_v2_query_forward_query_not_found.test.DISABLED} | 0 ...v2_query_found_and_not_found_locally_and_remote.test.DISABLED} | 0 ..._no_attributes_in_query_results_from_CB_and_CP1.test.DISABLED} | 0 ...no_attributes_in_query_results_from_CP1_and_CP2.test.DISABLED} | 0 ...ry_one_forward.test => fwd_v2_query_one_forward.test.DISABLED} | 0 ...tribute.test => fwd_v2_query_shadowed_attribute.test.DISABLED} | 0 ...=> fwd_v2_query_two_attributes_in_cpr_one_local.test.DISABLED} | 0 ...uery_two_attributes_in_one_cpr_three_in_another.test.DISABLED} | 0 ...le_forward.test => fwd_v2_update_double_forward.test.DISABLED} | 0 ...ration.test => fwd_v2_update_false_registration.test.DISABLED} | 0 ...v2_update_false_registration_and_one_more_value.test.DISABLED} | 0 ...lse_registration_and_one_more_value_in_provider.test.DISABLED} | 0 ... fwd_v2_update_five_different_cprs_in_one_query.test.DISABLED} | 0 ... => fwd_v2_update_forward_depends_on_attr_field.test.DISABLED} | 0 ...ate_found_and_not_found_in_local_and_three_cprs.test.DISABLED} | 0 ...e_one_forward.test => fwd_v2_update_one_forward.test.DISABLED} | 0 ...ribute.test => fwd_v2_update_shadowed_attribute.test.DISABLED} | 0 ...ate_three_found_on_cps_two_local_four_not_found.test.DISABLED} | 0 ...2_ngsiv2_query_five_different_cprs_in_one_query.test.DISABLED} | 0 ...d_v2_ngsiv2_query_forward_depends_on_attr_field.test.DISABLED} | 0 ...v2_query_found_and_not_found_locally_and_remote.test.DISABLED} | 0 ..._no_attributes_in_query_results_from_CB_and_CP1.test.DISABLED} | 0 ...no_attributes_in_query_results_from_CP1_and_CP2.test.DISABLED} | 0 ...y.test => fwd_v2_ngsiv2_query_one_forward_query.test.DISABLED} | 0 ....test => fwd_v2_ngsiv2_query_shadowed_attribute.test.DISABLED} | 0 ...v2_ngsiv2_query_two_attributes_in_cpr_one_local.test.DISABLED} | 0 ...uery_two_attributes_in_one_cpr_three_in_another.test.DISABLED} | 0 ...ard.test => fwd_v2_ngsiv2_update_double_forward.test.DISABLED} | 0 ...v2_update_false_registration_and_one_more_value.test.DISABLED} | 0 ...lse_registration_and_one_more_value_in_provider.test.DISABLED} | 0 ..._ngsiv2_update_five_different_cprs_in_one_query.test.DISABLED} | 0 ..._v2_ngsiv2_update_forward_depends_on_attr_field.test.DISABLED} | 0 ...ate_found_and_not_found_in_local_and_three_cprs.test.DISABLED} | 0 ...orward.test => fwd_v2_ngsiv2_update_one_forward.test.DISABLED} | 0 ...test => fwd_v2_ngsiv2_update_shadowed_attribute.test.DISABLED} | 0 ...ate_three_found_on_cps_two_local_four_not_found.test.DISABLED} | 0 ...bility.test => fwd_ngsv1_ngsv2_interoperability.test.DISABLED} | 0 ...est => supported_forwarding_entity_mixing_modes.test.DISABLED} | 0 ...utocast.test => create_attributes_with_autocast.test.DISABLED} | 0 ...ibutes.test => filters_in_autocasted_attributes.test.DISABLED} | 0 ...utocast.test => update_attributes_with_autocast.test.DISABLED} | 0 ...utocast.test => update_only_value_with_autocast.test.DISABLED} | 0 .../{query_for_arrays.test => query_for_arrays.test.DISABLED} | 0 ...=> ngsiv1_geobox_update_does_not_break_location.test.DISABLED} | 0 ...> ngsiv1_geojson_update_does_not_break_location.test.DISABLED} | 0 ...> ngsiv1_geoline_update_does_not_break_location.test.DISABLED} | 0 ... ngsiv1_geopoint_update_does_not_break_location.test.DISABLED} | 0 ...gsiv1_geopolygon_update_does_not_break_location.test.DISABLED} | 0 ...notif_queue.test => default_service_notif_queue.test.DISABLED} | 0 ...ice_notif_queue.test => per_service_notif_queue.test.DISABLED} | 0 ...ues.test => per_service_notif_queue_full_queues.test.DISABLED} | 0 ...ound_and_not_found_locally_and_remote_with_skip.test.DISABLED} | 0 ...query_two_attributes_in_cpr_one_local_with_skip.test.DISABLED} | 0 84 files changed, 0 insertions(+), 0 deletions(-) rename test/functionalTest/cases/0000_deprecated_checkings/{metadata_id_as_regular_metadata.test => metadata_id_as_regular_metadata.test.DISABLED} (100%) rename test/functionalTest/cases/0000_statistics_operation/{statistics_with_counters.test => statistics_with_counters.test.DISABLED} (100%) rename test/functionalTest/cases/0864_filter_string_scope/{filter_string_scope.test => filter_string_scope.test.DISABLED} (100%) rename test/functionalTest/cases/0971_GET_v2_entity_param_attrs/{GET_v2_entity_all_attrs.test => GET_v2_entity_all_attrs.test.DISABLED} (100%) rename test/functionalTest/cases/0971_GET_v2_entity_param_attrs/{GET_v2_entity_one_attr.test => GET_v2_entity_one_attr.test.DISABLED} (100%) rename test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/{GET_v2_entity_single_attribute_value.test => GET_v2_entity_single_attribute_value.test.DISABLED} (100%) rename test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/{GET_v2_entity_single_attribute_value_not_found.test => GET_v2_entity_single_attribute_value_not_found.test.DISABLED} (100%) rename test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/{GET_v2_entity_single_attribute_value_too_many.test => GET_v2_entity_single_attribute_value_too_many.test.DISABLED} (100%) rename test/functionalTest/cases/1068_metadata_value_as_compound/{vector_compound_in_metadata_for_api_v1.test => vector_compound_in_metadata_for_api_v1.test.DISABLED} (100%) rename test/functionalTest/cases/1103_order_by/{order_by_attribute_ngsiv1.test => order_by_attribute_ngsiv1.test.DISABLED} (100%) rename test/functionalTest/cases/1127_v1_append_strict/{v1_append_strict.test => v1_append_strict.test.DISABLED} (100%) rename test/functionalTest/cases/1127_v1_replace/{v1_replace.test => v1_replace.test.DISABLED} (100%) rename test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/{mq_as_scope_for_compound_values_in_v1_queries.test => mq_as_scope_for_compound_values_in_v1_queries.test.DISABLED} (100%) rename test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/{mq_as_scope_in_v1_queries.test => mq_as_scope_in_v1_queries.test.DISABLED} (100%) rename test/functionalTest/cases/1169_uri_param_options_map/{uri_param_options_map.test => uri_param_options_map.test.DISABLED} (100%) rename test/functionalTest/cases/1170_get_types_values_option/{get_types_values_option_with_null_and_empty_types.test => get_types_values_option_with_null_and_empty_types.test.DISABLED} (100%) rename test/functionalTest/cases/1170_get_types_values_option/{get_types_values_option_with_null_type_entities.test => get_types_values_option_with_null_type_entities.test.DISABLED} (100%) rename test/functionalTest/cases/1179_options_test/{GET_v2_ent_attr_val_accept_basic.test => GET_v2_ent_attr_val_accept_basic.test.DISABLED} (100%) rename test/functionalTest/cases/1677_geo_location_for_api_v2/{ngsiv2_location_in_ngsv1.test => ngsiv2_location_in_ngsv1.test.DISABLED} (100%) rename test/functionalTest/cases/1677_geo_location_for_api_v2/{ngsiv2_location_in_ngsv1_near.test => ngsiv2_location_in_ngsv1_near.test.DISABLED} (100%) rename test/functionalTest/cases/1956_bug_cross_sp_update/{bug_cross_sp_update.test => bug_cross_sp_update.test.DISABLED} (100%) rename test/functionalTest/cases/2457_json_tags_with_same_name/{json_tags_with_same_name.test => json_tags_with_same_name.test.DISABLED} (100%) rename test/functionalTest/cases/2723_entity_type_wrongly_as_pattern/{enity_type_wrongly_as_pattern.test => enity_type_wrongly_as_pattern.test.DISABLED} (100%) rename test/functionalTest/cases/2770_v1_append_not_updates_geo_point/{v1_append_not_updates_geo_point.test => v1_append_not_updates_geo_point.test.DISABLED} (100%) rename test/functionalTest/cases/2871_fwd_update_ok_but_not_found_response/{fwd_update_ok_but_not_found_response.test => fwd_update_ok_but_not_found_response.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_double_forward.test => fwd_v2_query_double_forward.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_false_registration.test => fwd_v2_query_false_registration.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_false_registration_and_one_more_value.test => fwd_v2_query_false_registration_and_one_more_value.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_false_registration_and_one_more_value_in_provider.test => fwd_v2_query_false_registration_and_one_more_value_in_provider.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_five_different_cprs_in_one_query.test => fwd_v2_query_five_different_cprs_in_one_query.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_forward_depends_on_attr_field.test => fwd_v2_query_forward_depends_on_attr_field.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_forward_query_not_found.test => fwd_v2_query_forward_query_not_found.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_found_and_not_found_locally_and_remote.test => fwd_v2_query_found_and_not_found_locally_and_remote.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_no_attributes_in_query_results_from_CB_and_CP1.test => fwd_v2_query_no_attributes_in_query_results_from_CB_and_CP1.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_no_attributes_in_query_results_from_CP1_and_CP2.test => fwd_v2_query_no_attributes_in_query_results_from_CP1_and_CP2.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_one_forward.test => fwd_v2_query_one_forward.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_shadowed_attribute.test => fwd_v2_query_shadowed_attribute.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_two_attributes_in_cpr_one_local.test => fwd_v2_query_two_attributes_in_cpr_one_local.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_query_two_attributes_in_one_cpr_three_in_another.test => fwd_v2_query_two_attributes_in_one_cpr_three_in_another.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_double_forward.test => fwd_v2_update_double_forward.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_false_registration.test => fwd_v2_update_false_registration.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_false_registration_and_one_more_value.test => fwd_v2_update_false_registration_and_one_more_value.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_false_registration_and_one_more_value_in_provider.test => fwd_v2_update_false_registration_and_one_more_value_in_provider.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_five_different_cprs_in_one_query.test => fwd_v2_update_five_different_cprs_in_one_query.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_forward_depends_on_attr_field.test => fwd_v2_update_forward_depends_on_attr_field.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_found_and_not_found_in_local_and_three_cprs.test => fwd_v2_update_found_and_not_found_in_local_and_three_cprs.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_one_forward.test => fwd_v2_update_one_forward.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_shadowed_attribute.test => fwd_v2_update_shadowed_attribute.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional/{fwd_v2_update_three_found_on_cps_two_local_four_not_found.test => fwd_v2_update_three_found_on_cps_two_local_four_not_found.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_query_five_different_cprs_in_one_query.test => fwd_v2_ngsiv2_query_five_different_cprs_in_one_query.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_query_forward_depends_on_attr_field.test => fwd_v2_ngsiv2_query_forward_depends_on_attr_field.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote.test => fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CB_and_CP1.test => fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CB_and_CP1.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CP1_and_CP2.test => fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CP1_and_CP2.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_query_one_forward_query.test => fwd_v2_ngsiv2_query_one_forward_query.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_query_shadowed_attribute.test => fwd_v2_ngsiv2_query_shadowed_attribute.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local.test => fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_query_two_attributes_in_one_cpr_three_in_another.test => fwd_v2_ngsiv2_query_two_attributes_in_one_cpr_three_in_another.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_update_double_forward.test => fwd_v2_ngsiv2_update_double_forward.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_update_false_registration_and_one_more_value.test => fwd_v2_ngsiv2_update_false_registration_and_one_more_value.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_update_false_registration_and_one_more_value_in_provider.test => fwd_v2_ngsiv2_update_false_registration_and_one_more_value_in_provider.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_update_five_different_cprs_in_one_query.test => fwd_v2_ngsiv2_update_five_different_cprs_in_one_query.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_update_forward_depends_on_attr_field.test => fwd_v2_ngsiv2_update_forward_depends_on_attr_field.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_update_found_and_not_found_in_local_and_three_cprs.test => fwd_v2_ngsiv2_update_found_and_not_found_in_local_and_three_cprs.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_update_one_forward.test => fwd_v2_ngsiv2_update_one_forward.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_update_shadowed_attribute.test => fwd_v2_ngsiv2_update_shadowed_attribute.test.DISABLED} (100%) rename test/functionalTest/cases/3068_cprs_full_functional_v2/{fwd_v2_ngsiv2_update_three_found_on_cps_two_local_four_not_found.test => fwd_v2_ngsiv2_update_three_found_on_cps_two_local_four_not_found.test.DISABLED} (100%) rename test/functionalTest/cases/3068_ngsi_v2_based_forwarding/{fwd_ngsv1_ngsv2_interoperability.test => fwd_ngsv1_ngsv2_interoperability.test.DISABLED} (100%) rename test/functionalTest/cases/3106_supported_forwarding_mode/{supported_forwarding_entity_mixing_modes.test => supported_forwarding_entity_mixing_modes.test.DISABLED} (100%) rename test/functionalTest/cases/3112_ngsiv1_autocast/{create_attributes_with_autocast.test => create_attributes_with_autocast.test.DISABLED} (100%) rename test/functionalTest/cases/3112_ngsiv1_autocast/{filters_in_autocasted_attributes.test => filters_in_autocasted_attributes.test.DISABLED} (100%) rename test/functionalTest/cases/3112_ngsiv1_autocast/{update_attributes_with_autocast.test => update_attributes_with_autocast.test.DISABLED} (100%) rename test/functionalTest/cases/3112_ngsiv1_autocast/{update_only_value_with_autocast.test => update_only_value_with_autocast.test.DISABLED} (100%) rename test/functionalTest/cases/3162_context_providers_legacy_non_primitive/{query_for_arrays.test => query_for_arrays.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geobox_update_does_not_break_location.test => ngsiv1_geobox_update_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geojson_update_does_not_break_location.test => ngsiv1_geojson_update_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geoline_update_does_not_break_location.test => ngsiv1_geoline_update_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geopoint_update_does_not_break_location.test => ngsiv1_geopoint_update_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3442_ngsiv1_interop_location_fix/{ngsiv1_geopolygon_update_does_not_break_location.test => ngsiv1_geopolygon_update_does_not_break_location.test.DISABLED} (100%) rename test/functionalTest/cases/3843_per_service_notif_queue/{default_service_notif_queue.test => default_service_notif_queue.test.DISABLED} (100%) rename test/functionalTest/cases/3843_per_service_notif_queue/{per_service_notif_queue.test => per_service_notif_queue.test.DISABLED} (100%) rename test/functionalTest/cases/3843_per_service_notif_queue/{per_service_notif_queue_full_queues.test => per_service_notif_queue_full_queues.test.DISABLED} (100%) rename test/functionalTest/cases/3851_skip_query_forwarding/{fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote_with_skip.test => fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote_with_skip.test.DISABLED} (100%) rename test/functionalTest/cases/3851_skip_query_forwarding/{fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local_with_skip.test => fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local_with_skip.test.DISABLED} (100%) diff --git a/test/functionalTest/cases/0000_deprecated_checkings/metadata_id_as_regular_metadata.test b/test/functionalTest/cases/0000_deprecated_checkings/metadata_id_as_regular_metadata.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_deprecated_checkings/metadata_id_as_regular_metadata.test rename to test/functionalTest/cases/0000_deprecated_checkings/metadata_id_as_regular_metadata.test.DISABLED diff --git a/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test b/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test rename to test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test.DISABLED diff --git a/test/functionalTest/cases/0864_filter_string_scope/filter_string_scope.test b/test/functionalTest/cases/0864_filter_string_scope/filter_string_scope.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0864_filter_string_scope/filter_string_scope.test rename to test/functionalTest/cases/0864_filter_string_scope/filter_string_scope.test.DISABLED diff --git a/test/functionalTest/cases/0971_GET_v2_entity_param_attrs/GET_v2_entity_all_attrs.test b/test/functionalTest/cases/0971_GET_v2_entity_param_attrs/GET_v2_entity_all_attrs.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0971_GET_v2_entity_param_attrs/GET_v2_entity_all_attrs.test rename to test/functionalTest/cases/0971_GET_v2_entity_param_attrs/GET_v2_entity_all_attrs.test.DISABLED diff --git a/test/functionalTest/cases/0971_GET_v2_entity_param_attrs/GET_v2_entity_one_attr.test b/test/functionalTest/cases/0971_GET_v2_entity_param_attrs/GET_v2_entity_one_attr.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0971_GET_v2_entity_param_attrs/GET_v2_entity_one_attr.test rename to test/functionalTest/cases/0971_GET_v2_entity_param_attrs/GET_v2_entity_one_attr.test.DISABLED diff --git a/test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value.test b/test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value.test rename to test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value.test.DISABLED diff --git a/test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value_not_found.test b/test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value_not_found.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value_not_found.test rename to test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value_not_found.test.DISABLED diff --git a/test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value_too_many.test b/test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value_too_many.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value_too_many.test rename to test/functionalTest/cases/0997_GET_v2_entity_single_attribute_value/GET_v2_entity_single_attribute_value_too_many.test.DISABLED diff --git a/test/functionalTest/cases/1068_metadata_value_as_compound/vector_compound_in_metadata_for_api_v1.test b/test/functionalTest/cases/1068_metadata_value_as_compound/vector_compound_in_metadata_for_api_v1.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1068_metadata_value_as_compound/vector_compound_in_metadata_for_api_v1.test rename to test/functionalTest/cases/1068_metadata_value_as_compound/vector_compound_in_metadata_for_api_v1.test.DISABLED diff --git a/test/functionalTest/cases/1103_order_by/order_by_attribute_ngsiv1.test b/test/functionalTest/cases/1103_order_by/order_by_attribute_ngsiv1.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1103_order_by/order_by_attribute_ngsiv1.test rename to test/functionalTest/cases/1103_order_by/order_by_attribute_ngsiv1.test.DISABLED diff --git a/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test b/test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test rename to test/functionalTest/cases/1127_v1_append_strict/v1_append_strict.test.DISABLED diff --git a/test/functionalTest/cases/1127_v1_replace/v1_replace.test b/test/functionalTest/cases/1127_v1_replace/v1_replace.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1127_v1_replace/v1_replace.test rename to test/functionalTest/cases/1127_v1_replace/v1_replace.test.DISABLED diff --git a/test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/mq_as_scope_for_compound_values_in_v1_queries.test b/test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/mq_as_scope_for_compound_values_in_v1_queries.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/mq_as_scope_for_compound_values_in_v1_queries.test rename to test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/mq_as_scope_for_compound_values_in_v1_queries.test.DISABLED diff --git a/test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/mq_as_scope_in_v1_queries.test b/test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/mq_as_scope_in_v1_queries.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/mq_as_scope_in_v1_queries.test rename to test/functionalTest/cases/1156_mq_as_uri_param_for_metadata/mq_as_scope_in_v1_queries.test.DISABLED diff --git a/test/functionalTest/cases/1169_uri_param_options_map/uri_param_options_map.test b/test/functionalTest/cases/1169_uri_param_options_map/uri_param_options_map.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1169_uri_param_options_map/uri_param_options_map.test rename to test/functionalTest/cases/1169_uri_param_options_map/uri_param_options_map.test.DISABLED diff --git a/test/functionalTest/cases/1170_get_types_values_option/get_types_values_option_with_null_and_empty_types.test b/test/functionalTest/cases/1170_get_types_values_option/get_types_values_option_with_null_and_empty_types.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1170_get_types_values_option/get_types_values_option_with_null_and_empty_types.test rename to test/functionalTest/cases/1170_get_types_values_option/get_types_values_option_with_null_and_empty_types.test.DISABLED diff --git a/test/functionalTest/cases/1170_get_types_values_option/get_types_values_option_with_null_type_entities.test b/test/functionalTest/cases/1170_get_types_values_option/get_types_values_option_with_null_type_entities.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1170_get_types_values_option/get_types_values_option_with_null_type_entities.test rename to test/functionalTest/cases/1170_get_types_values_option/get_types_values_option_with_null_type_entities.test.DISABLED diff --git a/test/functionalTest/cases/1179_options_test/GET_v2_ent_attr_val_accept_basic.test b/test/functionalTest/cases/1179_options_test/GET_v2_ent_attr_val_accept_basic.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1179_options_test/GET_v2_ent_attr_val_accept_basic.test rename to test/functionalTest/cases/1179_options_test/GET_v2_ent_attr_val_accept_basic.test.DISABLED diff --git a/test/functionalTest/cases/1677_geo_location_for_api_v2/ngsiv2_location_in_ngsv1.test b/test/functionalTest/cases/1677_geo_location_for_api_v2/ngsiv2_location_in_ngsv1.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1677_geo_location_for_api_v2/ngsiv2_location_in_ngsv1.test rename to test/functionalTest/cases/1677_geo_location_for_api_v2/ngsiv2_location_in_ngsv1.test.DISABLED diff --git a/test/functionalTest/cases/1677_geo_location_for_api_v2/ngsiv2_location_in_ngsv1_near.test b/test/functionalTest/cases/1677_geo_location_for_api_v2/ngsiv2_location_in_ngsv1_near.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1677_geo_location_for_api_v2/ngsiv2_location_in_ngsv1_near.test rename to test/functionalTest/cases/1677_geo_location_for_api_v2/ngsiv2_location_in_ngsv1_near.test.DISABLED diff --git a/test/functionalTest/cases/1956_bug_cross_sp_update/bug_cross_sp_update.test b/test/functionalTest/cases/1956_bug_cross_sp_update/bug_cross_sp_update.test.DISABLED similarity index 100% rename from test/functionalTest/cases/1956_bug_cross_sp_update/bug_cross_sp_update.test rename to test/functionalTest/cases/1956_bug_cross_sp_update/bug_cross_sp_update.test.DISABLED diff --git a/test/functionalTest/cases/2457_json_tags_with_same_name/json_tags_with_same_name.test b/test/functionalTest/cases/2457_json_tags_with_same_name/json_tags_with_same_name.test.DISABLED similarity index 100% rename from test/functionalTest/cases/2457_json_tags_with_same_name/json_tags_with_same_name.test rename to test/functionalTest/cases/2457_json_tags_with_same_name/json_tags_with_same_name.test.DISABLED diff --git a/test/functionalTest/cases/2723_entity_type_wrongly_as_pattern/enity_type_wrongly_as_pattern.test b/test/functionalTest/cases/2723_entity_type_wrongly_as_pattern/enity_type_wrongly_as_pattern.test.DISABLED similarity index 100% rename from test/functionalTest/cases/2723_entity_type_wrongly_as_pattern/enity_type_wrongly_as_pattern.test rename to test/functionalTest/cases/2723_entity_type_wrongly_as_pattern/enity_type_wrongly_as_pattern.test.DISABLED diff --git a/test/functionalTest/cases/2770_v1_append_not_updates_geo_point/v1_append_not_updates_geo_point.test b/test/functionalTest/cases/2770_v1_append_not_updates_geo_point/v1_append_not_updates_geo_point.test.DISABLED similarity index 100% rename from test/functionalTest/cases/2770_v1_append_not_updates_geo_point/v1_append_not_updates_geo_point.test rename to test/functionalTest/cases/2770_v1_append_not_updates_geo_point/v1_append_not_updates_geo_point.test.DISABLED diff --git a/test/functionalTest/cases/2871_fwd_update_ok_but_not_found_response/fwd_update_ok_but_not_found_response.test b/test/functionalTest/cases/2871_fwd_update_ok_but_not_found_response/fwd_update_ok_but_not_found_response.test.DISABLED similarity index 100% rename from test/functionalTest/cases/2871_fwd_update_ok_but_not_found_response/fwd_update_ok_but_not_found_response.test rename to test/functionalTest/cases/2871_fwd_update_ok_but_not_found_response/fwd_update_ok_but_not_found_response.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_double_forward.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_double_forward.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_double_forward.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_double_forward.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration_and_one_more_value.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration_and_one_more_value.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration_and_one_more_value.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration_and_one_more_value.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration_and_one_more_value_in_provider.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration_and_one_more_value_in_provider.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration_and_one_more_value_in_provider.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_false_registration_and_one_more_value_in_provider.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_five_different_cprs_in_one_query.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_five_different_cprs_in_one_query.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_five_different_cprs_in_one_query.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_five_different_cprs_in_one_query.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_forward_depends_on_attr_field.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_forward_depends_on_attr_field.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_forward_depends_on_attr_field.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_forward_depends_on_attr_field.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_forward_query_not_found.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_forward_query_not_found.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_forward_query_not_found.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_forward_query_not_found.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_found_and_not_found_locally_and_remote.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_found_and_not_found_locally_and_remote.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_found_and_not_found_locally_and_remote.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_found_and_not_found_locally_and_remote.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_no_attributes_in_query_results_from_CB_and_CP1.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_no_attributes_in_query_results_from_CB_and_CP1.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_no_attributes_in_query_results_from_CB_and_CP1.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_no_attributes_in_query_results_from_CB_and_CP1.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_no_attributes_in_query_results_from_CP1_and_CP2.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_no_attributes_in_query_results_from_CP1_and_CP2.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_no_attributes_in_query_results_from_CP1_and_CP2.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_no_attributes_in_query_results_from_CP1_and_CP2.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_one_forward.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_one_forward.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_one_forward.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_one_forward.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_shadowed_attribute.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_shadowed_attribute.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_shadowed_attribute.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_shadowed_attribute.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_two_attributes_in_cpr_one_local.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_two_attributes_in_cpr_one_local.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_two_attributes_in_cpr_one_local.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_two_attributes_in_cpr_one_local.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_two_attributes_in_one_cpr_three_in_another.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_two_attributes_in_one_cpr_three_in_another.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_two_attributes_in_one_cpr_three_in_another.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_query_two_attributes_in_one_cpr_three_in_another.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_double_forward.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_double_forward.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_double_forward.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_double_forward.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration_and_one_more_value.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration_and_one_more_value.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration_and_one_more_value.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration_and_one_more_value.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration_and_one_more_value_in_provider.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration_and_one_more_value_in_provider.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration_and_one_more_value_in_provider.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_false_registration_and_one_more_value_in_provider.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_five_different_cprs_in_one_query.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_five_different_cprs_in_one_query.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_five_different_cprs_in_one_query.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_five_different_cprs_in_one_query.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_forward_depends_on_attr_field.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_forward_depends_on_attr_field.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_forward_depends_on_attr_field.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_forward_depends_on_attr_field.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_found_and_not_found_in_local_and_three_cprs.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_found_and_not_found_in_local_and_three_cprs.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_found_and_not_found_in_local_and_three_cprs.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_found_and_not_found_in_local_and_three_cprs.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_one_forward.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_one_forward.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_one_forward.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_one_forward.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_shadowed_attribute.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_shadowed_attribute.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_shadowed_attribute.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_shadowed_attribute.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_three_found_on_cps_two_local_four_not_found.test b/test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_three_found_on_cps_two_local_four_not_found.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_three_found_on_cps_two_local_four_not_found.test rename to test/functionalTest/cases/3068_cprs_full_functional/fwd_v2_update_three_found_on_cps_two_local_four_not_found.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_five_different_cprs_in_one_query.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_five_different_cprs_in_one_query.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_five_different_cprs_in_one_query.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_five_different_cprs_in_one_query.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_forward_depends_on_attr_field.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_forward_depends_on_attr_field.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_forward_depends_on_attr_field.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_forward_depends_on_attr_field.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CB_and_CP1.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CB_and_CP1.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CB_and_CP1.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CB_and_CP1.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CP1_and_CP2.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CP1_and_CP2.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CP1_and_CP2.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_no_attributes_in_query_results_from_CP1_and_CP2.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_one_forward_query.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_one_forward_query.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_one_forward_query.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_one_forward_query.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_shadowed_attribute.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_shadowed_attribute.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_shadowed_attribute.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_shadowed_attribute.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_two_attributes_in_one_cpr_three_in_another.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_two_attributes_in_one_cpr_three_in_another.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_two_attributes_in_one_cpr_three_in_another.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_query_two_attributes_in_one_cpr_three_in_another.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_double_forward.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_double_forward.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_double_forward.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_double_forward.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_false_registration_and_one_more_value.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_false_registration_and_one_more_value.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_false_registration_and_one_more_value.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_false_registration_and_one_more_value.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_false_registration_and_one_more_value_in_provider.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_false_registration_and_one_more_value_in_provider.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_false_registration_and_one_more_value_in_provider.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_false_registration_and_one_more_value_in_provider.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_five_different_cprs_in_one_query.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_five_different_cprs_in_one_query.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_five_different_cprs_in_one_query.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_five_different_cprs_in_one_query.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_forward_depends_on_attr_field.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_forward_depends_on_attr_field.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_forward_depends_on_attr_field.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_forward_depends_on_attr_field.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_found_and_not_found_in_local_and_three_cprs.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_found_and_not_found_in_local_and_three_cprs.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_found_and_not_found_in_local_and_three_cprs.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_found_and_not_found_in_local_and_three_cprs.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_one_forward.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_one_forward.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_one_forward.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_one_forward.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_shadowed_attribute.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_shadowed_attribute.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_shadowed_attribute.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_shadowed_attribute.test.DISABLED diff --git a/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_three_found_on_cps_two_local_four_not_found.test b/test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_three_found_on_cps_two_local_four_not_found.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_three_found_on_cps_two_local_four_not_found.test rename to test/functionalTest/cases/3068_cprs_full_functional_v2/fwd_v2_ngsiv2_update_three_found_on_cps_two_local_four_not_found.test.DISABLED diff --git a/test/functionalTest/cases/3068_ngsi_v2_based_forwarding/fwd_ngsv1_ngsv2_interoperability.test b/test/functionalTest/cases/3068_ngsi_v2_based_forwarding/fwd_ngsv1_ngsv2_interoperability.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3068_ngsi_v2_based_forwarding/fwd_ngsv1_ngsv2_interoperability.test rename to test/functionalTest/cases/3068_ngsi_v2_based_forwarding/fwd_ngsv1_ngsv2_interoperability.test.DISABLED diff --git a/test/functionalTest/cases/3106_supported_forwarding_mode/supported_forwarding_entity_mixing_modes.test b/test/functionalTest/cases/3106_supported_forwarding_mode/supported_forwarding_entity_mixing_modes.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3106_supported_forwarding_mode/supported_forwarding_entity_mixing_modes.test rename to test/functionalTest/cases/3106_supported_forwarding_mode/supported_forwarding_entity_mixing_modes.test.DISABLED diff --git a/test/functionalTest/cases/3112_ngsiv1_autocast/create_attributes_with_autocast.test b/test/functionalTest/cases/3112_ngsiv1_autocast/create_attributes_with_autocast.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3112_ngsiv1_autocast/create_attributes_with_autocast.test rename to test/functionalTest/cases/3112_ngsiv1_autocast/create_attributes_with_autocast.test.DISABLED diff --git a/test/functionalTest/cases/3112_ngsiv1_autocast/filters_in_autocasted_attributes.test b/test/functionalTest/cases/3112_ngsiv1_autocast/filters_in_autocasted_attributes.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3112_ngsiv1_autocast/filters_in_autocasted_attributes.test rename to test/functionalTest/cases/3112_ngsiv1_autocast/filters_in_autocasted_attributes.test.DISABLED diff --git a/test/functionalTest/cases/3112_ngsiv1_autocast/update_attributes_with_autocast.test b/test/functionalTest/cases/3112_ngsiv1_autocast/update_attributes_with_autocast.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3112_ngsiv1_autocast/update_attributes_with_autocast.test rename to test/functionalTest/cases/3112_ngsiv1_autocast/update_attributes_with_autocast.test.DISABLED diff --git a/test/functionalTest/cases/3112_ngsiv1_autocast/update_only_value_with_autocast.test b/test/functionalTest/cases/3112_ngsiv1_autocast/update_only_value_with_autocast.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3112_ngsiv1_autocast/update_only_value_with_autocast.test rename to test/functionalTest/cases/3112_ngsiv1_autocast/update_only_value_with_autocast.test.DISABLED diff --git a/test/functionalTest/cases/3162_context_providers_legacy_non_primitive/query_for_arrays.test b/test/functionalTest/cases/3162_context_providers_legacy_non_primitive/query_for_arrays.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3162_context_providers_legacy_non_primitive/query_for_arrays.test rename to test/functionalTest/cases/3162_context_providers_legacy_non_primitive/query_for_arrays.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geobox_update_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geobox_update_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geobox_update_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geobox_update_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geojson_update_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geojson_update_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geojson_update_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geojson_update_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geoline_update_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geoline_update_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geoline_update_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geoline_update_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopoint_update_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopoint_update_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopoint_update_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopoint_update_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopolygon_update_does_not_break_location.test b/test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopolygon_update_does_not_break_location.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopolygon_update_does_not_break_location.test rename to test/functionalTest/cases/3442_ngsiv1_interop_location_fix/ngsiv1_geopolygon_update_does_not_break_location.test.DISABLED diff --git a/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test b/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test rename to test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test.DISABLED diff --git a/test/functionalTest/cases/3843_per_service_notif_queue/per_service_notif_queue.test b/test/functionalTest/cases/3843_per_service_notif_queue/per_service_notif_queue.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3843_per_service_notif_queue/per_service_notif_queue.test rename to test/functionalTest/cases/3843_per_service_notif_queue/per_service_notif_queue.test.DISABLED diff --git a/test/functionalTest/cases/3843_per_service_notif_queue/per_service_notif_queue_full_queues.test b/test/functionalTest/cases/3843_per_service_notif_queue/per_service_notif_queue_full_queues.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3843_per_service_notif_queue/per_service_notif_queue_full_queues.test rename to test/functionalTest/cases/3843_per_service_notif_queue/per_service_notif_queue_full_queues.test.DISABLED diff --git a/test/functionalTest/cases/3851_skip_query_forwarding/fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote_with_skip.test b/test/functionalTest/cases/3851_skip_query_forwarding/fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote_with_skip.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3851_skip_query_forwarding/fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote_with_skip.test rename to test/functionalTest/cases/3851_skip_query_forwarding/fwd_v2_ngsiv2_query_found_and_not_found_locally_and_remote_with_skip.test.DISABLED diff --git a/test/functionalTest/cases/3851_skip_query_forwarding/fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local_with_skip.test b/test/functionalTest/cases/3851_skip_query_forwarding/fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local_with_skip.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3851_skip_query_forwarding/fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local_with_skip.test rename to test/functionalTest/cases/3851_skip_query_forwarding/fwd_v2_ngsiv2_query_two_attributes_in_cpr_one_local_with_skip.test.DISABLED From a8e012b9f54d6d170d98d57fb3fe8acee81223fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 12:49:11 +0200 Subject: [PATCH 302/390] FIX disabled tests and documentation --- doc/manuals/deprecated.md | 1 + ..._params.test => pagination_error_in_uri_params.test.DISABLED} | 0 .../{accept_fail_01.test => accept_fail_01.test.DISABLED} | 0 ...ype_header.test => missing_content_type_header.test.DISABLED} | 0 ...otifyContext.test => jsonParseGetNotifyContext.test.DISABLED} | 0 ...QueryContext.test => jsonParsePostQueryContext.test.DISABLED} | 0 ...t => query_sequence_attributes_as_json_objects.test.DISABLED} | 0 ...ence_attributes_as_json_objects_with_compounds.test.DISABLED} | 0 ...eoquery_bad_coords.test => geoquery_bad_coords.test.DISABLED} | 0 ...query_circle_json.test => geoquery_circle_json.test.DISABLED} | 0 ...ery_polygon_json.test => geoquery_polygon_json.test.DISABLED} | 0 ...nge.test => location_no_actual_location_change.test.DISABLED} | 0 .../{query_sequence.test => query_sequence.test.DISABLED} | 0 .../{delete_entity.test => delete_entity.test.DISABLED} | 0 ...ute_value.test => update_empty_attribute_value.test.DISABLED} | 0 ...> update_sequence_empty_value_in_context_value.test.DISABLED} | 0 ...ate_sequence_empty_value_in_context_value_json.test.DISABLED} | 0 ..._update_query.test => http_tenant_update_query.test.DISABLED} | 0 ..._service_path.test => update_with_service_path.test.DISABLED} | 0 ...ce.test => service_ignore_without_multiservice.test.DISABLED} | 0 ...te_v1_forward.test => v2_reg_create_v1_forward.test.DISABLED} | 0 ...ttern.test => legacy-forwarding-with-idpattern.test.DISABLED} | 0 ...test => regid_global_pattern_query_forwarding2.test.DISABLED} | 0 23 files changed, 1 insertion(+) rename test/functionalTest/cases/0000_bad_requests/{pagination_error_in_uri_params.test => pagination_error_in_uri_params.test.DISABLED} (100%) rename test/functionalTest/cases/0000_content_related_headers/{accept_fail_01.test => accept_fail_01.test.DISABLED} (100%) rename test/functionalTest/cases/0000_content_related_headers/{missing_content_type_header.test => missing_content_type_header.test.DISABLED} (100%) rename test/functionalTest/cases/0000_json_parse/{jsonParseGetNotifyContext.test => jsonParseGetNotifyContext.test.DISABLED} (100%) rename test/functionalTest/cases/0000_json_parse/{jsonParsePostQueryContext.test => jsonParsePostQueryContext.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_compound_values/{query_sequence_attributes_as_json_objects.test => query_sequence_attributes_as_json_objects.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_compound_values/{query_sequence_attributes_as_json_objects_with_compounds.test => query_sequence_attributes_as_json_objects_with_compounds.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_geolocation_query/{geoquery_bad_coords.test => geoquery_bad_coords.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_geolocation_query/{geoquery_circle_json.test => geoquery_circle_json.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_geolocation_query/{geoquery_polygon_json.test => geoquery_polygon_json.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_geolocation_query/{location_no_actual_location_change.test => location_no_actual_location_change.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_query/{query_sequence.test => query_sequence.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_update/{delete_entity.test => delete_entity.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_update/{update_empty_attribute_value.test => update_empty_attribute_value.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_update/{update_sequence_empty_value_in_context_value.test => update_sequence_empty_value_in_context_value.test.DISABLED} (100%) rename test/functionalTest/cases/0000_ngsi10_update/{update_sequence_empty_value_in_context_value_json.test => update_sequence_empty_value_in_context_value_json.test.DISABLED} (100%) rename test/functionalTest/cases/0322_multitenancy/{http_tenant_update_query.test => http_tenant_update_query.test.DISABLED} (100%) rename test/functionalTest/cases/0392_service_path_update_and_query/{update_with_service_path.test => update_with_service_path.test.DISABLED} (100%) rename test/functionalTest/cases/0725_multiservice_ignore/{service_ignore_without_multiservice.test => service_ignore_without_multiservice.test.DISABLED} (100%) rename test/functionalTest/cases/3004_v2_reg_create/{v2_reg_create_v1_forward.test => v2_reg_create_v1_forward.test.DISABLED} (100%) rename test/functionalTest/cases/3458_legacy-forwarding-with-idpattern/{legacy-forwarding-with-idpattern.test => legacy-forwarding-with-idpattern.test.DISABLED} (100%) rename test/functionalTest/cases/3463_regid_global_pattern_query_forwarding/{regid_global_pattern_query_forwarding2.test => regid_global_pattern_query_forwarding2.test.DISABLED} (100%) diff --git a/doc/manuals/deprecated.md b/doc/manuals/deprecated.md index 9b289331b4..ef0574e4a7 100644 --- a/doc/manuals/deprecated.md +++ b/doc/manuals/deprecated.md @@ -39,6 +39,7 @@ A list of deprecated features and the version in which they were deprecated foll * `POST /NGSI10/updateContext` * `POST /v1/queryContext` * `POST /NGSI10/queryContext` + * Finally, the last remaining NGSIv1 operations where removed in Orion 4.0.0 * `attributes` field in `POST /v2/op/query` is in Orion 1.15.0. It is a combination of `attrs` (to select which attributes to include in the response to the query) and unary attribute filter in `q` within `expression` (to return only entities which have these attributes). Use them instead. diff --git a/test/functionalTest/cases/0000_bad_requests/pagination_error_in_uri_params.test b/test/functionalTest/cases/0000_bad_requests/pagination_error_in_uri_params.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_bad_requests/pagination_error_in_uri_params.test rename to test/functionalTest/cases/0000_bad_requests/pagination_error_in_uri_params.test.DISABLED diff --git a/test/functionalTest/cases/0000_content_related_headers/accept_fail_01.test b/test/functionalTest/cases/0000_content_related_headers/accept_fail_01.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_content_related_headers/accept_fail_01.test rename to test/functionalTest/cases/0000_content_related_headers/accept_fail_01.test.DISABLED diff --git a/test/functionalTest/cases/0000_content_related_headers/missing_content_type_header.test b/test/functionalTest/cases/0000_content_related_headers/missing_content_type_header.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_content_related_headers/missing_content_type_header.test rename to test/functionalTest/cases/0000_content_related_headers/missing_content_type_header.test.DISABLED diff --git a/test/functionalTest/cases/0000_json_parse/jsonParseGetNotifyContext.test b/test/functionalTest/cases/0000_json_parse/jsonParseGetNotifyContext.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_json_parse/jsonParseGetNotifyContext.test rename to test/functionalTest/cases/0000_json_parse/jsonParseGetNotifyContext.test.DISABLED diff --git a/test/functionalTest/cases/0000_json_parse/jsonParsePostQueryContext.test b/test/functionalTest/cases/0000_json_parse/jsonParsePostQueryContext.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_json_parse/jsonParsePostQueryContext.test rename to test/functionalTest/cases/0000_json_parse/jsonParsePostQueryContext.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_compound_values/query_sequence_attributes_as_json_objects.test b/test/functionalTest/cases/0000_ngsi10_compound_values/query_sequence_attributes_as_json_objects.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_compound_values/query_sequence_attributes_as_json_objects.test rename to test/functionalTest/cases/0000_ngsi10_compound_values/query_sequence_attributes_as_json_objects.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_compound_values/query_sequence_attributes_as_json_objects_with_compounds.test b/test/functionalTest/cases/0000_ngsi10_compound_values/query_sequence_attributes_as_json_objects_with_compounds.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_compound_values/query_sequence_attributes_as_json_objects_with_compounds.test rename to test/functionalTest/cases/0000_ngsi10_compound_values/query_sequence_attributes_as_json_objects_with_compounds.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_bad_coords.test b/test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_bad_coords.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_bad_coords.test rename to test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_bad_coords.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_circle_json.test b/test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_circle_json.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_circle_json.test rename to test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_circle_json.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_polygon_json.test b/test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_polygon_json.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_polygon_json.test rename to test/functionalTest/cases/0000_ngsi10_geolocation_query/geoquery_polygon_json.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_geolocation_query/location_no_actual_location_change.test b/test/functionalTest/cases/0000_ngsi10_geolocation_query/location_no_actual_location_change.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_geolocation_query/location_no_actual_location_change.test rename to test/functionalTest/cases/0000_ngsi10_geolocation_query/location_no_actual_location_change.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_query/query_sequence.test b/test/functionalTest/cases/0000_ngsi10_query/query_sequence.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_query/query_sequence.test rename to test/functionalTest/cases/0000_ngsi10_query/query_sequence.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_update/delete_entity.test b/test/functionalTest/cases/0000_ngsi10_update/delete_entity.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_update/delete_entity.test rename to test/functionalTest/cases/0000_ngsi10_update/delete_entity.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_update/update_empty_attribute_value.test b/test/functionalTest/cases/0000_ngsi10_update/update_empty_attribute_value.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_update/update_empty_attribute_value.test rename to test/functionalTest/cases/0000_ngsi10_update/update_empty_attribute_value.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_update/update_sequence_empty_value_in_context_value.test b/test/functionalTest/cases/0000_ngsi10_update/update_sequence_empty_value_in_context_value.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_update/update_sequence_empty_value_in_context_value.test rename to test/functionalTest/cases/0000_ngsi10_update/update_sequence_empty_value_in_context_value.test.DISABLED diff --git a/test/functionalTest/cases/0000_ngsi10_update/update_sequence_empty_value_in_context_value_json.test b/test/functionalTest/cases/0000_ngsi10_update/update_sequence_empty_value_in_context_value_json.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0000_ngsi10_update/update_sequence_empty_value_in_context_value_json.test rename to test/functionalTest/cases/0000_ngsi10_update/update_sequence_empty_value_in_context_value_json.test.DISABLED diff --git a/test/functionalTest/cases/0322_multitenancy/http_tenant_update_query.test b/test/functionalTest/cases/0322_multitenancy/http_tenant_update_query.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0322_multitenancy/http_tenant_update_query.test rename to test/functionalTest/cases/0322_multitenancy/http_tenant_update_query.test.DISABLED diff --git a/test/functionalTest/cases/0392_service_path_update_and_query/update_with_service_path.test b/test/functionalTest/cases/0392_service_path_update_and_query/update_with_service_path.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0392_service_path_update_and_query/update_with_service_path.test rename to test/functionalTest/cases/0392_service_path_update_and_query/update_with_service_path.test.DISABLED diff --git a/test/functionalTest/cases/0725_multiservice_ignore/service_ignore_without_multiservice.test b/test/functionalTest/cases/0725_multiservice_ignore/service_ignore_without_multiservice.test.DISABLED similarity index 100% rename from test/functionalTest/cases/0725_multiservice_ignore/service_ignore_without_multiservice.test rename to test/functionalTest/cases/0725_multiservice_ignore/service_ignore_without_multiservice.test.DISABLED diff --git a/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create_v1_forward.test b/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create_v1_forward.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3004_v2_reg_create/v2_reg_create_v1_forward.test rename to test/functionalTest/cases/3004_v2_reg_create/v2_reg_create_v1_forward.test.DISABLED diff --git a/test/functionalTest/cases/3458_legacy-forwarding-with-idpattern/legacy-forwarding-with-idpattern.test b/test/functionalTest/cases/3458_legacy-forwarding-with-idpattern/legacy-forwarding-with-idpattern.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3458_legacy-forwarding-with-idpattern/legacy-forwarding-with-idpattern.test rename to test/functionalTest/cases/3458_legacy-forwarding-with-idpattern/legacy-forwarding-with-idpattern.test.DISABLED diff --git a/test/functionalTest/cases/3463_regid_global_pattern_query_forwarding/regid_global_pattern_query_forwarding2.test b/test/functionalTest/cases/3463_regid_global_pattern_query_forwarding/regid_global_pattern_query_forwarding2.test.DISABLED similarity index 100% rename from test/functionalTest/cases/3463_regid_global_pattern_query_forwarding/regid_global_pattern_query_forwarding2.test rename to test/functionalTest/cases/3463_regid_global_pattern_query_forwarding/regid_global_pattern_query_forwarding2.test.DISABLED From f49fb58c83a9d762d68a9fd67e196e18024e9fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 14:57:26 +0200 Subject: [PATCH 303/390] FIX ftest --- doc/manuals/deprecated.md | 1 + src/lib/common/errorMessages.h | 2 +- .../log_deprecate_warning.test | 177 +-- .../notification_different_sizes.test | 10 +- .../statistics_with_full_counters.test | 4 +- .../previous_value_with_compounds_ngsiv1.test | 403 ------ ...cial_metadata_in_notifications_ngsiv1.test | 943 ------------- .../patch_subscription_bugfix_to_attrs.test | 7 +- ...ch_subscription_bugfix_to_except_attr.test | 7 +- ...tification_problem_with_custom_header.test | 295 ---- test/functionalTest/testHarness.sh | 1243 ----------------- 11 files changed, 31 insertions(+), 3061 deletions(-) delete mode 100644 test/functionalTest/cases/2507_csub_metadata_field/previous_value_with_compounds_ngsiv1.test delete mode 100644 test/functionalTest/cases/2507_csub_metadata_field/special_metadata_in_notifications_ngsiv1.test delete mode 100644 test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test delete mode 100755 test/functionalTest/testHarness.sh diff --git a/doc/manuals/deprecated.md b/doc/manuals/deprecated.md index 9b289331b4..c2d1f2ed69 100644 --- a/doc/manuals/deprecated.md +++ b/doc/manuals/deprecated.md @@ -39,6 +39,7 @@ A list of deprecated features and the version in which they were deprecated foll * `POST /NGSI10/updateContext` * `POST /v1/queryContext` * `POST /NGSI10/queryContext` + * NGSIv1 format in subscription notifications (`notification.atttrsFormat` set to `legacy`) removed in Orion 4.0.0 * `attributes` field in `POST /v2/op/query` is in Orion 1.15.0. It is a combination of `attrs` (to select which attributes to include in the response to the query) and unary attribute filter in `q` within `expression` (to return only entities which have these attributes). Use them instead. diff --git a/src/lib/common/errorMessages.h b/src/lib/common/errorMessages.h index a638981806..d50bf8acf7 100644 --- a/src/lib/common/errorMessages.h +++ b/src/lib/common/errorMessages.h @@ -62,7 +62,7 @@ #define ERROR_DESC_BAD_REQUEST_INVALID_JTYPE_ENTID "Invalid JSON type for entity id" #define ERROR_DESC_BAD_REQUEST_INVALID_JTYPE_ENTTYPE "Invalid JSON type for entity type" #define ERROR_DESC_BAD_REQUEST_INVALID_JTYPE_SCOPE "invalid JSON type for scope value: must be string" -#define ERROR_DESC_BAD_REQUEST_INVALID_ATTRSFORMAT "invalid attrsFormat, accepted values: legacy, normalized, simplifiedNormalized, keyValues, simplifiedkeyValues, values" +#define ERROR_DESC_BAD_REQUEST_INVALID_ATTRSFORMAT "invalid attrsFormat, accepted values: normalized, simplifiedNormalized, keyValues, simplifiedkeyValues, values" #define ERROR_DESC_BAD_REQUEST_INVALID_STATUS "status is not valid: it has to be either active, inactive or oneshot" #define ERROR_DESC_BAD_REQUEST_INVALID_RANGE "ranges only valid for equal and not equal ops" #define ERROR_DESC_BAD_REQUEST_INVALID_LIST "lists only valid for equal and not equal ops" diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index fe57f14647..a340c8cd99 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -41,12 +41,8 @@ brokerStart CB 0 IPV4 -logDeprecate -statCounters # 06. Retrieve registrations (which uses legacyForwarding mode) # 07. Forwarded query using legacyForwarding mode # 08. Update query using legacyForwarding mode -# 09. Create subscription using attrsFormat legacy -# 10. Retrieve subscriptions (which uses attrsFormat legacy) -# 11. Update subscription using attrsFormat legacy -# 12. Trigger notification using attrsFormat legacy -# 13. Get WARNING trace in logs -# 14. Get statistics and see deprecatedFeatures counters +# 09. Get WARNING trace in logs +# 10. Get statistics and see deprecatedFeatures counters # echo "01. Query E1-T1" @@ -163,77 +159,14 @@ echo echo -echo "09. Create subscription using attrsFormat legacy" -echo "================================================" -payload='{ - "subject": { - "entities": [ - { - "id": "E2", - "type": "T2" - } - ] - }, - "notification": { - "http": { - "url": "http://localhost:1234" - }, - "attrsFormat": "legacy" - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") - - -echo "10. Retrieve subscriptions (which uses attrsFormat legacy)" -echo "==========================================================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "11. Update subscription using attrsFormat legacy" -echo "================================================" -payload='{ - "notification": { - "http": { - "url": "http://localhost:1234" - }, - "attrsFormat": "legacy" - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID -X PATCH --payload "$payload" -echo -echo - - -echo "12. Trigger notification using attrsFormat legacy" -echo "=================================================" -payload='{ - "id": "E2", - "type": "T2", - "A": { - "value": 1, - "type": "Number" - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "13. Get WARNING trace in logs" +echo "09. Get WARNING trace in logs" echo "=============================" -cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' | sed -e "s/$REG_ID/REG_ID/g" | sed -e "s/$SUB_ID/SUB_ID/g" +cat /tmp/contextBroker.log | grep 'WARN' | awk -F 'msg=' '{print $2}' | sed -e "s/$REG_ID/REG_ID/g" echo echo -echo "14. Get statistics and see deprecatedFeatures counters" +echo "10. Get statistics and see deprecatedFeatures counters" echo "======================================================" orionCurl --url /statistics echo @@ -388,72 +321,7 @@ Content-Length: 95 } -09. Create subscription using attrsFormat legacy -================================================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f\-]{24}) -Content-Length: 0 - - - -10. Retrieve subscriptions (which uses attrsFormat legacy) -========================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 288 - -[ - { - "id": "REGEX([0-9a-f\-]{24})", - "notification": { - "attrs": [], - "attrsFormat": "legacy", - "covered": false, - "http": { - "url": "http://localhost:1234" - }, - "onlyChangedAttrs": false - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "id": "E2", - "type": "T2" - } - ] - } - } -] - - -11. Update subscription using attrsFormat legacy -================================================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -12. Trigger notification using attrsFormat legacy -================================================= -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E2?type=T2 -Content-Length: 0 - - - -13. Get WARNING trace in logs +09. Get WARNING trace in logs ============================= Deprecated NGSIv1 request received: POST /v1/queryContext, request payload (48 bytes): { "entities": [ { "type": "T1", "id": "E1" } ] }, response code: 200 Deprecated NGSIv1 request received: GET /v1/contextEntities/E/attributes/A, response code: 200 @@ -463,25 +331,23 @@ Deprecated usage of geo:point detected in attribute location at entity update, p Deprecated usage of legacyForwarding mode in registration creation (regId: REG_ID) Deprecated usage of legacyForwarding mode detected in existing registration (regId: REG_ID) Deprecated usage of legacyForwarding mode in query forwarding operation (regId: REG_ID) -Raising alarm ForwardingError localhost:9801/v2/queryContext: forwarding failure for sender-thread: Couldn't connect to server -Deprecated usage of legacyForwarding mode in update forwarding operation (regId: REG_ID) -Raising alarm ForwardingError localhost:9801/v2/updateContext: forwarding failure for sender-thread: Couldn't connect to server -Raising alarm BadInput 127.0.0.1: The requested entity has not been found. Check type and id +Notification (regId: REG_ID) response NOT OK, http code: 400 +Raising alarm BadInput 127.0.0.1: JSON Parse Error: unknown field: /error +Raising alarm ForwardingError localhost:9801/v2/queryContext: error parsing reply from prov app: {"error":"BadRequest","description":"JSON Parse Error: unknown field: /error"} Releasing alarm BadInput 127.0.0.1 -Deprecated usage of notification legacy format in subscription creation (subId: SUB_ID) -Deprecated usage of notification legacy format detected in existing subscription (subId: SUB_ID) -Deprecated usage of notification legacy format in subscription modification (subId: SUB_ID) -Deprecated usage of notification legacy format in notification (subId: SUB_ID) -Raising alarm NotificationError localhost:1234/: notification failure for queue worker: Couldn't connect to server +Deprecated usage of legacyForwarding mode in update forwarding operation (regId: REG_ID) +Notification (regId: REG_ID) response NOT OK, http code: 400 +Raising alarm BadInput 127.0.0.1: JSON Parse Error: unknown field: /error +Raising alarm ForwardingError localhost:9801/v2/updateContext: error parsing reply from prov app: {"error":"BadRequest","description":"JSON Parse Error: unknown field: /error"} -14. Get statistics and see deprecatedFeatures counters +10. Get statistics and see deprecatedFeatures counters ====================================================== HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 566 +Content-Length: 468 { "counters": { @@ -490,14 +356,14 @@ Content-Length: 566 "ngsiv1Forwarding": 4, "ngsiv1Requests": 3 }, - "jsonRequests": 8, - "noPayloadRequests": 5, + "jsonRequests": 5, + "noPayloadRequests": 4, "requests": { "/statistics": { "GET": 1 }, "/v2/entities": { - "POST": 2 + "POST": 1 }, "/v2/entities/{id}/attrs/{name}": { "GET": 1, @@ -506,13 +372,6 @@ Content-Length: 566 "/v2/registrations": { "GET": 1, "POST": 1 - }, - "/v2/subscriptions": { - "GET": 1, - "POST": 1 - }, - "/v2/subscriptions/{id}": { - "PATCH": 1 } }, "requestsLegacy": { diff --git a/test/functionalTest/cases/0000_large_requests/notification_different_sizes.test b/test/functionalTest/cases/0000_large_requests/notification_different_sizes.test index 79b31be3bd..3c50b6e889 100644 --- a/test/functionalTest/cases/0000_large_requests/notification_different_sizes.test +++ b/test/functionalTest/cases/0000_large_requests/notification_different_sizes.test @@ -63,7 +63,6 @@ payload='{ ] }, "notification": { - "attrsFormat": "legacy", "http": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" }, @@ -87,7 +86,6 @@ payload='{ ] }, "notification": { - "attrsFormat": "legacy", "http": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" }, @@ -111,7 +109,6 @@ payload='{ ] }, "notification": { - "attrsFormat": "legacy", "http": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" }, @@ -135,7 +132,6 @@ payload='{ ] }, "notification": { - "attrsFormat": "legacy", "http": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" }, @@ -298,9 +294,9 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) 08. Check notification sizes ============================ -Sending message 1 to HTTP server: sending message of REGEX((19496|19501|19497|19502)) bytes to HTTP server -Sending message 2 to HTTP server: sending message of REGEX((21496|21501|21497|21502)) bytes to HTTP server -Sending message 3 to HTTP server: sending message of REGEX((8100498|8100503|8100499|8100504)) bytes to HTTP server +Sending message 1 to HTTP server: sending message of REGEX((19400|19401|19405|19406)) bytes to HTTP server +Sending message 2 to HTTP server: sending message of REGEX((21400|21401|21405|21406)) bytes to HTTP server +Sending message 3 to HTTP server: sending message of REGEX((8100402|8100403|8100407|8100408)) bytes to HTTP server --TEARDOWN-- diff --git a/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test b/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test index 14dda84090..dfc021a52d 100644 --- a/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test +++ b/test/functionalTest/cases/0000_statistics_operation/statistics_with_full_counters.test @@ -31,7 +31,7 @@ brokerStart CB 0 IPv4 -statCounters # # This test is special and doesn't follow the typical steps pattern -# Instead of that, we just stimulate some possible API rute (without output) +# Instead of that, we just stimulate some possible API routes (without output) # and do GET /statistics after that # @@ -46,7 +46,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1624 +Content-Length: 1602 { "counters": { diff --git a/test/functionalTest/cases/2507_csub_metadata_field/previous_value_with_compounds_ngsiv1.test b/test/functionalTest/cases/2507_csub_metadata_field/previous_value_with_compounds_ngsiv1.test deleted file mode 100644 index 5e8d90fb59..0000000000 --- a/test/functionalTest/cases/2507_csub_metadata_field/previous_value_with_compounds_ngsiv1.test +++ /dev/null @@ -1,403 +0,0 @@ -# Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -previousValue with compound cases - ---SHELL-INIT-- -dbInit CB -brokerStart CB -accumulatorStart --pretty-print - ---SHELL-- - -# -# Same script as previous_value_with_compounds.test but using "legacy" as attrsFormat -# -# 01. Create subscription with previousValue as metadata -# 02. Create E1 with attributes A: 1 -# 03. Dump and reset: see notification A:1 and no previousValue -# 04. Update A: [1, {b: foo} ] -# 05. Dump and reset: see notification A: [1, {b: foo} ], previousValue: 1 -# 06. Update A: {x: 1, y: [true, foo] } -# 07. Dump and reset: see notification A: {x: 1, y: [true: foo] }, previousValue: [1, {b: foo} ] -# 08. Update A: x -# 09. Dump and reset: see notification A: x, previousValue: {x: 1, y: [true, foo] } -# - -echo "01. Create subscription with previousValue as metadata" -echo "======================================================" -payload='{ - "subject": { - "entities": [ - { - "id": "E1", - "type": "T" - } - ], - "condition": { - "attrs": [ ] - } - }, - "notification": { - "http": { - "url": "http://localhost:'$LISTENER_PORT'/notify" - }, - "attrs": [ ], - "attrsFormat": "legacy", - "metadata": [ "previousValue" ] - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "02. Create E1 with attributes A: 1" -echo "==================================" -payload='{ - "type": "T", - "id": "E1", - "A": { - "type": "Number", - "value": 1 - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "03. Dump and reset: see notification A:1 and no previousValue" -echo "=============================================================" -accumulatorDump -accumulatorReset -echo -echo - - -echo "04. Update A: [1, {b: foo} ]" -echo "============================" -payload='{ - "A": { - "type": "ComplexVector", - "value": [1, {"b": "foo"} ] - } -}' -orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" -echo -echo - - -echo "05. Dump and reset: see notification A: [1, {b: foo} ], previousValue: 1" -echo "========================================================================" -accumulatorDump -accumulatorReset -echo -echo - - -echo "06. Update A: {x: 1, y: [true, foo] }" -echo "=====================================" -payload='{ - "A": { - "type": "ComplexObject", - "value": {"x": 1, "y": [true, "foo"] } - } -}' -orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" -echo -echo - - -echo "07. Dump and reset: see notification A: {x: 1, y: [true: foo] }, previousValue: [1, {b: foo} ]" -echo "==============================================================================================" -accumulatorDump -accumulatorReset -echo -echo - -echo "08. Update A: x" -echo "===============" -payload='{ - "A": { - "type": "Text", - "value": "x" - } -}' -orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" -echo -echo - - -echo "09. Dump and reset: see notification A: x, previousValue: {x: 1, y: [true, foo] }" -echo "=================================================================================" -accumulatorDump -accumulatorReset -echo -echo - - ---REGEXPECT-- -01. Create subscription with previousValue as metadata -====================================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -02. Create E1 with attributes A: 1 -================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T -Content-Length: 0 - - - -03. Dump and reset: see notification A:1 and no previousValue -============================================================= -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 255 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "name": "A", - "type": "Number", - "value": 1 - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - -04. Update A: [1, {b: foo} ] -============================ -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -05. Dump and reset: see notification A: [1, {b: foo} ], previousValue: 1 -======================================================================== -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 341 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "previousValue", - "type": "Number", - "value": 1 - } - ], - "name": "A", - "type": "ComplexVector", - "value": [ - 1, - { - "b": "foo" - } - ] - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - -06. Update A: {x: 1, y: [true, foo] } -===================================== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -07. Dump and reset: see notification A: {x: 1, y: [true: foo] }, previousValue: [1, {b: foo} ] -============================================================================================== -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 371 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "previousValue", - "type": "ComplexVector", - "value": [ - 1, - { - "b": "foo" - } - ] - } - ], - "name": "A", - "type": "ComplexObject", - "value": { - "x": 1, - "y": [ - true, - "foo" - ] - } - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - -08. Update A: x -=============== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -09. Dump and reset: see notification A: x, previousValue: {x: 1, y: [true, foo] } -================================================================================= -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 350 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "previousValue", - "type": "ComplexObject", - "value": { - "x": 1, - "y": [ - true, - "foo" - ] - } - } - ], - "name": "A", - "type": "Text", - "value": "x" - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - ---TEARDOWN-- -brokerStop CB -accumulatorStop $LISTENER_PORT -dbDrop CB diff --git a/test/functionalTest/cases/2507_csub_metadata_field/special_metadata_in_notifications_ngsiv1.test b/test/functionalTest/cases/2507_csub_metadata_field/special_metadata_in_notifications_ngsiv1.test deleted file mode 100644 index d04b13c74c..0000000000 --- a/test/functionalTest/cases/2507_csub_metadata_field/special_metadata_in_notifications_ngsiv1.test +++ /dev/null @@ -1,943 +0,0 @@ -# Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Special metadata in NGSIv1 notifications - ---SHELL-INIT-- -dbInit CB -brokerStart CB -accumulatorStart --pretty-print - ---SHELL-- - -# -# Same script as special_metadata_in_notifications_ngsiv1.test but using "legacy" as attrsFormat -# -# 01. Subscribe to E.* for A, B and C; triggered by B, C or D -# 02. Create E1 with attributes A:1, B:2, C:3, D:4 -# 03. Dump and reset: see notification with A, B and C -# 04. Update B:20 -# 05. Dump and reset: see notification with actionType=update and previousValue=2 for B -# 06. Update B:20, C:30 -# 07. Dump and reset: see notification with actionType=update for B and C, previousValue=20 for B, previousValue=3 for C -# 08. Update A:1, D:40 -# 09. Dump and reset: see notification with actionType=update and previousValue=1 for A -# 10. Update A:10, D:41 -# 11. Dump and reset: see notification with actionType=update and previousValue=10 for A -# 12. Create E2 with attributes A:x, B:y, C:z, D:u -# 13. Dump and reset: see notification with actionType=append for A, B, C -# 14. Update subscription conditions -# 15. Update E1 A:10, D:41 (forced) -# 16. Update E2 A:x, B:y, C:z, D:u (forced) -# 15. Dump and reset: see notification with 2 entities with A, B and C -# - - -echo "01. Subscribe to E.* for A, B and C; triggered by B, C or D" -echo "===========================================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern": "E.*", - "type": "T" - } - ], - "condition": { - "attrs": [ "B", "C", "D" ] - } - }, - "notification": { - "http": { - "url": "http://localhost:'$LISTENER_PORT'/notify" - }, - "attrs": [ "A", "B", "C" ], - "attrsFormat": "legacy", - "metadata": [ "actionType", "previousValue" ] - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - -SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") - - -echo "02. Create E1 with attributes A:1, B:2, C:3, D:4" -echo "================================================" -payload='{ - "type": "T", - "id": "E1", - "A": { - "type": "Number", - "value": 1 - }, - "B": { - "type": "Number", - "value": 2 - }, - "C": { - "type": "Number", - "value": 3 - }, - "D": { - "type": "Number", - "value": 4 - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "03. Dump and reset: see notification with A, B and C" -echo "====================================================" -accumulatorDump -accumulatorReset -echo -echo - - -echo "04. Update B:20" -echo "===============" -payload='{ - "B": { - "type": "Number", - "value": 20 - } -}' -orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" -echo -echo - - -echo "05. Dump and reset: see notification with actionType=update and previousValue=2 for B" -echo "=====================================================================================" -accumulatorDump -accumulatorReset -echo -echo - - -echo "06. Update B:20, C:30" -echo "=====================" -payload='{ - "B": { - "type": "Number", - "value": 20 - }, - "C": { - "type": "Number", - "value": 30 - } -}' -orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" -echo -echo - - -echo "07. Dump and reset: see notification with actionType=update for B and C, previousValue=20 for B, previousValue=3 for C" -echo "======================================================================================================================" -accumulatorDump -accumulatorReset -echo -echo - -echo "08. Update A:1, D:40" -echo "====================" -payload='{ - "A": { - "type": "Number", - "value": 1 - }, - "D": { - "type": "Number", - "value": 40 - } -}' -orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" -echo -echo - - -echo "09. Dump and reset: see notification with actionType=update and previousValue=1 for A" -echo "=====================================================================================" -accumulatorDump -accumulatorReset -echo -echo - - -echo "10. Update A:10, D:41" -echo "=====================" -payload='{ - "A": { - "type": "Number", - "value": 10 - }, - "D": { - "type": "Number", - "value": 41 - } -}' -orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" -echo -echo - - -echo "11. Dump and reset: see notification with actionType=update and previousValue=10 for A" -echo "======================================================================================" -accumulatorDump -accumulatorReset -echo -echo - - -echo "12. Create E2 with attributes A:x, B:y, C:z, D:u" -echo "================================================" -payload='{ - "type": "T", - "id": "E2", - "A": { - "type": "Text", - "value": "x" - }, - "B": { - "type": "Text", - "value": "y" - }, - "C": { - "type": "Text", - "value": "z" - }, - "D": { - "type": "Text", - "value": "u" - } -}' -orionCurl --url /v2/entities --payload "$payload" -echo -echo - - -echo "13. Dump and reset: see notification with actionType=append for A, B, C" -echo "=======================================================================" -accumulatorDump -accumulatorReset -echo -echo -echo - - -echo "14. Update subscription conditions" -echo "==================================" -payload='{ - "subject": { - "entities": [ - { - "idPattern": "E.*", - "type": "T" - } - ], - "condition": { - "attrs": [ "B", "C", "D" ] - } - } -}' -orionCurl --url /v2/subscriptions/$SUB_ID -X PATCH --payload "$payload" -echo -echo - - -echo "15. Update E1 A:10, D:41 (forced)" -echo "=================================" -payload='{ - "A": { - "type": "Number", - "value": 10 - }, - "D": { - "type": "Number", - "value": 41 - } -}' -orionCurl --url /v2/entities/E1/attrs?options=forcedUpdate -X PATCH --payload "$payload" -echo -echo - - -echo "16. Update E2 A:x, B:y, C:z, D:u (forced)" -echo "=========================================" -payload='{ - "A": { - "type": "Text", - "value": "x" - }, - "B": { - "type": "Text", - "value": "y" - }, - "C": { - "type": "Text", - "value": "z" - }, - "D": { - "type": "Text", - "value": "u" - } -}' -orionCurl --url /v2/entities/E2/attrs?options=forcedUpdate --payload "$payload" -echo -echo - - -echo "17. Dump and reset: see notification with 2 entities with A, B and C" -echo "====================================================================" -accumulatorDump -accumulatorReset -echo -echo - - ---REGEXPECT-- -01. Subscribe to E.* for A, B and C; triggered by B, C or D -=========================================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -02. Create E1 with attributes A:1, B:2, C:3, D:4 -================================================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=T -Content-Length: 0 - - - -03. Dump and reset: see notification with A, B and C -==================================================== -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 534 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "append" - } - ], - "name": "A", - "type": "Number", - "value": 1 - }, - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "append" - } - ], - "name": "B", - "type": "Number", - "value": 2 - }, - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "append" - } - ], - "name": "C", - "type": "Number", - "value": 3 - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - -04. Update B:20 -=============== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -05. Dump and reset: see notification with actionType=update and previousValue=2 for B -===================================================================================== -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 452 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "name": "A", - "type": "Number", - "value": 1 - }, - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "update" - }, - { - "name": "previousValue", - "type": "Number", - "value": 2 - } - ], - "name": "B", - "type": "Number", - "value": 20 - }, - { - "name": "C", - "type": "Number", - "value": 3 - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - -06. Update B:20, C:30 -===================== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -07. Dump and reset: see notification with actionType=update for B and C, previousValue=20 for B, previousValue=3 for C -====================================================================================================================== -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 572 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "name": "A", - "type": "Number", - "value": 1 - }, - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "update" - }, - { - "name": "previousValue", - "type": "Number", - "value": 20 - } - ], - "name": "B", - "type": "Number", - "value": 20 - }, - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "update" - }, - { - "name": "previousValue", - "type": "Number", - "value": 3 - } - ], - "name": "C", - "type": "Number", - "value": 30 - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - -08. Update A:1, D:40 -==================== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -09. Dump and reset: see notification with actionType=update and previousValue=1 for A -===================================================================================== -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 453 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "update" - }, - { - "name": "previousValue", - "type": "Number", - "value": 1 - } - ], - "name": "A", - "type": "Number", - "value": 1 - }, - { - "name": "B", - "type": "Number", - "value": 20 - }, - { - "name": "C", - "type": "Number", - "value": 30 - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - -10. Update A:10, D:41 -===================== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -11. Dump and reset: see notification with actionType=update and previousValue=10 for A -====================================================================================== -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 454 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "update" - }, - { - "name": "previousValue", - "type": "Number", - "value": 1 - } - ], - "name": "A", - "type": "Number", - "value": 10 - }, - { - "name": "B", - "type": "Number", - "value": 20 - }, - { - "name": "C", - "type": "Number", - "value": 30 - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - -12. Create E2 with attributes A:x, B:y, C:z, D:u -================================================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E2?type=T -Content-Length: 0 - - - -13. Dump and reset: see notification with actionType=append for A, B, C -======================================================================= -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 534 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "append" - } - ], - "name": "A", - "type": "Text", - "value": "x" - }, - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "append" - } - ], - "name": "B", - "type": "Text", - "value": "y" - }, - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "append" - } - ], - "name": "C", - "type": "Text", - "value": "z" - } - ], - "id": "E2", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - - -14. Update subscription conditions -================================== -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -15. Update E1 A:10, D:41 (forced) -================================= -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -16. Update E2 A:x, B:y, C:z, D:u (forced) -========================================= -HTTP/1.1 204 No Content -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) - - - -17. Dump and reset: see notification with 2 entities with A, B and C -==================================================================== -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 455 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "update" - }, - { - "name": "previousValue", - "type": "Number", - "value": 10 - } - ], - "name": "A", - "type": "Number", - "value": 10 - }, - { - "name": "B", - "type": "Number", - "value": 20 - }, - { - "name": "C", - "type": "Number", - "value": 30 - } - ], - "id": "E1", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 687 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "update" - }, - { - "name": "previousValue", - "type": "Text", - "value": "x" - } - ], - "name": "A", - "type": "Text", - "value": "x" - }, - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "update" - }, - { - "name": "previousValue", - "type": "Text", - "value": "y" - } - ], - "name": "B", - "type": "Text", - "value": "y" - }, - { - "metadatas": [ - { - "name": "actionType", - "type": "Text", - "value": "update" - }, - { - "name": "previousValue", - "type": "Text", - "value": "z" - } - ], - "name": "C", - "type": "Text", - "value": "z" - } - ], - "id": "E2", - "isPattern": "false", - "type": "T" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "originator": "localhost", - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= - - ---TEARDOWN-- -brokerStop CB -accumulatorStop $LISTENER_PORT -dbDrop CB diff --git a/test/functionalTest/cases/2647_patch_subscription_bugfix/patch_subscription_bugfix_to_attrs.test b/test/functionalTest/cases/2647_patch_subscription_bugfix/patch_subscription_bugfix_to_attrs.test index c9277dbbae..762a14683a 100644 --- a/test/functionalTest/cases/2647_patch_subscription_bugfix/patch_subscription_bugfix_to_attrs.test +++ b/test/functionalTest/cases/2647_patch_subscription_bugfix/patch_subscription_bugfix_to_attrs.test @@ -67,8 +67,7 @@ payload='{ "url": "http://localhost:1234" }, "exceptAttrs": ["temperature", "humidity"], - "metadata": [ "previousValue", "actionType" , "*"], - "attrsFormat": "legacy" + "metadata": [ "previousValue", "actionType" , "*"] }, "throttling": 5, "expires": "2016-04-05T14:00:00.00Z", @@ -128,14 +127,14 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 670 +Content-Length: 674 { "description": "One subscription to rule them all", "expires": "2016-04-05T14:00:00.000Z", "id": "REGEX([0-9a-f]{24})", "notification": { - "attrsFormat": "legacy", + "attrsFormat": "normalized", "covered": false, "exceptAttrs": [ "temperature", diff --git a/test/functionalTest/cases/2647_patch_subscription_bugfix/patch_subscription_bugfix_to_except_attr.test b/test/functionalTest/cases/2647_patch_subscription_bugfix/patch_subscription_bugfix_to_except_attr.test index bc90656ff9..2f59d462b8 100644 --- a/test/functionalTest/cases/2647_patch_subscription_bugfix/patch_subscription_bugfix_to_except_attr.test +++ b/test/functionalTest/cases/2647_patch_subscription_bugfix/patch_subscription_bugfix_to_except_attr.test @@ -67,8 +67,7 @@ payload='{ "url": "http://localhost:1234" }, "attrs": ["temperature", "humidity"], - "metadata": [ "previousValue", "actionType" , "*"], - "attrsFormat": "legacy" + "metadata": [ "previousValue", "actionType" , "*"] }, "throttling": 5, "expires": "2016-04-05T14:00:00.00Z", @@ -128,7 +127,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 664 +Content-Length: 668 { "description": "One subscription to rule them all", @@ -139,7 +138,7 @@ Content-Length: 664 "temperature", "humidity" ], - "attrsFormat": "legacy", + "attrsFormat": "normalized", "covered": false, "http": { "url": "http://localhost:1234" diff --git a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test b/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test deleted file mode 100644 index b4e9cc5695..0000000000 --- a/test/functionalTest/cases/3154_notification_problem_with_custom_header/notification_problem_with_custom_header.test +++ /dev/null @@ -1,295 +0,0 @@ -# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Notification Templates, with refresh of the subscription cache - ---SHELL-INIT-- -dbInit CB -brokerStart CB 0 -accumulatorStart --pretty-print - ---SHELL-- - -# -# 01. Create custom subscription -# 02. Create custom subscription with payload inside httpCustom and with "attrsFormat" == "legacy" -# 03. Create entity to trigger the subscriptions -# 04. GET the subscriptions -# 05. Dump accumulator and see first notification in legacy mode, second notification not in legacy mode, as custom payload overrides -# - -echo "01. Create custom subscription" -echo "==============================" -payload='{ - "description": "not custom", - "expires": "2040-01-01T14:00:00.00Z", - "subject": { - "entities": [ - { - "idPattern" : ".*" - } - ], - "condition": { - "attrs": [] - } - }, - "notification": { - "attrs": [], - "attrsFormat": "legacy", - "httpCustom": { - "url": "http://localhost:'${LISTENER_PORT}'/notify" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo '02. Create custom subscription with payload inside httpCustom and with "attrsFormat" == "legacy"' -echo "================================================================================================" -payload='{ - "description": "custom", - "expires": "2040-01-01T14:00:00.00Z", - "subject": { - "entities": [ - { - "idPattern" : ".*" - } - ], - "condition": { - "attrs": [] - } - }, - "notification": { - "attrs": [], - "attrsFormat": "legacy", - "httpCustom": { - "url": "http://localhost:'${LISTENER_PORT}'/notify", - "payload": "{ %22A1%22: %22Value of A1: ${A1}%22 }" - } - } -}' -orionCurl --url /v2/subscriptions --payload "$payload" -echo -echo - - -echo "03. Create entity to trigger the subscriptions" -echo "==============================================" -payload='{ - "id": "E1", - "A1": "a1", - "A2": "a2", - "A3": 13 -}' -orionCurl --url /v2/entities?options=keyValues --payload "$payload" -echo -echo - - -echo "04. GET the subscriptions" -echo "=========================" -orionCurl --url /v2/subscriptions -echo -echo - - -echo "05. Dump accumulator and see first notification in legacy mode, second notification not in legacy mode, as custom payload overrides" -echo "===================================================================================================================================" -accumulatorDump -echo -echo - - ---REGEXPECT-- -01. Create custom subscription -============================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -02. Create custom subscription with payload inside httpCustom and with "attrsFormat" == "legacy" -================================================================================================ -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/subscriptions/REGEX([0-9a-f]{24}) -Content-Length: 0 - - - -03. Create entity to trigger the subscriptions -============================================== -HTTP/1.1 201 Created -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Location: /v2/entities/E1?type=Thing -Content-Length: 0 - - - -04. GET the subscriptions -========================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 1012 - -[ - { - "description": "not custom", - "expires": "2040-01-01T14:00:00.000Z", - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "legacy", - "covered": false, - "httpCustom": { - "url": "http://localhost:9997/notify" - }, - "lastNotification": "REGEX(.*)", - "lastSuccess": "REGEX(.*)", - "lastSuccessCode": 200, - "onlyChangedAttrs": false, - "timesSent": 1 - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*" - } - ] - } - }, - { - "description": "custom", - "expires": "2040-01-01T14:00:00.000Z", - "id": "REGEX([0-9a-f]{24})", - "notification": { - "attrs": [], - "attrsFormat": "legacy", - "covered": false, - "httpCustom": { - "payload": "{ %22A1%22: %22Value of A1: ${A1}%22 }", - "url": "http://localhost:9997/notify" - }, - "lastNotification": "REGEX(.*)", - "lastSuccess": "REGEX(.*)", - "lastSuccessCode": 200, - "onlyChangedAttrs": false, - "timesSent": 1 - }, - "status": "active", - "subject": { - "condition": { - "attrs": [], - "notifyOnMetadataChange": true - }, - "entities": [ - { - "idPattern": ".*" - } - ] - } - } -] - - -05. Dump accumulator and see first notification in legacy mode, second notification not in legacy mode, as custom payload overrides -=================================================================================================================================== -#SORT_START -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 316 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: application/json; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[12]) - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "name": "A1", - "type": "Text", - "value": "a1" - }, - { - "name": "A2", - "type": "Text", - "value": "a2" - }, - { - "name": "A3", - "type": "Number", - "value": 13 - } - ], - "id": "E1", - "isPattern": "false", - "type": "Thing" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "" - } - } - ], - "subscriptionId": "REGEX([0-9a-f]{24})" -} -======================================= -POST http://localhost:REGEX(\d+)/notify -Fiware-Servicepath: / -Content-Length: 27 -User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) -Ngsiv2-Attrsformat: custom -Host: localhost:REGEX(\d+) -Accept: application/json -Content-Type: text/plain; charset=utf-8 -Fiware-Correlator: REGEX([0-9a-f\-]{36}; cbnotif=[12]) - -{ - "A1": "Value of A1: a1" -} -======================================= -#SORT_END - - ---TEARDOWN-- -brokerStop CB -dbDrop CB -accumulatorStop diff --git a/test/functionalTest/testHarness.sh b/test/functionalTest/testHarness.sh deleted file mode 100755 index 9b30488480..0000000000 --- a/test/functionalTest/testHarness.sh +++ /dev/null @@ -1,1243 +0,0 @@ -#!/bin/bash -# -*- coding: latin-1 -*- -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es -# -# Author: Ken Zangelin - - -date -testStartTime=$(date +%s.%2N) -MAX_TRIES=${CB_MAX_TRIES:-3} -echo $testStartTime > /tmp/brokerStartCounter - - -# ----------------------------------------------------------------------------- -# -# DISABLED - funct tests that are disabled, for some reason -# -DISABLED=('test/functionalTest/cases/0000_bad_requests/exit.test' \ - 'test/functionalTest/cases/0917_queryContext_behaves_differently/query_with_and_without_forwarding.test' \ - 'test/functionalTest/cases/0000_ipv6_support/ipv4_only.test' \ - 'test/functionalTest/cases/0000_ipv6_support/ipv6_only.test' \ - 'test/functionalTest/cases/1310_suspect_200OK/suspect_200OK.test'); - - - -# ------------------------------------------------------------------------------ -# -# Find out in which directory this script resides -# -dirname=$(dirname $0) - -if [ "$dirname" == "" ] -then - dirname="." -fi - -cd $dirname -export SCRIPT_HOME=$(pwd) -cd - > /dev/null 2>&1 - - - -# ------------------------------------------------------------------------------ -# -# Debug mode? -# -if [ "$ORION_FT_DEBUG" == "1" ] -then - _debug='on' -fi - - - -# ----------------------------------------------------------------------------- -# -# Log file for debugging -# -rm -f /tmp/orionFuncTestDebug.log -echo $(date) > /tmp/orionFuncTestDebug.log - - - -# ----------------------------------------------------------------------------- -# -# Env vars -# -export LC_ALL=C -export NAME="testFile" -declare -A testErrorV -typeset -i testError -declare -A okOnSecondV -typeset -i okOnSecond -declare -A okOnThirdV -typeset -i okOnThird -declare -A okOnPlus3V -typeset -i okOnPlus3 -declare -A skipV -typeset -i skips -declare -A disabledTestV -typeset -i disabledTests -declare -A notInFlavourTestV -typeset -i notInFlavourTests - -export DIFF=$SCRIPT_HOME/testDiff.py -testError=0 -okOnSecond=0 -okOnThird=0 -okOnPlus3=0 -skips=0 -disabledTests=0 -notInFlavourTests=0 - - -# ----------------------------------------------------------------------------- -# -# Default value of skipList taken from an env var, to make things a little -# easier in distros with constantly failing tests -# -skipList="$CB_SKIP_LIST" - - - -# ----------------------------------------------------------------------------- -# -# usage -# -function usage() -{ - sfile="Usage: "$(basename $0) - empty=$(echo $sfile | tr 'a-zA-z/0-9.:' ' ') - echo "$sfile [-u (usage)]" - echo "$empty [-v (verbose)]" - echo "$empty [--filter ]" - echo "$empty [--match ]" - echo "$empty [--keep (don't remove output files)]" - echo "$empty [--dryrun (don't execute any tests)]" - echo "$empty [--dir ]" - echo "$empty [--fromIx ]" - echo "$empty [--toIx ]" - echo "$empty [--ixList ]" - echo "$empty [--skipList ]" - echo "$empty [--stopOnError (stop at first error encountered)]" - echo "$empty [--no-duration (removes duration mark on successful tests)]" - echo "$empty [--noCache (force broker to be started with the option --noCache)]" - echo "$empty [--cache (force broker to be started without the option --noCache)]" - echo "$empty [--noThreadpool (do not use a threadpool, unless specified by a test case. If not set, a thread pool of 200:20 is used by default in test cases which do not set notificationMode options)]" - echo "$empty [--xbroker (use external brokers, i.e. this script will *not* start any brokers, including context providers)]" - echo "$empty [ ]*" - echo - echo "* Please note that if a directory is passed as parameter, its entire path must be given, not only the directory-name" - echo "* If a file is passed as parameter, its entire file-name must be given, including '.test'" - echo "" - echo "Env Vars:" - echo "CB_MAX_TRIES: The number of tries before giving up on a failing test case" - echo "CB_SKIP_LIST: Default value for option --skipList" - echo "CB_SKIP_FUNC_TESTS: List of names of func tests to skip" - echo "CB_NO_CACHE: Start the broker without subscription cache (if set to 'ON')" - echo "CB_THREADPOOL: Start the broker without thread pool (if set to 'OFF')" - echo "CB_DIFF_TOOL: To view diff of failing tests with diff/tkdiff/meld/... (e.g. export CB_DIFF_TOOL=tkdiff)" - echo "CB_WITH_EXTERNAL_BROKER: The broker is started externally - not 'automatically' by the test harness (if set to 'ON')" - echo "" - echo "FT_FROM_IX: alternative to commandline parameter 'fromIx', index of test where to start (inclusive) " - echo "FT_TO_IX: alternative to commandline parameter 'toIx', index of test where to end (inclusive)" - echo - echo "Please note that, if using CB_WITH_EXTERNAL_BROKER (or --xbroker, which is the same), only a single test case should be run." - echo - exit $1 -} - - - -# ----------------------------------------------------------------------------- -# -# vMsg -# -function vMsg() -{ - if [ "$verbose" = "on" ] - then - echo $ME: $* - fi -} - - - -# ----------------------------------------------------------------------------- -# -# exitFunction -# -function exitFunction() -{ - exitCode=$1 - errorText=$2 - testFile=$3 - errorString=$4 - errorFile=$5 - forced=$6 - - echo -n "(FAIL $exitCode - $errorText) " - - if [ "$stopOnError" == "on" ] || [ "$forced" == "DIE" ] - then - echo $ME/$NAME: $errorString - - if [ "$errorFile" != "" ] && [ -f "$errorFile" ] - then - cat $errorFile 2> /dev/null - fi - - exit $exitCode - fi - - echo >> /tmp/orionFuncTestLog - echo '----- ' $NAME ' -----' >> /tmp/orionFuncTestLog - echo $errorString >> /tmp/orionFuncTestLog - - if [ "$errorFile" != "" ] && [ -f "$errorFile" ] - then - cat $errorFile >> /tmp/orionFuncTestLog 2> /dev/null - echo >> /tmp/orionFuncTestLog - fi - - echo >> /tmp/orionFuncTestLog - - testErrorV[$testError]=$testFile - testError=$testError+1 - toBeStopped=true; -} - - - -# ------------------------------------------------------------------------------ -# -# ME - name of script, to be used in error and verbose messages -# -ME=$(basename $0) -vMsg "$ME, in directory $SCRIPT_HOME" - - - -# ------------------------------------https://github.com/telefonicaid/fiware-orion/pull/394#discussion_r13321709------------------------------------------ -# -# Argument parsing -# -typeset -i fromIx -typeset -i toIx -verbose=off -dryrun=off -keep=off -stopOnError=off -testFilter=${TEST_FILTER:-"*.test"} -match="" -dir=$SCRIPT_HOME/cases -dirOrFile="" -dirGiven=no -filterGiven=no -showDuration=on -fromIx=0 -toIx=0 -ixList="" -noCache="" -threadpool=ON -xbroker=off - -vMsg "parsing options" -while [ "$#" != 0 ] -do - if [ "$1" == "-u" ]; then usage 0; - elif [ "$1" == "-v" ]; then verbose=on; - elif [ "$1" == "--dryrun" ]; then dryrun=on; - elif [ "$1" == "--keep" ]; then keep=on; - elif [ "$1" == "--stopOnError" ]; then stopOnError=on; - elif [ "$1" == "--filter" ]; then testFilter="$2"; filterGiven=yes; shift; - elif [ "$1" == "--match" ]; then match="$2"; shift; - elif [ "$1" == "--dir" ]; then dir="$2"; dirGiven=yes; shift; - elif [ "$1" == "--fromIx" ]; then fromIx=$2; shift; - elif [ "$1" == "--toIx" ]; then toIx=$2; shift; - elif [ "$1" == "--ixList" ]; then ixList=$2; shift; - elif [ "$1" == "--skipList" ]; then skipList=$2; shift; - elif [ "$1" == "--no-duration" ]; then showDuration=off; - elif [ "$1" == "--noCache" ]; then noCache=ON; - elif [ "$1" == "--cache" ]; then noCache=OFF; - elif [ "$1" == "--noThreadpool" ]; then threadpool=OFF; - elif [ "$1" == "--xbroker" ]; then xbroker=ON; - else - if [ "$dirOrFile" == "" ] - then - dirOrFile="$1" - else - echo $0: bad parameter/option: "'"${1}"'"; - echo - usage 1 - fi - fi - shift -done - -vMsg "options parsed" - -# ----------------------------------------------------------------------------- -# -# The function brokerStart looks at the env var CB_NO_CACHE to decide -# whether to start the broker with the --noCache option or not -# -if [ "$noCache" != "" ] -then - export CB_NO_CACHE=$noCache -fi - -# ----------------------------------------------------------------------------- -# -# The function brokerStart looks at the env var CB_THREADPOOL to decide -# whether to start the broker with pool of threads or not. -# Do not overwrite if a value is passed from environment -# -if [ "$CB_THREADPOOL" == "" ] -then - export CB_THREADPOOL=$threadpool -fi - -# ----------------------------------------------------------------------------- -# -# Check if fromIx is set through an env var and use if nothing -# else is set through commandline parameter -# -if [ "$FT_FROM_IX" != "" ] && [ $fromIx == 0 ] -then - fromIx=$FT_FROM_IX -fi - -# ----------------------------------------------------------------------------- -# -# Check if toIx is set through an env var and use if nothing -# else is set through commandline parameter -# -if [ "$FT_TO_IX" != "" ] && [ $toIx == 0 ] -then - toIx=$FT_TO_IX -fi - -echo "Run tests $fromIx to $toIx" - -# ------------------------------------------------------------------------------ -# -# Check unmatching --dir and 'parameter that is a directory' AND -# unmatching --filter and 'parameter that is a file' -# -# 1. If it is a directory - just change the 'dir' variable and continue -# 2. Else, it must be a file, or a filter. -# If the -# -singleFile=No -if [ "$dirOrFile" != "" ] -then - vMsg dirOrFile: $dirOrFile - vMsg dirGiven: $dirGiven - vMsg filterGiven: $filterGiven - vMsg dir: $dir - vMsg testFilter: $testFilter - - if [ -d "$dirOrFile" ] - then - if [ "$dirGiven" == "yes" ] - then - echo "$0: both '--dir' option and directory parameter given - not allowed" - exit 1 - fi - dir="$dirOrFile" - else - if [ "$filterGiven" == "yes" ] - then - echo "$0: both '--filter' option and file parameter given - not allowed" - exit 1 - fi - - singleFile=Yes - # - # If just a filename is given, keep the directory as is. - # If a whole path is given, use the directory-part as directory and the file-part as filter - # - dirPart=$(dirname $dirOrFile) - filePath=$(basename $dirOrFile) - xdir=$(basename $dirPart); - vMsg "dirPart: $dirPart" - vMsg "filePath: $filePath" - - if [ "$dirPart" != "." ] - then - dir=$(dirname $dirOrFile) - testFilter=$(basename $dirOrFile) - - # Last dir + test file ? - if [ -d test/functionalTest/cases/$dirPart ] - then - dirOrFile=test/functionalTest/cases/$dirPart - fi - else - testFilter=$(basename $dirOrFile) - fi - fi -fi - -# -# The option of running against an external broker "--xbroker" only works (for now, at least) with a single test case. -# Strange things may happen (due to the state inside the broker) if more that one test case are launched. -# This check avoid this situation. -# -# If in the future we want to be able to run more than one test case against an external broker, we'd need to make sure -# that each test case undoes all internal state inside the external broker. E.g. delete subscriptions, entities, etc. -# -if [ "$singleFile" == "No" ] && [ "$xbroker" == "ON" ] -then - echo "External broker can only be used with individual test cases" - exit 1 -fi - -vMsg directory: $dir -vMsg testFilter: $testFilter -vMsg "Script in $SCRIPT_HOME" - - - -# ----------------------------------------------------------------------------- -# -# Other global variables -# -toBeStopped=false - - - -# ------------------------------------------------------------------------------ -# -# xbroker - if this CLI is set, then the broker is not to be started as part of -# the test suite - another broker is assumed to be running already -# -if [ "$xbroker" == "ON" ] -then - export CB_WITH_EXTERNAL_BROKER=ON -fi - - - -# ----------------------------------------------------------------------------- -# -# Init files already sourced? -# -if [ "$CONTEXTBROKER_TESTENV_SOURCED" != "YES" ] -then - if [ -f "$SCRIPT_HOME/testEnv.sh" ] - then - # First, we try with a testEnv.sh file in the script home - vMsg Sourcing $SCRIPT_HOME/testEnv.sh - source $SCRIPT_HOME/testEnv.sh - elif [ -f "$SCRIPT_HOME/../../scripts/testEnv.sh" ] - then - # Second, we try with a testEnv.sh file in the script/testEnv.sh (realtive to git repo home). - # Note that the script home in this case is test/functionalTest - vMsg Sourcing $SCRIPT_HOME/../../scripts/testEnv.sh - source $SCRIPT_HOME/../../scripts/testEnv.sh - else - echo "------------------------------------------------------------------" - echo "Please source testEnv.sh before running the functional test suite." - echo "------------------------------------------------------------------" - exit 1 - fi -fi - -if [ "$CONTEXTBROKER_HARNESS_FUNCTIONS_SOURCED" != "YES" ] -then - if [ -f $SCRIPT_HOME/harnessFunctions.sh ] - then - vMsg Sourcing $SCRIPT_HOME/harnessFunctions.sh - source $SCRIPT_HOME/harnessFunctions.sh - else - echo "--------------------------------------------------------------------------------------------" - echo "Please source $SCRIPT_HOME/harnessFunctions.sh before running the functional test suite." - echo "--------------------------------------------------------------------------------------------" - exit 1 - fi -fi - - - -# ------------------------------------------------------------------------------ -# -# Preparations - cd to the test directory -# -dMsg Functional Tests Starting ... - -if [ "$dirOrFile" != "" ] && [ -d "$dirOrFile" ] -then - cd $dirOrFile -elif [ ! -d "$dir" ] -then - exitFunction 1 "$dir is not a directory" "HARNESS" "$dir" "" DIE -else - cd $dir -fi - -echo "Orion Functional tests starting" > /tmp/orionFuncTestLog -date >> /tmp/orionFuncTestLog - - - -# ------------------------------------------------------------------------------ -# -# Preparations - number of test cases -# -vMsg find in $(pwd), filter: $testFilter -if [ "$match" == "" ] -then - fileList=$(find . -name "$testFilter" | sort | sed 's/^.\///') -else - fileList=$(find . -name "$testFilter" | grep "$match" | sort | sed 's/^.\///') -fi -vMsg "fileList: $fileList" -typeset -i noOfTests -typeset -i testNo - - - -# ------------------------------------------------------------------------------ -# -# Count total number of tests (for progressing info in messages) -# -for i in $fileList -do - noOfTests=$noOfTests+1 -done - - - -# ------------------------------------------------------------------------------ -# -# fileCleanup - -# -function fileCleanup() -{ - filename=$1 - keepOutputFiles=$2 - path=$3 - dir=$(dirname $path) - - vMsg "---------------------------------------------------------" - vMsg "In fileCleanup for $filename in $dir" - vMsg "---------------------------------------------------------" - - if [ "$keepOutputFiles" != "on" ] - then - olddir=$PWD - cd $dir - - rm $filename.name 2> /dev/null - rm $filename.shellInit 2> /dev/null - rm $filename.shellInit.* 2> /dev/null - rm $filename.shell 2> /dev/null - rm $filename.shell.* 2> /dev/null - rm $filename.teardown 2> /dev/null - rm $filename.teardown.* 2> /dev/null - rm $filename.valgrind.out 2> /dev/null - rm $filename.valgrind.stop.out 2> /dev/null - rm $filename.out 2> /dev/null - rm $filename.regexpect 2> /dev/null - rm $filename.out.sorted 2> /dev/null - rm $filename.regexpect.sorted 2> /dev/null - rm $filename.blockSortDiff.out 2> /dev/null - rm $filename.diff 2> /dev/null - - cd $olddir - fi -} - - - -# ------------------------------------------------------------------------------ -# -# fileCreation - create the files for test execution -# -function fileCreation() -{ - path=$1 - filename=$2 - ret=0 - - dirname=$(dirname $path) - filename=$(basename $path .test) - - if [ "$dirname" != "." ] && [ "$dirname" != "" ] - then - pathWithoutExt=$dirname/$filename - vMsg New path: $path - else - pathWithoutExt=$filename - fi - - vMsg Creating test files for $pathWithoutExt - - - # - # Extract the NAME - # - NAME=$(sed -n '/--NAME--/,/^--/p' $path | grep -v "^--") - if [ "$NAME" == "" ] - then - exitFunction 2 "--NAME-- part is missing" "$path" "($path)" "" DIE - exit 2 # Just in case - fi - - # - # Extract the shell init script - # - if [ $(grep "\-\-SHELL\-INIT\-\-" $path | wc -l) -eq 1 ] - then - TEST_SHELL_INIT=${pathWithoutExt}.shellInit - vMsg "Creating $TEST_SHELL_INIT at $PWD" - sed -n '/--SHELL-INIT--/,/^--/p' $path | grep -v "^--" > $TEST_SHELL_INIT - else - exitFunction 3 "--SHELL-INIT-- part is missing" $path "($path)" "" DIE - fi - - # - # Extract the test shell script - # - if [ $(grep "\-\-SHELL\-\-" $path | wc -l) -eq 1 ] - then - TEST_SHELL=${pathWithoutExt}.shell - vMsg "Creating $TEST_SHELL at $PWD" - sed -n '/--SHELL--/,/^--/p' $path | grep -v "^--" > $TEST_SHELL - else - exitFunction 4 "--SHELL-- part is missing" $path "($path)" "" DIE - fi - - # - # Extract the REGEXPECT part - # - if [ $(grep "\-\-REGEXPECT\-\-" $path | wc -l) -eq 1 ] - then - TEST_REGEXPECT=${pathWithoutExt}.regexpect - vMsg "Creating $TEST_REGEXPECT at $PWD" - sed -n '/--REGEXPECT--/,/^--/p' $path | grep -v "^--" | sed '/^##/d' > $TEST_REGEXPECT - else - exitFunction 5 "--REGEXPECT-- part is missing" $path "($path)" "" DIE - fi - - # - # Extract the teardown script - # - if [ $(grep "\-\-TEARDOWN\-\-" $path | wc -l) -eq 1 ] - then - TEST_TEARDOWN=${pathWithoutExt}.teardown - vMsg "Creating $TEST_TEARDOWN at $PWD" - sed -n '/--TEARDOWN--/,/^--/p' $path | grep -v "^--" > $TEST_TEARDOWN - else - exitFunction 6 "--TEARDOWN-- part is missing" $path "($path)" "" DIE - fi -} - - - -# ------------------------------------------------------------------------------ -# -# partExecute -# -function partExecute() -{ - what=$1 - path=$2 - forcedDie=$3 - __tryNo=$4 - - vMsg Executing $what part for $path - dirname=$(dirname $path) - filename=$(basename $path .test) - - if [ "$dirname" != "." ] && [ "$dirname" != "" ] - then - path=$dirname/$filename.test - fi - - # - # Prepare to execute - # - chmod 755 $dirname/$filename.$what - rm -f $dirname/$filename.$what.stderr - rm -f $dirname/$filename.$what.stdout - $dirname/$filename.$what > $dirname/$filename.$what.stdout 2> $dirname/$filename.$what.stderr - exitCode=$? - linesInStderr=$(wc -l $dirname/$filename.$what.stderr | awk '{ print $1}' 2> /dev/null) - - # - # Check that stdout is empty - # - if [ "$linesInStderr" != "" ] && [ "$linesInStderr" != "0" ] - then - if [ $__tryNo == $MAX_TRIES ] - then - exitFunction 7 "$what: output on stderr" $path "($path): $what produced output on stderr" $dirname/$filename.$what.stderr "$forcedDie" - else - echo -n "(ERROR 7 - $what: output on stderr) " - fi - - partExecuteResult=7 - return - fi - - - # - # Check that exit code is ZERO - # - if [ "$exitCode" != "0" ] - then - if [ $__tryNo == $MAX_TRIES ] - then - exitFunction 8 $path "$what exited with code $exitCode" "($path)" $dirname/$filename.$what.stderr "$forcedDie" - else - echo -n "(ERROR 8 - $what: exited with code $exitCode) " - fi - - partExecuteResult=8 - return - fi - - - # - # Compare produced output with expected output - # - if [ "$what" == "shell" ] - then - mv $dirname/$filename.$what.stdout $dirname/$filename.out # We are very much used to this name ... - sed -i 's/[[:space:]]*$//' $dirname/$filename.out # Remove trailing whitespace in .out (reduces diff noise) - - # - # Special sorted diff or normal REGEX diff ? - # - blockDiff='no' - grep '^#SORT_START$' $dirname/$filename.regexpect > /dev/null 2>&1 - if [ $? == 0 ] - then - $SCRIPT_HOME/blockSortDiff.sh --referenceFile $dirname/$filename.regexpect --brokerOutputFile $dirname/$filename.out > $dirname/$filename.blockSortDiff.out - exitCode=$? - blockDiff='yes' - else - PYTHONIOENCODING=utf8 $DIFF -r $dirname/$filename.regexpect -i $dirname/$filename.out > $dirname/$filename.diff - exitCode=$? - fi - - if [ "$exitCode" != "0" ] - then - if [ $__tryNo == $MAX_TRIES ] - then - exitFunction 9 ".out and .regexpect differ" $path "($path) output not as expected" $dirname/$filename.diff - else - echo -n "(ERROR 9 - .out and .regexpect differ) " - fi - - if [ "$CB_DIFF_TOOL" != "" ] && [ $__tryNo == $MAX_TRIES ] - then - endDate=$(date) - if [ $blockDiff == 'yes' ] - then - $CB_DIFF_TOOL $dirname/$filename.regexpect.sorted $dirname/$filename.out.sorted - else - $CB_DIFF_TOOL $dirname/$filename.regexpect $dirname/$filename.out - fi - fi - partExecuteResult=9 - return - fi - fi - - partExecuteResult=0 -} - - - -# ------------------------------------------------------------------------------ -# -# runTest - the function that runs ONE test case -# -# 1. Remove old output files -# 2.1. Create the various test files from '$path' -# 3.1. Run the SHELL-INIT part -# 3.2. If [ $? != 0 ] || [ STDERR != empty ] ERROR -# 4.1. Run the SHELL part -# 4.2. If [ $? != 0 ] || [ STDERR != empty ] ERROR -# 5.1. Run the TEARDOWN part -# 5.2. If [ $? != 0 ] || [ STDERR != empty ] ERROR -# 6.1. Compare output with regexpect (or expect) -# 6.2. Not EQUAL: ERROR -# 7. If [ "$keep" != "yes" ] Remove all output files -# -# -function runTest() -{ - path=$1 - _tryNo=$2 - - runTestStatus="ok" - - vMsg path=$path - dirname=$(dirname $path) - filename=$(basename $path .test) - dir="" - - if [ "$dirname" != "." ] && [ "$dirname" != "" ] - then - path=$dirname/$filename.test - vMsg New path: $path - fi - - vMsg running test $path - - # 1. Remove old output files - fileCleanup $filename removeAll $path - if [ "$toBeStopped" == "yes" ] - then - echo toBeStopped == yes - runTestStatus="stopped" - return - fi - - # 2. Create the various test files from '$path' - fileCreation $path $filename - if [ "$toBeStopped" == "yes" ] - then - runTestStatus="stopped2" - return - fi - - # 3. Run the SHELL-INIT part - vMsg Executing SHELL-INIT part for $path - chmod 755 $dirname/$filename.shellInit - rm -f $dirname/$filename.shellInit.stderr - rm -f $dirname/$filename.shellInit.stdout - $dirname/$filename.shellInit > $dirname/$filename.shellInit.stdout 2> $dirname/$filename.shellInit.stderr - exitCode=$? - linesInStderr=$(wc -l $dirname/$filename.shellInit.stderr | awk '{ print $1}' 2> /dev/null) - - if [ "$linesInStderr" != "" ] && [ "$linesInStderr" != "0" ] - then - exitFunction 10 "SHELL-INIT produced output on stderr" $path "($path)" $dirname/$filename.shellInit.stderr - runTestStatus="shell-init-error" - return - fi - - if [ "$exitCode" != "0" ] - then - - # - # 3.2 Run the SHELL-INIT part AGAIN - # - # This 're-run' of the SHELL-INIT part is due to errors we've seen that seem to be caused by - # a try to start a broker while the old one (from the previous functest) is still running. - # No way to test this, except with some patience. - # - # We have seen 'ERROR 11' around once every 500-1000 functests (the suite is of almost 400 tests) - # and this fix, if working, will make us not see those 'ERROR 11' again. - # If we keep seeing 'ERROR 11' after this change then we will need to investigate further. - # - sleep 1 - rm -f $dirname/$filename.shellInit.stderr - rm -f $dirname/$filename.shellInit.stdout - $dirname/$filename.shellInit > $dirname/$filename.shellInit.stdout 2> $dirname/$filename.shellInit.stderr - exitCode=$? - linesInStderr=$(wc -l $dirname/$filename.shellInit.stderr | awk '{ print $1}' 2> /dev/null) - - if [ "$linesInStderr" != "" ] && [ "$linesInStderr" != "0" ] - then - exitFunction 20 "SHELL-INIT II produced output on stderr" $path "($path)" $dirname/$filename.shellInit.stderr - runTestStatus="shell-init-output-on-stderr" - return - fi - - if [ "$exitCode" != "0" ] - then - exitFunction 11 "SHELL-INIT exited with code $exitCode" $path "($path)" "" DIE - runTestStatus="shell-init-exited-with-"$exitCode - return - fi - fi - - # 4. Run the SHELL part (which also compares - FIXME P2: comparison should be moved to separate function) - partExecute shell $path "DontDie - only for SHELL-INIT" $_tryNo - shellResult=$partExecuteResult - if [ "$toBeStopped" == "yes" ] - then - runTestStatus="shell-failed" - return - fi - - # 5. Run the TEARDOWN part - partExecute teardown $path "DIE" 0 - teardownResult=$partExecuteResult - vMsg "teardownResult: $teardownResult" - vMsg "shellResult: $shellResult" - - if [ "$shellResult" == "0" ] && [ "$teardownResult" == "0" ] - then - # 6. Remove output files - vMsg "Remove output files: fileCleanup $filename $keep" - fileCleanup $filename $keep $path - else - file=$(basename $path .test) - cp /tmp/contextBroker.log $file.contextBroker.log - runTestStatus="test-failed" - fi -} - - - -# ----------------------------------------------------------------------------- -# -# testDisabled -# -function testDisabled -{ - testcase=$1 - typeset -i dIx - dIx=0 - while [ $dIx -lt ${#DISABLED[@]} ] - do - # Comparison is done based in filename, skipping the path (see https://stackdiary.com/tutorials/bash-get-filename-from-path/) - if [ "${testcase##*/}" == "${DISABLED[$dIx]##*/}" ] - then - echo "Disabled" - - # - # NOTE: In a non-disabled test, running inside the valgrind test suite, the function 'localBrokerStart()' (from harnessFunctions.sh) - # redirects the output of "valgrind contextBroker" to the file /tmp/valgrind.out. - # Later, the valgrind test suite uses the existence of this file (/tmp/valgrind.out) to detect errors in the valgrind execution. - # But, in the case of a disabled func test, we will not start the test case. and thus we will not reach 'localBrokerStart()', so the - # file will not be created and an error will be flagged by the valgrind test suite. - # The simplest solution is to simply create the file here, in the case of a disabled test. - # - echo "Disabled" > /tmp/valgrind.out - return - fi - dIx=$dIx+1 - done - echo NOT Disabled -} - - - -# ----------------------------------------------------------------------------- -# -# testMatchExprFlavour -# -function testMatchExprFlavour -{ - testcase=$1 - - if grep -q JEXL_EXPR_FLAVOUR $testcase - then - if $(contextBroker --version | grep -q jexl-expr) - then - echo NOT Disabled - else - echo "Disabled" - echo "Disabled" > /tmp/valgrind.out - fi - else - echo NOT Disabled - fi -} - - - -# ------------------------------------------------------------------------------ -# -# Main loop -# -vMsg Total number of tests: $noOfTests -testNo=0 -for testFile in $fileList -do - if [ -d "$testFile" ] - then - continue - fi - - testNo=$testNo+1 - - if [ $fromIx != 0 ] && [ $testNo -lt $fromIx ] - then - continue; - fi - - if [ $toIx != 0 ] && [ $testNo -gt $toIx ] - then - continue; - fi - - # - # Disabled test? - # - disabled=$(testDisabled $testFile) - if [ "$disabled" == "Disabled" ] - then - disabledTestV[$disabledTests]=$testNo': '$testFile - disabledTests=$disabledTests+1 - continue - fi - - # - # Should the test be skipped due to it doesn't mach in the contextBroker flavour? - # - notInFlavour=$(testMatchExprFlavour $testFile) - if [ "$notInFlavour" == "Disabled" ] - then - notInFlavourTestV[$notInFlavourTests]=$testNo': '$testFile - notInFlavourTests=$notInFlavourTests+1 - continue - fi - - if [ "$ixList" != "" ] - then - hit=$(echo ' '$ixList' ' | grep ' '$testNo' ') - if [ "$hit" == "" ] - then - # Test case not found in ix-list, so it is not executed - continue - fi - fi - - if [ "$CB_SKIP_FUNC_TESTS" != "" ] - then - hit=$(echo ' '$CB_SKIP_FUNC_TESTS' ' | grep ' '$testFile' ') - if [ "$hit" != "" ] - then - # Test case found in skip-list, so it is skipped - skipV[$skips]=$testNo': '$testFile - skips=$skips+1 - continue - fi - fi - - if [ "$skipList" != "" ] - then - hit=$(echo ' '$skipList' ' | grep ' '$testNo' ') - if [ "$hit" != "" ] - then - # Test case found in skip-list, so it is skipped - skipV[$skips]=$testNo': '$testFile - skips=$skips+1 - continue - fi - fi - - startDate=$(date) - start=$(date --date="$startDate" +%s) - endDate="" - typeset -i tryNo - tryNo=1 - - if [ "$dryrun" == "off" ] - then - while [ $tryNo -le $MAX_TRIES ] - do - if [ "$verbose" == "off" ] - then - tryNoInfo="" - if [ $tryNo != "1" ] - then - tryNoInfo="(intent $tryNo)" - fi - - init=$testFile" ................................................................................................................." - init=${init:0:110} - printf "%04d/%d: %s %s " "$testNo" "$noOfTests" "$init" "$tryNoInfo" - else - printf "Running test %04d/%d: %s\n" "$testNo" "$noOfTests" "$testFile" - fi - - runTest $testFile $tryNo - if [ "$shellResult" == "0" ] - then - if [ $tryNo != 1 ] - then - if [ $tryNo == 2 ] - then - okOnSecondV[$okOnSecond]=$testFile - okOnSecond=$okOnSecond+1 - elif [ $tryNo == 3 ] - then - okOnThirdV[$okOnThird]=$testFile - okOnThird=$okOnThird+1 - else - okOnPlus3V[$okOnPlus3]=$testFile - okOnPlus3=$okOnPlus3+1 - fi - echo "OK" - fi - break - else - tryNo=$tryNo+1 - echo - fi - done - else - if [ "$verbose" == "off" ] - then - init=$testFile" ................................................................................................................." - init=${init:0:110} - printf "%04d/%d: %s " "$testNo" "$noOfTests" "$init" - else - printf "Running test %04d/%d: %s\n" "$testNo" "$noOfTests" "$testFile" - fi - fi - - if [ "$endDate" == "" ] # Could have been set in 'partExecute' - then - endDate=$(date) - fi - - end=$(date --date="$endDate" +%s) - typeset -i secs - secs=$end-$start - if [ "$showDuration" == "on" ] - then - if [ $secs -lt 10 ] - then - xsecs=0$secs - else - xsecs=$secs - fi - echo $xsecs seconds - else - echo "SUCCESS" - fi -done - - -testEndTime=$(date +%s.%2N) -testDiffTime=$(echo $testEndTime - $testStartTime | bc)" seconds" -echo Total test time: $testDiffTime - - -typeset -i ix -exitCode=0 -# ------------------------------------------------------------------------------ -# -# Check for errors - if any, print to stdout -# -if [ "$testError" != "0" ] -then - echo - echo "Orion Functional Test Log File:" - echo "================================================================================" - cat /tmp/orionFuncTestLog 2> /dev/null - echo "================================================================================" - echo - echo "----------- Failing tests ------------------" - - ix=0 - while [ $ix -lt $testError ] - do - echo " o " ${testErrorV[$ix]} - ix=$ix+1 - done - exitCode=1 -fi - - - -# ------------------------------------------------------------------------------ -# -# Check for reintents -# -if [ "$okOnSecond" != "0" ] -then - echo - echo "$okOnSecond test cases OK in the second attempt:" - - ix=0 - while [ $ix -lt $okOnSecond ] - do - echo " o " ${okOnSecondV[$ix]} - ix=$ix+1 - done -fi - -if [ "$okOnThird" != "0" ] -then - echo - echo "$okOnThird test cases OK in the third attempt:" - - ix=0 - while [ $ix -lt $okOnThird ] - do - echo " o " ${okOnThirdV[$ix]} - ix=$ix+1 - done -fi - -if [ "$okOnPlus3" != "0" ] -then - echo - echo "$okOnPlus3 test cases OK after three or more failed attempts:" - - ix=0 - while [ $ix -lt $okOnPlus3 ] - do - echo " o " ${okOnPlus3V[$ix]} - ix=$ix+1 - done -fi - -if [ $skips != 0 ] -then - echo - echo WARNING: $skips test cases skipped: - ix=0 - while [ $ix -lt $skips ] - do - echo " o " ${skipV[$ix]} - ix=$ix+1 - done -fi - -if [ $disabledTests != 0 ] -then - echo - echo WARNING: $disabledTests test cases disabled: - ix=0 - while [ $ix -lt $disabledTests ] - do - echo " o " ${disabledTestV[$ix]} - ix=$ix+1 - done -fi - -if [ $notInFlavourTests != 0 ] -then - echo - echo WARNING: $notInFlavourTests test cases were not executed due to contexBroker not matching flavour: - ix=0 - while [ $ix -lt $notInFlavourTests ] - do - echo " o " ${notInFlavourTestV[$ix]} - ix=$ix+1 - done -fi - -exit $exitCode From 6d5d01beacb5d80987b712039c1131413d5567e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 15:05:36 +0200 Subject: [PATCH 304/390] REMOVE archive --- archive/README.md | 1 + archive/proxyCoap/README.md | 59 -- .../cases/0283_coap_support/coap_basic.test | 158 ----- .../coap_command_line_options.test | 51 -- .../cases/0283_coap_support/coap_version.test | 45 -- archive/proxyCoap/harnessFunctions_coap.sh | 251 -------- archive/proxyCoap/src/app/CMakeLists.txt | 96 --- archive/proxyCoap/src/app/CoapController.cpp | 270 --------- archive/proxyCoap/src/app/CoapController.h | 52 -- archive/proxyCoap/src/app/HttpMessage.cpp | 134 ----- archive/proxyCoap/src/app/HttpMessage.h | 44 -- archive/proxyCoap/src/app/HttpProxy.cpp | 236 -------- archive/proxyCoap/src/app/HttpProxy.h | 40 -- archive/proxyCoap/src/app/proxyCoap.cpp | 202 ------- archive/proxyCoap/src/app/version.h | 33 -- archive/proxyCoap/testEnv_coap.sh | 27 - archive/vagrant/Vagrantfile | 42 -- archive/vagrant/doc/manuals/vagrant.md | 28 - archive/xml/README.md | 102 ---- archive/xml/xmlCheck/envVarSubstitute.sh | 108 ---- archive/xml/xmlCheck/xmlCheck.sh | 560 ------------------ archive/xml/xmlCheck/xmlExtractor.py | 190 ------ 22 files changed, 1 insertion(+), 2728 deletions(-) create mode 100644 archive/README.md delete mode 100644 archive/proxyCoap/README.md delete mode 100644 archive/proxyCoap/cases/0283_coap_support/coap_basic.test delete mode 100644 archive/proxyCoap/cases/0283_coap_support/coap_command_line_options.test delete mode 100644 archive/proxyCoap/cases/0283_coap_support/coap_version.test delete mode 100644 archive/proxyCoap/harnessFunctions_coap.sh delete mode 100644 archive/proxyCoap/src/app/CMakeLists.txt delete mode 100644 archive/proxyCoap/src/app/CoapController.cpp delete mode 100644 archive/proxyCoap/src/app/CoapController.h delete mode 100644 archive/proxyCoap/src/app/HttpMessage.cpp delete mode 100644 archive/proxyCoap/src/app/HttpMessage.h delete mode 100644 archive/proxyCoap/src/app/HttpProxy.cpp delete mode 100644 archive/proxyCoap/src/app/HttpProxy.h delete mode 100644 archive/proxyCoap/src/app/proxyCoap.cpp delete mode 100644 archive/proxyCoap/src/app/version.h delete mode 100644 archive/proxyCoap/testEnv_coap.sh delete mode 100644 archive/vagrant/Vagrantfile delete mode 100644 archive/vagrant/doc/manuals/vagrant.md delete mode 100644 archive/xml/README.md delete mode 100755 archive/xml/xmlCheck/envVarSubstitute.sh delete mode 100755 archive/xml/xmlCheck/xmlCheck.sh delete mode 100755 archive/xml/xmlCheck/xmlExtractor.py diff --git a/archive/README.md b/archive/README.md new file mode 100644 index 0000000000..db347d96a9 --- /dev/null +++ b/archive/README.md @@ -0,0 +1 @@ +After 6 years without touching this directory, we are purging it for Orion 4.0.0 release. If you want to get its content check [the previous version GitHub tree](https://github.com/telefonicaid/fiware-orion/tree/3.12.0/archive). diff --git a/archive/proxyCoap/README.md b/archive/proxyCoap/README.md deleted file mode 100644 index d6e03b638d..0000000000 --- a/archive/proxyCoap/README.md +++ /dev/null @@ -1,59 +0,0 @@ -This directory holds proxyCoap, no longer mantained as part of the "mainstream" code in Orion (the one -under the src/ directory at repository root) but maybe useful in the future. - -## Building - -This directory cannot be built in "standalone mode" (as the archiving work described at issue #1202 is not -completed), so by the moment if you want to compile proxyCoap you have to move the -proxyCoap/src/app/proxyCoap directory to its original location at src/app in the repository root, then add -the following to root CMakeLists.txt file: - -``` -ADD_SUBDIRECTORY(src/app/proxyCoap) -``` - -just after the line: - -``` -ADD_SUBDIRECTORY(src/app/contextBroker) -``` - -In addition, you need to do the following before running `make` to compile: - -* Install cantcoap (with dependencies). Note that we are using a particular snapshot of the code (corresponding - to around July 21st, 2014) given that cantcoap repository doesn't provide any releasing mechanism. - - -``` -sudo yum install clang CUnit-devel - -git clone https://github.com/staropram/cantcoap -cd cantcoap -git checkout 749e22376664dd3adae17492090e58882d3b28a7 -make -sudo cp cantcoap.h /usr/local/include -sudo cp dbg.h /usr/local/include -sudo cp nethelper.h /usr/local/include -sudo cp libcantcoap.a /usr/local/lib -``` - - -## Testing - -The cases/ directory contains a set of test harness files used in the past to test the -proxyCoap functionality. You need to install COAP client (an example application included in the -libcoap sources) before to use them: - -``` -wget http://sourceforge.net/projects/libcoap/files/coap-18/libcoap-4.1.1.tar.gz/download -mv download libcoap-4.1.1.tar.gz -tar xvzf libcoap-4.1.1.tar.gz -cd libcoap-4.1.1 -./configure -make -sudo cp examples/coap-client /usr/local/bin -``` - -## Miscellanea - -This directory also includes parts of harnessFunction.sh and testEnv.sh files related with CoAP. diff --git a/archive/proxyCoap/cases/0283_coap_support/coap_basic.test b/archive/proxyCoap/cases/0283_coap_support/coap_basic.test deleted file mode 100644 index 4d17b3e6ec..0000000000 --- a/archive/proxyCoap/cases/0283_coap_support/coap_basic.test +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -Coap Basic interaction - ---SHELL-INIT-- -dbInit CB -brokerStart CB -valgrindSleep 2 -proxyCoapStart - ---SHELL-- - -valgrindSleep 5 -echo "+++++++ Create entity +++++++++" -payload='{ - "attributes": [ - { - "name": "temperature", - "type": "centigrade", - "value": "30" - } - ] -}' -coapCurl --url /v1/contextEntities/Room1 --payload "$payload" --json -X post - - -echo "+++++++ Query +++++++++" -valgrindSleep 2 -coapCurl --url /v1/contextEntities/Room1 --json - - -echo "+++++++ Update +++++++++" -payload='{ - "attributes": [ - { - "name": "temperature", - "type": "centigrade", - "value": "29" - } - ] -}' -coapCurl --url /v1/contextEntities/Room1 --payload "$payload" --json -X put - -echo "+++++++ Query +++++++++" -valgrindSleep 2 -coapCurl --url /v1/contextEntities/Room1 --json - ---REGEXPECT-- -+++++++ Create entity +++++++++ -v:1 t:0 tkl:0 c:2 id:REGEX(\d+) - -{ - "contextResponses": [ - { - "attributes": [ - { - "name": "temperature", - "type": "centigrade", - "value": "" - } - ], - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ], - "id": "Room1", - "isPattern": "false", - "type": "" -} -+++++++ Query +++++++++ -v:1 t:0 tkl:0 c:1 id:REGEX(\d+) - -{ - "contextElement": { - "attributes": [ - { - "name": "temperature", - "type": "centigrade", - "value": "30" - } - ], - "id": "Room1", - "isPattern": "false", - "type": "" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } -} -+++++++ Update +++++++++ -v:1 t:0 tkl:0 c:3 id:REGEX(\d+) - -{ - "contextResponses": [ - { - "attributes": [ - { - "name": "temperature", - "type": "centigrade", - "value": "" - } - ], - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} -+++++++ Query +++++++++ -v:1 t:0 tkl:0 c:1 id:REGEX(\d+) - -{ - "contextElement": { - "attributes": [ - { - "name": "temperature", - "type": "centigrade", - "value": "29" - } - ], - "id": "Room1", - "isPattern": "false", - "type": "" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } -} ---TEARDOWN-- -proxyCoapStop -brokerStop CB -dbDrop CB diff --git a/archive/proxyCoap/cases/0283_coap_support/coap_command_line_options.test b/archive/proxyCoap/cases/0283_coap_support/coap_command_line_options.test deleted file mode 100644 index 9b850b0cc1..0000000000 --- a/archive/proxyCoap/cases/0283_coap_support/coap_command_line_options.test +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - ---NAME-- -proxyCoap command-line options - ---SHELL-INIT-- - ---SHELL-- -proxyCoap -u - ---REGEXPECT-- -Usage: proxyCoap [option '-U' (extended usage)] - [option '-u' (usage)] - [option '--version' (show version)] - [option '-logDir' ] - [option '-t' ] - [option '--silent' (silent mode)] - [option '-v' (verbose mode)] - [option '-vv' (verbose2 mode)] - [option '-vvv' (verbose3 mode)] - [option '-vvvv' (verbose4 mode)] - [option '-vvvvv' (verbose5 mode)] - [option '-logAppend' (append to log-file)] - - [option '-fg' (don't start as daemon)] - [option '-localIp' ] - [option '-port' ] - [option '-cbHost' ] - [option '-cbPort' ] - [option '-ipv4' (use ip v4 only)] - [option '-ipv6' (use ip v6 only)] - ---TEARDOWN-- diff --git a/archive/proxyCoap/cases/0283_coap_support/coap_version.test b/archive/proxyCoap/cases/0283_coap_support/coap_version.test deleted file mode 100644 index 35bb18c164..0000000000 --- a/archive/proxyCoap/cases/0283_coap_support/coap_version.test +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - ---NAME-- -proxyCoap version ---SHELL-INIT-- - ---SHELL-- -# We have found sometimes problems with using CLI options with two hyphens in the .test files, breaking the testHarness.sh script. -# However, in this case, it seems to work with the double-hyphened "--version" argument. - -contextBroker --version - ---REGEXPECT-- -REGEX(\d+\.\d+\.\d+.*) -Copyright 2013 Telefonica Investigacion y Desarrollo, S.A.U -Orion Context Broker is free software: you can redistribute it and/or -modify it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -Orion Context Broker is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -General Public License for more details. - -Telefonica I+D ---TEARDOWN-- diff --git a/archive/proxyCoap/harnessFunctions_coap.sh b/archive/proxyCoap/harnessFunctions_coap.sh deleted file mode 100644 index 983c57bc91..0000000000 --- a/archive/proxyCoap/harnessFunctions_coap.sh +++ /dev/null @@ -1,251 +0,0 @@ -# Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es -# -# -# In order to "recover" the following functions, they should be moved back -# to harnessFunctions.sh file (and enabled with export -f at the end of that file) - -# ------------------------------------------------------------------------------ -# -# proxyCoapStart -# -function proxyCoapStart() -{ - extraParams=$* - - proxyCoap $extraParams -cbPort $CB_PORT - - # Test to see whether we have a proxy running. If not raise an error - running_proxyCoap=$(ps -fe | grep ' proxyCoap' | grep "cbPort $CB_PORT" | wc -l) - if [ "$running_proxyCoap" == "" ] - then - echo "Unable to start proxyCoap" - exit 1 - fi -} - - - -# ------------------------------------------------------------------------------ -# -# proxyCoapStop -# -function proxyCoapStop -{ - port=$COAP_PORT - - # Test to see if we have a proxy running if so kill it! - running_proxyCoap=$(ps -fe | grep proxyCoap | wc -l) - if [ $running_proxyCoap -ne 1 ]; then - kill $(ps -fe | grep proxyCoap | awk '{print $2}') 2> /dev/null - # Wait some time so the proxy can finish properly - sleep 1 - running_proxyCoap=$(ps -fe | grep proxyCoap | wc -l) - if [ $running_proxyCoap -ne 1 ]; then - # If the proxy refuses to stop politely, kill the process by brute force - kill -9 $(ps -fe | grep proxyCoap | awk '{print $2}') 2> /dev/null - sleep 1 - running_proxyCoap=$(ps -fe | grep proxyCoap | wc -l) - if [ $running_proxyCoap -ne 1 ]; then - echo "Existing proxyCoap is immortal, can not be killed!" - exit 1 - fi - fi - fi -} - - -# ------------------------------------------------------------------------------ -# -# coapCurl -# -# Options: -# -X (default: according to curl. GET if no payload, POST if with payload) -# --host (default: localhost) -# --port (default: $COAP_PORT) -# --url (default: empty string) -# --payload (default: NO PAYLOAD. Possible values: [filename | "${string}"]) -# --in (input payload) (default: xml => application/xml, If 'json': application/json) -# --out (output payload (default: xml => application/xml, If 'json': application/json) -# --json (in/out JSON) (if --in/out is used AFTER --json, it overrides) -# --tenant (tenant in HTTP header) -# --servicePath (Service Path in HTTP header) -# --noPayloadCheck (skip paylosd check filter) -# -# Any parameters are sent as is to 'curl' -# -function coapCurl() -{ - # - # Default values - # - _method="" - _host="0.0.0.0" - _port=$COAP_PORT - _url="" - _payload="" - _inFormat="" - _outFormat="" - _json="" - _tenant="" - _servicePath="" - _xtra='' - _noPayloadCheck='off' - - while [ "$#" != 0 ] - do - if [ "$1" == "-X" ]; then _method="$2"; shift; - elif [ "$1" == "--host" ]; then _host="$2"; shift; - elif [ "$1" == "--port" ]; then _port="$2"; shift; - elif [ "$1" == "--url" ]; then _url="$2"; shift; - elif [ "$1" == "--payload" ]; then _payload="$2"; shift; - elif [ "$1" == "--in" ]; then _inFormat="$2"; shift; - elif [ "$1" == "--out" ]; then _outFormat="$2"; shift; - elif [ "$1" == "--json" ]; then _inFormat=application/json; _outFormat=application/json; - elif [ "$1" == "--tenant" ]; then _tenant="$2"; shift; - elif [ "$1" == "--servicePath" ]; then _servicePath="$2"; shift; - elif [ "$1" == "--noPayloadCheck" ]; then _noPayloadCheck=on; - else _xtra="$_xtra $1"; shift; - fi - - shift - done - - # - # Sanity check of parameters - # - if [ "$_url" == "" ] - then - echo "No URL"; - return 1; - fi - - - # - # Fix for 'Content-Type' and 'Accept' short names 'xml' and 'json' - # - if [ "$_inFormat" == "xml" ]; then _inFormat=application/xml; fi; - if [ "$_outFormat" == "xml" ]; then _outFormat=application/xml; fi; - if [ "$_inFormat" == "json" ]; then _inFormat=application/json; fi; - if [ "$_outFormat" == "json" ]; then _outFormat=application/json; fi; - - - - # - # Cleanup 'compound' variables, so that we don't inherit values from previous calls - # - _METHOD='' - _URL='' - _ACCEPT='' - _CONTENTFORMAT='' - - if [ "$_inFormat" != "" ]; then _CONTENTFORMAT="-t $_inFormat"; fi - if [ "$_outFormat" != "" ]; then _ACCEPT="-A $_outFormat"; fi - if [ "$_method" != "" ]; then _METHOD=' -m '$_method; fi - - _URL="coap://$_host:$_port$_url" - - # usage: coap-client [-A type...] [-t type] [-b [num,]size] [-B seconds] [-e text] - # [-g group] [-m method] [-N] [-o file] [-P addr[:port]] [-p port] - # [-s duration] [-O num,text] [-T string] [-v num] URI - # - # URI can be an absolute or relative coap URI, - # -A type... accepted media types as comma-separated list of - # symbolic or numeric values - # -t type content type for given resource for PUT/POST - # -b [num,]size block size to be used in GET/PUT/POST requests - # (value must be a multiple of 16 not larger than 1024) - # If num is present, the request chain will start at - # block num - # -B seconds break operation after waiting given seconds - # (default is 90) - # -e text include text as payload (use percent-encoding for - # non-ASCII characters) - # -f file file to send with PUT/POST (use '-' for STDIN) - # -g group join the given multicast group - # -m method request method (get|put|post|delete), default is 'get' - # -N send NON-confirmable message - # -o file output received data to this file (use '-' for STDOUT) - # -p port listen on specified port - # -s duration subscribe for given duration [s] - # -v num verbosity level (default: 3) - # -O num,text add option num with contents text to request - # -P addr[:port] use proxy (automatically adds Proxy-Uri option to - # request) - # -T token include specified token - # - # examples: - # coap-client -m get coap://[::1]/ - # coap-client -m get coap://[::1]/.well-known/core - # coap-client -m get -T cafe coap://[::1]/time - # echo 1000 | coap-client -m put -T cafe coap://[::1]/time -f - - - - _BUILTINS='-B 1 -b 1024' - - -# echo '=====================================================================================' - if [ "$_payload" != "" ] - then -# echo "echo "$_payload" | ./coap-client -f - $_BUILTINS $_METHOD $_ACCEPT $_CONTENTFORMAT $_URL" - _response=$(echo "$_payload" | coap-client -f - $_BUILTINS $_METHOD $_ACCEPT $_CONTENTFORMAT $_URL) - else -# echo "./coap-client $_BUILTINS $_METHOD $_ACCEPT $_CONTENTFORMAT $_URL" - _response=$(coap-client $_BUILTINS $_METHOD $_ACCEPT $_CONTENTFORMAT $_URL) - fi -# echo '=====================================================================================' - - - # Get headers - _responseHeaders=$(echo "$_response" | head -n 1) - - # Strip headers and garbage bytes from response - _response=$(echo "$_response" | tail -n +2 | head -c-3) - - - echo $_responseHeaders - echo - - # - # Print and beautify response body (IF ANY) - # - if [ "$_noPayloadCheck" == "on" ] - then - echo $_response - else - if [ "$_response" != "" ] - then - if [ "$_outFormat" == application/xml ] || [ "$_outFormat" == "" ] - then - # FIXME P10: XML removal - #echo $_response | xmllint --format - - echo $_response | python -mjson.tool - elif [ "$_outFormat" == application/json ] - then - vMsg "JSON check for:" $_response - echo $_response | python -mjson.tool - else - # FIXME P10: XML removal - #echo $_response | xmllint --format - - echo $_response | python -mjson.tool - fi - fi - fi -} diff --git a/archive/proxyCoap/src/app/CMakeLists.txt b/archive/proxyCoap/src/app/CMakeLists.txt deleted file mode 100644 index 00d6d28b07..0000000000 --- a/archive/proxyCoap/src/app/CMakeLists.txt +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) - -SET (SOURCES - proxyCoap.cpp - HttpProxy.cpp - HttpMessage.cpp - CoapController.cpp -) - - -SET (HEADERS - HttpProxy.h - HttpMessage.h - CoapController.h -) - -SET (STATIC_LIBS - lm - pa -) - -# Include directories -# ----------------------------------------------------------------- -include_directories("${PROJECT_SOURCE_DIR}/src/app") -include_directories("${PROJECT_SOURCE_DIR}/src/lib") -include_directories("${RAPIDXML_INCLUDE}") - - -# Lib directories -# ------------------------------------------------------------ -link_directories("/usr/local/lib/") -link_directories("/usr/lib64/") -link_directories("/usr/lib/x86_64-linux-gnu") - -# FIXME P4: Issue #476: remove or keep? -link_directories("/opt/lib/") - - - -# Executable declaration -# ------------------------------------------------------------ -ADD_EXECUTABLE(proxyCoap ${SOURCES} ${HEADERS}) - -IF(${DISTRO} STREQUAL "Fedora_20") - MESSAGE("proxyCoap: Fedora DISTRO: '${DISTRO}'") - TARGET_LINK_LIBRARIES(proxyCoap ${STATIC_LIBS} cantcoap curl -lboost_thread) -ELSEIF(${DISTRO} STREQUAL "Ubuntu_13.10") - MESSAGE("proxyCoap: Ubuntu DISTRO: '${DISTRO}'") - TARGET_LINK_LIBRARIES(proxyCoap ${STATIC_LIBS} cantcoap curl boost_thread boost_system pthread) -ELSEIF(${DISTRO} STREQUAL "Ubuntu_14.04.1_LTS") - # FIXME P4: Issue #476: remove or keep? - MESSAGE("proxyCoap: Ubuntu DISTRO: '${DISTRO}'") - TARGET_LINK_LIBRARIES(proxyCoap ${STATIC_LIBS} cantcoap curl boost_thread boost_system pthread) -ELSEIF(${DISTRO} STREQUAL "Ubuntu_14.04.3_LTS") - # FIXME P4: Issue #476: remove or keep? - MESSAGE("proxyCoap: Ubuntu DISTRO: '${DISTRO}'") - TARGET_LINK_LIBRARIES(proxyCoap ${STATIC_LIBS} cantcoap curl boost_thread boost_system pthread) -ELSEIF(${DISTRO} STREQUAL "Ubuntu_14.10") - # ADDED FOR UBUNTU 14.10 - MESSAGE("contextBroker: Ubuntu DISTRO: '${DISTRO}'") - TARGET_LINK_LIBRARIES(proxyCoap ${STATIC_LIBS} cantcoap curl boost_thread boost_system pthread) -ELSEIF(${DISTRO} STREQUAL "Debian_8.0") - MESSAGE("proxyCoap: Debian DISTRO: '${DISTRO}'") - TARGET_LINK_LIBRARIES(proxyCoap ${STATIC_LIBS} cantcoap curl boost_thread boost_system pthread) -ELSEIF(${DISTRO} MATCHES "CentOS_6.*") - MESSAGE("proxyCoap: CentOS DISTRO: '${DISTRO}'") - TARGET_LINK_LIBRARIES(proxyCoap ${STATIC_LIBS} cantcoap curl -lboost_thread-mt) -ELSE() - MESSAGE("proxyCoap: Other DISTRO: '${DISTRO}'") - TARGET_LINK_LIBRARIES(proxyCoap ${STATIC_LIBS} cantcoap curl -lboost_thread -lboost_system -lpthread) -ENDIF() - -# target_link_libraries (proxyCoap cantcoap curl) - -INSTALL(TARGETS proxyCoap - RUNTIME DESTINATION bin) diff --git a/archive/proxyCoap/src/app/CoapController.cpp b/archive/proxyCoap/src/app/CoapController.cpp deleted file mode 100644 index f9dbbb2b93..0000000000 --- a/archive/proxyCoap/src/app/CoapController.cpp +++ /dev/null @@ -1,270 +0,0 @@ -/* -* -* Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: TID Developer -*/ - -#include "CoapController.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "logMsg/logMsg.h" -#include "logMsg/traceLevels.h" - -#include "proxyCoap/HttpProxy.h" -#include "proxyCoap/HttpMessage.h" - -#define PORT_STRING_SIZE 6 - -CoapController::CoapController(const char *_host, unsigned short _httpPort, unsigned short _coapPort) -{ - // Read host and port values - httpPort = _httpPort; - coapPort = _coapPort; - host.assign(_host); - - char* portString = new char[PORT_STRING_SIZE]; - snprintf(portString, PORT_STRING_SIZE, "%d", _coapPort); - coapPortStr.assign(portString); -} - -/* **************************************************************************** -* -* sendDatagram - -*/ -int CoapController::sendDatagram(int sockfd, boost::scoped_ptr& res, sockaddr* recvFrom) -{ - socklen_t addrLen = sizeof(struct sockaddr_in); - if (recvFrom->sa_family == AF_INET6) - { - addrLen = sizeof(struct sockaddr_in6); - } - - ssize_t sent = sendto(sockfd, res->getPDUPointer(), res->getPDULength(), 0, recvFrom, addrLen); - - if (sent < 0) - { - LM_W(("Error sending packet: %ld.", sent)); - return 1; - } - else - { - LM_T(LmtCoap, ("Sent: %ld bytes", sent)); - } - - return 0; -} - -int CoapController::sendError(int sockfd, CoapPDU* req, sockaddr* recvFrom, CoapPDU::Code code) -{ - boost::scoped_ptr res(new CoapPDU()); - - res->setVersion(1); - res->setMessageID(req->getMessageID()); - res->setCode(code); - res->setType(CoapPDU::COAP_ACKNOWLEDGEMENT); - res->setToken(req->getTokenPointer(), req->getTokenLength()); - - return sendDatagram(sockfd, res, recvFrom); -} - -/* **************************************************************************** -* -* callback - -*/ -int CoapController::callback(CoapPDU* request, int sockfd, struct sockaddr_storage* recvFrom) -{ - // Translate request from CoAP to HTTP and send it to MHD through loopback - std::string httpResponse; - httpResponse = sendHttpRequest(host.c_str(), httpPort, request); - - if (httpResponse == "") - { - // Could not get an answer from HTTP module - sendError(sockfd, request, (sockaddr*)recvFrom, CoapPDU::COAP_INTERNAL_SERVER_ERROR); - return 1; - } - - // Parse HTTP response - boost::scoped_ptr hm(new HttpMessage(httpResponse)); - - // If CoAP message is too big, must send error to requester - if (hm->contentLength() > COAP_BUFFER_SIZE) - { - sendError(sockfd, request, (sockaddr*)recvFrom, CoapPDU::COAP_REQUEST_ENTITY_TOO_LARGE); - return 1; - } - - // Translate response from HTTP to CoAP - boost::scoped_ptr coapResponse(hm->toCoap()); - - if (!coapResponse) - { - // Could not translate HTTP into CoAP - sendError(sockfd, request, (sockaddr*)recvFrom, CoapPDU::COAP_INTERNAL_SERVER_ERROR); - return 1; - } - - // Prepare appropriate response in CoAP - coapResponse->setVersion(1); - coapResponse->setMessageID(request->getMessageID()); - coapResponse->setToken(request->getTokenPointer(), request->getTokenLength()); - - // Set type - switch (request->getType()) - { - case CoapPDU::COAP_CONFIRMABLE: - case CoapPDU::COAP_NON_CONFIRMABLE: - coapResponse->setType(CoapPDU::COAP_ACKNOWLEDGEMENT); - break; - - case CoapPDU::COAP_ACKNOWLEDGEMENT: - case CoapPDU::COAP_RESET: - break; - - default: - return 1; - break; - }; - - // Send the packet - sendDatagram(sockfd, coapResponse, (sockaddr*) recvFrom); - return 0; -} - - - -/* **************************************************************************** -* -* serve - -*/ -void CoapController::serve() -{ - // Buffers for UDP and URIs - char buffer[COAP_BUFFER_SIZE]; - char uriBuffer[COAP_URI_BUFFER_SIZE]; - int recvURILen = 0; - int ret = 0; - - // Storage for handling receive address - struct sockaddr_storage recvAddr; - socklen_t recvAddrLen = sizeof(struct sockaddr_storage); - - // Prepare binding address - struct addrinfo *bindAddr = NULL; - struct addrinfo hints; - - // Setting up bind address - memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_socktype = SOCK_DGRAM; - hints.ai_flags |= AI_NUMERICSERV; - hints.ai_family = AF_INET; // ipv4, PF_INET6 for ipv6 or PF_UNSPEC to let OS decide - - int error = getaddrinfo(host.c_str(), coapPortStr.c_str(), &hints, &bindAddr); - if (error) - { - LM_W(("Could not start CoAP server: Error getting address info: %s.", gai_strerror(error))); - return; - } - - // Setting up the UDP socket - int sd = socket(bindAddr->ai_family, bindAddr->ai_socktype, bindAddr->ai_protocol); - - // Binding socket - if (bind(sd, bindAddr->ai_addr, bindAddr->ai_addrlen) != 0) - { - LM_W(("Could not start CoAP server: Error binding socket")); - return; - } - - while (1) - { - // zero out the buffer - memset(buffer, 0, COAP_BUFFER_SIZE); - - // receive packet - ret = recvfrom(sd, &buffer, COAP_BUFFER_SIZE, 0, (sockaddr*) &recvAddr, &recvAddrLen); - if (ret == -1) - { - LM_W(("Error receiving data")); - continue; - } - - boost::scoped_ptr recvPDU(new CoapPDU((uint8_t*) buffer, COAP_BUFFER_SIZE, COAP_BUFFER_SIZE)); - - // validate packet - if (ret > COAP_BUFFER_SIZE) - { - LM_W(("PDU too large to fit in pre-allocated buffer")); - continue; - } - recvPDU->setPDULength(ret); - if (recvPDU->validate() != 1) - { - LM_W(("Malformed CoAP packet")); - continue; - } - LM_T(LmtCoap, ("Valid CoAP PDU received")); - - // Treat URI - if (recvPDU->getURI(uriBuffer, COAP_URI_BUFFER_SIZE, &recvURILen) != 0) - { - LM_W(("Error retrieving URI")); - continue; - } - - if (recvURILen == 0) - { - LM_T(LmtCoap, ("There is no URI associated with this Coap PDU")); - } - else - { - // Invoke a callback thread - boost::thread* workerThread = new boost::thread(boost::bind(&CoapController::callback, this, recvPDU.get(), sd, &recvAddr)); - - // Wait for thread to finnish (like using no threads at all) for now - workerThread->join(); - - continue; - } - - // no URI, handle cases - - // code == 0, no payload, this is a ping request, send RST? - if ((recvPDU->getPDULength() == 0) && (recvPDU->getCode() == 0)) - { - LM_T(LmtCoap, ("CoAP ping request")); - } - - } -} diff --git a/archive/proxyCoap/src/app/CoapController.h b/archive/proxyCoap/src/app/CoapController.h deleted file mode 100644 index 515a56b2cd..0000000000 --- a/archive/proxyCoap/src/app/CoapController.h +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef COAP_CONTROLLER_H -#define COAP_CONTROLLER_H -/* -* -* Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: TID Developer -*/ - -#include -#include -#include - -#include "cantcoap.h" - -class CoapController -{ - - std::string host; - std::string coapPortStr; - unsigned short httpPort; - unsigned short coapPort; - - int sendDatagram(int sockfd, boost::scoped_ptr& res, sockaddr* recvFrom); - int sendError(int sockfd, CoapPDU* req, sockaddr* recvFrom, CoapPDU::Code code); - - int callback(CoapPDU* request, int sockfd, struct sockaddr_storage* recvFrom); - -public: - CoapController(const char* _host, unsigned short _httpPort, unsigned short _coapPort); - void serve(); -}; - -#endif // COAP_CONTROLLER_H diff --git a/archive/proxyCoap/src/app/HttpMessage.cpp b/archive/proxyCoap/src/app/HttpMessage.cpp deleted file mode 100644 index a9b04933dd..0000000000 --- a/archive/proxyCoap/src/app/HttpMessage.cpp +++ /dev/null @@ -1,134 +0,0 @@ -/* -* -* Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: TID Developer -*/ - -#include "proxyCoap/HttpMessage.h" - -#include "logMsg/logMsg.h" -#include "logMsg/traceLevels.h" - -#include "rest/HttpHeaders.h" - -#include // atoi -#include // istringstream - -HttpMessage::HttpMessage(std::string theMessage) -{ - _httpCode = 0; - _contentLength = 0; - _contentType = ""; - _body = ""; - - std::istringstream iss(theMessage); - std::string line; - - bool isHeader = true; - while (std::getline(iss, line)) - { - // Clean last char in headers - if (isHeader) - { - line = line.substr(0, line.size() - 1); - } - - if (line.empty()) - { - isHeader = false; - } - - if (isHeader) - { - // Is it a header or the first line with the HTTP code? - int pos = line.find(":"); - if (pos < 0) - { - // Get HTTP code - // HTTP code is always 3 chars long starting at position 9 - // - // HTTP/1.1 200 OK - // XXX - // - _httpCode = atoi(line.substr(9, 3).c_str()); - } - else - { - // Get other headers - int temp = line.find(HTTP_CONTENT_LENGTH); - if (temp >= 0) - { - _contentLength = atoi(line.substr(pos + 2).c_str()); - } - - temp = line.find(HTTP_CONTENT_TYPE); - if (temp >= 0) - { - _contentType = line.substr(pos + 2); - } - } - } - else - { - // We are parsing the body now (as is) - _body += line; - } - } -} - -CoapPDU* HttpMessage::toCoap() -{ - CoapPDU* pdu = new CoapPDU(); - - // Set code - switch (this->_httpCode) - { - case 200: - pdu->setCode(CoapPDU::COAP_CONTENT); - break; - - case 415: - pdu->setCode(CoapPDU::COAP_UNSUPPORTED_CONTENT_FORMAT); - break; - - default: - pdu->httpStatusToCode(this->_httpCode); - break; - } - - // Set payload - uint8_t* data = (uint8_t*) this->_body.c_str(); - std::size_t length = this->_body.length(); - pdu->setPayload(data, length); - - // Set content-type - if (this->_contentType == "application/json") - { - pdu->setContentFormat(CoapPDU::COAP_CONTENT_FORMAT_APP_JSON); - } - else - { - pdu->setContentFormat(CoapPDU::COAP_CONTENT_FORMAT_APP_XML); - } - - return pdu; -} diff --git a/archive/proxyCoap/src/app/HttpMessage.h b/archive/proxyCoap/src/app/HttpMessage.h deleted file mode 100644 index 24e533ed8d..0000000000 --- a/archive/proxyCoap/src/app/HttpMessage.h +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef HTTP_MESSAGE_H -#define HTTP_MESSAGE_H -/* -* -* Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: TID Developer -*/ -#include -#include "cantcoap.h" - -class HttpMessage -{ - int _httpCode; - int _contentLength; - std::string _contentType; - std::string _body; - - public: - HttpMessage(std::string theMessage); - - CoapPDU* toCoap(); - int contentLength() { return _contentLength; } -}; - -#endif // HTTP_MESSAGE_H diff --git a/archive/proxyCoap/src/app/HttpProxy.cpp b/archive/proxyCoap/src/app/HttpProxy.cpp deleted file mode 100644 index 098fc26ee4..0000000000 --- a/archive/proxyCoap/src/app/HttpProxy.cpp +++ /dev/null @@ -1,236 +0,0 @@ -/* -* -* Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: TID Developer -*/ - -#include "proxyCoap/HttpProxy.h" - -#include "logMsg/logMsg.h" -#include "logMsg/traceLevels.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - - -/* **************************************************************************** -* -* writeMemoryCallback - -*/ -size_t writeMemoryCallback(void* contents, size_t size, size_t nmemb, void* userp) -{ - size_t realsize = size * nmemb; - MemoryStruct* mem = (MemoryStruct *) userp; - - mem->memory = (char*) realloc(mem->memory, mem->size + realsize + 1); - if (mem->memory == NULL) - { - LM_W(("Not enough memory (realloc returned NULL)\n")); - return 0; - } - - memcpy(&(mem->memory[mem->size]), contents, realsize); - mem->size += realsize; - mem->memory[mem->size] = 0; - - return realsize; -} - - -/* **************************************************************************** -* -* sendHttpRequest - -*/ -std::string sendHttpRequest(const char* host, unsigned short port, CoapPDU* request) -{ - int recvURILen = 0; - CURL* curl = curl_easy_init(); - MemoryStruct httpResponse; - CURLcode res; - char uriBuffer[COAP_URI_BUFFER_SIZE]; - - if (curl) - { - // Allocate to hold HTTP response - httpResponse.memory = (char*) malloc(1); // will grow as needed - httpResponse.size = 0; // no data at this point - - // --- Set HTTP verb - std::string httpVerb = ""; - - switch(request->getCode()) - { - case CoapPDU::COAP_POST: - httpVerb = "POST"; - break; - - case CoapPDU::COAP_PUT: - httpVerb = "PUT"; - break; - - case CoapPDU::COAP_DELETE: - httpVerb = "DELETE"; - break; - - case CoapPDU::COAP_EMPTY: - case CoapPDU::COAP_GET: - default: - httpVerb = "GET"; - break; - } - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, httpVerb.c_str()); - LM_T(LmtCoap, ("Got an HTTP %s", httpVerb.c_str())); - - - // --- Prepare headers - struct curl_slist* headers = NULL; - CoapPDU::CoapOption* options = request->getOptions(); - int numOptions = request->getNumOptions(); - - for (int i = 0; i < numOptions ; i++) - { - u_int16_t opt = options[i].optionNumber; - std::string string = ""; - u_int8_t buffer[options[i].optionValueLength + 1]; - - memcpy(buffer, options[i].optionValuePointer, options[i].optionValueLength); - - switch (opt) - { - case CoapPDU::COAP_OPTION_URI_PATH: - buffer[options[i].optionValueLength] = '\0'; - LM_T(LmtCoap, ("Got URI_PATH option: '%s'", buffer)); - break; - - case CoapPDU::COAP_OPTION_CONTENT_FORMAT: - switch (buffer[0]) - { - case CoapPDU::COAP_CONTENT_FORMAT_APP_JSON: - string = "Content-type: application/json"; - break; - - case CoapPDU::COAP_CONTENT_FORMAT_APP_XML: - string = "Content-type: application/xml"; - break; - - default: - string = "Content-type: application/json"; - break; - } - headers = curl_slist_append(headers, string.c_str()); - LM_T(LmtCoap, ("Got CONTENT-FORMAT option: '%s'", string.c_str())); - break; - - case CoapPDU::COAP_OPTION_ACCEPT: - switch (buffer[0]) - { - case CoapPDU::COAP_CONTENT_FORMAT_APP_JSON: - string = "Accept: application/json"; - break; - - case CoapPDU::COAP_CONTENT_FORMAT_APP_XML: - string = "Accept: application/xml"; - break; - - default: - string = "Accept: application/json"; - break; - } - headers = curl_slist_append(headers, string.c_str()); - LM_T(LmtCoap, ("Got ACCEPT option: '%s'", string.c_str())); - break; - - default: - LM_T(LmtCoap, ("Got unknown option")); - break; - } - } - - - // Set Content-length - if (request->getPayloadLength() > 0) - { - std::stringstream contentLengthStringStream; - contentLengthStringStream << request->getPayloadLength(); - std::string finalString = "Content-length: " + contentLengthStringStream.str(); - headers = curl_slist_append(headers, finalString.c_str()); - LM_T(LmtCoap, ("Got: '%s'", finalString.c_str())); - - // --- Set contents - char* payload = (char*) request->getPayloadCopy(); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (u_int8_t*) payload); - } - - // Set Expect - headers = curl_slist_append(headers, "Expect: "); - - // --- Prepare URL - request->getURI(uriBuffer, COAP_URI_BUFFER_SIZE, &recvURILen); - char url[strlen(host) + recvURILen + 1]; - strncpy(url, host, strlen(host)); - if (recvURILen > 0) - strncat(url, uriBuffer, recvURILen); - url[strlen(host) + recvURILen] = '\0'; - LM_T(LmtCoap, ("URL: '%s'", url)); - - // --- Prepare CURL handle with obtained options - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_PORT, port); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // Allow redirection (?) - curl_easy_setopt(curl, CURLOPT_HEADER, 1); // Activate include the header in the body output - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); // Put headers in place - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &writeMemoryCallback); // Send data here - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &httpResponse); // Custom data for response handling - - - // --- Do HTTP Request - res = curl_easy_perform(curl); - if (res != CURLE_OK) - { - LM_W(("curl_easy_perform() failed: %s\n", curl_easy_strerror(res))); - - // --- Cleanup curl environment - curl_slist_free_all(headers); - curl_easy_cleanup(curl); - - return ""; - } - - // --- Cleanup curl environment - curl_slist_free_all(headers); - curl_easy_cleanup(curl); - } - - std::string ret; - ret.assign(httpResponse.memory, httpResponse.size); - free(httpResponse.memory); - return ret; -} - diff --git a/archive/proxyCoap/src/app/HttpProxy.h b/archive/proxyCoap/src/app/HttpProxy.h deleted file mode 100644 index 14b9d40350..0000000000 --- a/archive/proxyCoap/src/app/HttpProxy.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef HTTP_PROXY_H -#define HTTP_PROXY_H -/* -* -* Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: TID Developer -*/ -#include "cantcoap.h" -#include "proxyCoap/HttpMessage.h" - -struct MemoryStruct { - char* memory; - size_t size; -}; - -static const int COAP_URI_BUFFER_SIZE = 255; -static const int COAP_BUFFER_SIZE = 1024; - -extern std::string sendHttpRequest(const char* host, unsigned short port, CoapPDU* request); - -#endif // HTTP_PROXY_H diff --git a/archive/proxyCoap/src/app/proxyCoap.cpp b/archive/proxyCoap/src/app/proxyCoap.cpp deleted file mode 100644 index 3d4b34be8c..0000000000 --- a/archive/proxyCoap/src/app/proxyCoap.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* -* -* Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: TID Developer -*/ - -#include -#include -#include -#include -#include -#include // getppid, for, setuid, etc. -#include // open -#include -#include -#include -#include - -#include "parseArgs/parseArgs.h" -#include "parseArgs/paConfig.h" -#include "parseArgs/paBuiltin.h" -#include "logMsg/logMsg.h" -#include "logMsg/traceLevels.h" - -#include "rest/rest.h" - -#include "common/sem.h" -#include "common/globals.h" -#include "common/Timer.h" -#include "common/compileInfo.h" - - -#include "common/string.h" - -#include "proxyCoap/CoapController.h" -#include "proxyCoap/version.h" - -/* **************************************************************************** -* -* Option variables -*/ -bool fg; -char bindAddress[MAX_LEN_IP]; -int port; -char cbHost[64]; -int cbPort; -bool useOnlyIPv4; -bool useOnlyIPv6; - -/* **************************************************************************** -* -* parse arguments -*/ -PaArgument paArgs[] = -{ - { "-fg", &fg, "FOREGROUND", PaBool, PaOpt, false, false, true, "don't start as daemon" }, - { "-localIp", bindAddress, "LOCALIP", PaString, PaOpt, _i "0.0.0.0", PaNL, PaNL, "IP to receive new connections" }, - { "-port", &port, "PORT", PaInt, PaOpt, 5683, PaNL, PaNL, "port to receive new connections" }, - - { "-cbHost", cbHost, "FWD_HOST", PaString, PaOpt, _i "localhost", PaNL, PaNL, "host for forwarding CoAP requests" }, - { "-cbPort", &cbPort, "FWD_PORT", PaInt, PaOpt, 1026, 0, 65000, "HTTP port for forwarding CoAP requests" }, - - { "-ipv4", &useOnlyIPv4, "USEIPV4", PaBool, PaOpt, false, false, true, "use ip v4 only" }, - { "-ipv6", &useOnlyIPv6, "USEIPV6", PaBool, PaOpt, false, false, true, "use ip v6 only" }, - - - PA_END_OF_ARGS -}; - -/* **************************************************************************** -* -* sigHandler - -*/ -void sigHandler(int sigNo) -{ - LM_T(LmtPresent, ("In sigHandler - caught signal %d", sigNo)); - - switch (sigNo) - { - case SIGINT: - case SIGTERM: - LM_X(1, ("Received signal %d", sigNo)); - break; - } -} - -const char* description = - "\n" - "proxyCoap version details:\n" - " version: " PROXYCOAP_VERSION"\n" - " git hash: " GIT_HASH "\n" - " compile time: " COMPILE_TIME "\n" - " compiled by: " COMPILED_BY "\n" - " compiled in: " COMPILED_IN "\n"; - - - -/* **************************************************************************** -* -* daemonize - -*/ -void daemonize(void) -{ - pid_t pid; - pid_t sid; - - // already daemon - if (getppid() == 1) - return; - - pid = fork(); - if (pid == -1) - LM_X(1, ("fork: %s", strerror(errno))); - - // Exiting father process - if (pid > 0) - exit(0); - - // Change the file mode mask */ - umask(0); - - // Removing the controlling terminal - sid = setsid(); - if (sid == -1) - LM_X(1, ("setsid: %s", strerror(errno))); - - // Change current working directory. - // This prevents the current directory from being locked; hence not being able to remove it. - if (chdir("/") == -1) - LM_X(1, ("chdir: %s", strerror(errno))); -} - - - -/* **************************************************************************** -* -* main - -*/ -int main(int argC, char* argV[]) -{ - - signal(SIGINT, sigHandler); - signal(SIGTERM, sigHandler); - - paConfig("remove builtin", "-d"); - paConfig("remove builtin", "-r"); - paConfig("remove builtin", "-w"); - paConfig("remove builtin", "-F"); - paConfig("remove builtin", "-B"); - paConfig("remove builtin", "-b"); - paConfig("remove builtin", "-?"); - paConfig("remove builtin", "-toDo"); - paConfig("remove builtin", "-lmnc"); - paConfig("remove builtin", "-lmca"); - paConfig("remove builtin", "-lmkl"); - paConfig("remove builtin", "-lmll"); - paConfig("remove builtin", "-assert"); - paConfig("remove builtin", "-version"); - paConfig("remove builtin", "-h"); - paConfig("remove builtin", "-help"); - - paConfig("man synopsis", (void*) "[options]"); - paConfig("man shortdescription", (void*) "Options:"); - paConfig("man description", (void*) description); - paConfig("man author", (void*) "Telefonica I+D"); - paConfig("man exitstatus", (void*) "proxyCoap is a daemon. If it exits, something is wrong ..."); - paConfig("man version", (void*) PROXYCOAP_VERSION); - paConfig("log to screen", (void*) true); - paConfig("log to file", (void*) true); - paConfig("log file line format", (void*) "TYPE:DATE:EXEC-AUX/FILE[LINE] FUNC: TEXT"); - paConfig("screen line format", (void*) "TYPE@TIME FUNC[LINE]: TEXT"); - paConfig("builtin prefix", (void*) "PROXYCOAP_"); - paConfig("usage and exit on any warning", (void*) true); - - paParse(paArgs, argC, (char**) argV, 1, false); - - if (fg == false) - daemonize(); - - CoapController* cc = new CoapController(bindAddress, cbPort, port); - cc->serve(); -} - diff --git a/archive/proxyCoap/src/app/version.h b/archive/proxyCoap/src/app/version.h deleted file mode 100644 index bb2427fa30..0000000000 --- a/archive/proxyCoap/src/app/version.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef PROXYCOAP_VERSION_H -#define PROXYCOAP_VERSION_H - -/* -* -* Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -* -* This file is part of Orion Context Broker. -* -* Orion Context Broker is free software: you can redistribute it and/or -* modify it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* Orion Context Broker is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -* General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -* -* For those usages not covered by this license please contact with -* iot_support at tid dot es -* -* Author: TID Developer -*/ - - - -#define PROXYCOAP_VERSION "0.0.1-beta" - -#endif diff --git a/archive/proxyCoap/testEnv_coap.sh b/archive/proxyCoap/testEnv_coap.sh deleted file mode 100644 index ad37b76e29..0000000000 --- a/archive/proxyCoap/testEnv_coap.sh +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es -# -# -# In order to "recover" the following functions, they should be moved back -# to testEnv.sh file - -# o COAP_PORT - port where proxyCoap listens for connections - -export COAP_PORT=${COAP_PORT:-5683} diff --git a/archive/vagrant/Vagrantfile b/archive/vagrant/Vagrantfile deleted file mode 100644 index 645776fd27..0000000000 --- a/archive/vagrant/Vagrantfile +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es -# -# Author: Leandro Guillen - -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # For a complete reference, please see the online documentation at vagrantup.com. - - # Cent OS 6.5 base image - config.vm.box = "chef/centos-6.5" - - # This script will be run at startup - config.vm.provision :shell, path: "../../scripts/bootstrap/centos65.sh" - - # Network configuration - config.vm.network "forwarded_port", host: 1026, guest: 1026 # Orion port - -end diff --git a/archive/vagrant/doc/manuals/vagrant.md b/archive/vagrant/doc/manuals/vagrant.md deleted file mode 100644 index 3b83c99e82..0000000000 --- a/archive/vagrant/doc/manuals/vagrant.md +++ /dev/null @@ -1,28 +0,0 @@ -## Vagrant support -If you want yo have an Orion Context Broker ready to develop in your machine easily we provide a Vagrant file so that you can get up and running. Just run: - - vagrant up - -You just need to install Vagrant if you want to use this, and at least Virtualbox. Especially useful for those that use virtual machines to develop on a Mac or a Linux distribution that is not CentOS. Keep in mind that the Vagrant file specifies CentOS 6.5 as a base operating system. - -For example, to compile in debug mode and install it in the home directory simply run: - - vagrant ssh -c 'INSTALL_DIR=/home/vagrant make di -C fiware-orion' - -After a few minutes you can run contextBroker. Again, you can either ssh into the machine with - - vagrant ssh - -and run it from the command-line, or directly run the broker with something like - - vagrant ssh -c 'contextBroker -multiservice -t 0-255' - -Orion Context Broker will be accessible at `127.0.0.1:1026`. - -You can also use these commands to automate building and running from your favorite IDE. - -*NOTE:* The virtualbox machine that is created uses additional resources to those from the broker itself. It uses around 512 MiB of RAM and around 1.20 GiB of disk space with Orion already compiled in debug mode. - - - -The bootstrap script basically goes through the installation instructions in the README. \ No newline at end of file diff --git a/archive/xml/README.md b/archive/xml/README.md deleted file mode 100644 index 13c313595b..0000000000 --- a/archive/xml/README.md +++ /dev/null @@ -1,102 +0,0 @@ -Old versions of Orion (0.28.0 and before) supported XML. This directory contains some stuff -related with that functionality that is not longer needed. - -In particular, the xmlCheck checker (which was used to check that XML in unittest testData/ -and test harness fragments were compliant with NGSI XSD). - -In addition, the following XML-based unit tests were removed. If you want to have a look to them, -have a look at code just before merging PR #2104 (this is the commit number: -https://github.com/telefonicaid/fiware-orion/commit/d6969490c8db77b2147fccbd986aab49303b5e8c). - -Deleted files containing only XML-based unit tests: - -* serviceRoutines/badNgsi10Request_test.cpp -* serviceRoutines/badNgsi9Request_test.cpp -* serviceRoutines/badRequest_test.cpp -* serviceRoutines/deleteAttributeValueInstance_test.cpp -* serviceRoutines/deleteIndividualContextEntityAttribute_test.cpp -* serviceRoutines/deleteIndividualContextEntity_test.cpp -* serviceRoutines/getAttributeValueInstance_test.cpp -* serviceRoutines/getContextEntitiesByEntityId_test.cpp -* serviceRoutines/getContextEntityAttributes_test.cpp -* serviceRoutines/getContextEntityTypeAttributeContainer_test.cpp -* serviceRoutines/getContextEntityTypeAttribute_test.cpp -* serviceRoutines/getContextEntityTypes_test.cpp -* serviceRoutines/getEntityByIdAttributeByName_test.cpp -* serviceRoutines/getIndividualContextEntityAttribute_test.cpp -* serviceRoutines/getIndividualContextEntity_test.cpp -* serviceRoutines/getNgsi10ContextEntityTypesAttribute_test.cpp -* serviceRoutines/getNgsi10ContextEntityTypes_test.cpp -* serviceRoutines/leakTreat_test.cpp -* serviceRoutines/logTraceTreat_test.cpp -* serviceRoutines/postContextEntitiesByEntityId_test.cpp -* serviceRoutines/postContextEntityAttributes_test.cpp -* serviceRoutines/postContextEntityTypeAttributeContainer_test.cpp -* serviceRoutines/postContextEntityTypeAttribute_test.cpp -* serviceRoutines/postContextEntityTypes_test.cpp -* serviceRoutines/postDiscoverContextAvailability_test.cpp -* serviceRoutines/postEntityByIdAttributeByName_test.cpp -* serviceRoutines/postIndividualContextEntityAttribute_test.cpp -* serviceRoutines/postIndividualContextEntity_test.cpp -* serviceRoutines/postQueryContext_test.cpp -* serviceRoutines/postRegisterContext_test.cpp -* serviceRoutines/postSubscribeContextAvailability_test.cpp -* serviceRoutines/postSubscribeContext_test.cpp -* serviceRoutines/postUnsubscribeContextAvailability_test.cpp -* serviceRoutines/postUnsubscribeContext_test.cpp -* serviceRoutines/postUpdateContextAvailabilitySubscription_test.cpp -* serviceRoutines/postUpdateContextSubscription_test.cpp -* serviceRoutines/postUpdateContext_test.cpp -* serviceRoutines/putAttributeValueInstance_test.cpp -* serviceRoutines/putIndividualContextEntity_test.cpp - -The following XML-based test were also removed (most of them were in DISABLED -state just before removing it): - -* TEST(compoundValue, DISABLED_updateContextValueVectorFiveItemsPlusBadOne) -* TEST(DiscoverContextAvailabilityRequest, DISABLED_entityIdIdAsAttribute_xml) -* TEST(DiscoverContextAvailabilityRequest, DISABLED_entityIdIsPattern_xml) -* TEST(DiscoverContextAvailabilityRequest, DISABLED_entityIdTypeAsField_xml) -* TEST(DiscoverContextAvailabilityRequest, DISABLED_entityIdType_xml) -* TEST(NotifyContextAvailabilityRequest, DISABLED_badEntityAttribute_xml) -* TEST(NotifyContextRequest, DISABLED_predetectedError) -* TEST(NotifyContextRequest, DISABLED_xml_invalidEntityIdAttribute) -* TEST(QueryContextRequest, DISABLED_emptyEntityList_xml) -* TEST(QueryContextRequest, DISABLED_emptyScopeType_xml) -* TEST(QueryContextRequest, DISABLED_emptyScopeValue_xml) -* TEST(QueryContextRequest, DISABLED_entityIdIdAsAttribute_xml) -* TEST(QueryContextRequest, DISABLED_entityIdIsPattern_xml) -* TEST(QueryContextRequest, DISABLED_entityIdType_xml) -* TEST(QueryContextRequest, DISABLED_fill) -* TEST(QueryContextRequest, DISABLED_noAttributeExpression_xml) -* TEST(QueryContextRequest, DISABLED_noEntityList_xml) -* TEST(QueryContextRequest, DISABLED_noRestriction_xml) -* TEST(QueryContextRequest, DISABLED_noScopeType_xml) -* TEST(QueryContextRequest, DISABLED_noScopeValue_xml) -* TEST(QueryContextRequest, DISABLED_overwriteEntityIdIsPattern_xml) -* TEST(QueryContextRequest, DISABLED_overwriteEntityIdType_xml) -* TEST(QueryContextRequest, DISABLED_unsupportedEntityIdAttribute_xml) -* TEST(QueryContextRequest, emptyEntityIdId_xml) -* TEST(QueryContextResponse, DISABLED_ok_xml) -* TEST(RegisterContextRequest, DISABLED_durationError) -* TEST(RegisterContextRequest, DISABLED_emptyContextMetadataName) -* TEST(RegisterContextRequest, DISABLED_emptyContextMetadataValue) -* TEST(RegisterContextRequest, DISABLED_emptyContextRegistration) -* TEST(RegisterContextRequest, DISABLED_emptyContextRegistrationAttributeIsDomain) -* TEST(RegisterContextRequest, DISABLED_emptyContextRegistrationAttributeName) -* TEST(RegisterContextRequest, DISABLED_emptyEntityIdList) -* TEST(RegisterContextRequest, DISABLED_emptyRegistrationMetadataValue) -* TEST(RegisterContextRequest, DISABLED_entityIdWithEmptyId) -* TEST(RegisterContextRequest, DISABLED_entityIdWithNoId) -* TEST(RegisterContextRequest, DISABLED_invalidAttributeName) -* TEST(RegisterContextRequest, DISABLED_noEntityIdList) -* TEST(RegisterContextRequest, DISABLED_present) -* TEST(restReply, DISABLED_restErrorReplyGet) -* TEST(SubscribeContextAvailabilityRequest, DISABLD_xml_noReference) -* TEST(SubscribeContextAvailabilityRequest, DISABLED_xml_badEntityId) -* TEST(SubscribeContextAvailabilityRequest, DISABLED_xml_entityIdIsPatternAsBothFieldAndAttribute) -* TEST(SubscribeContextAvailabilityRequest, DISABLED_xml_entityIdTypeAsBothFieldAndAttribute) -* TEST(SubscribeContextRequest, DISABLED_invalidEntityIdAttribute_xml) -* TEST(UpdateContextAvailabilitySubscriptionRequest, DISABLED_xml_invalidEntityAttribute) -* TEST(UpdateContextResponse, DISABLED_constructors) -* TEST(UpdateContextSubscriptionResponse, DISABLED_constructors) diff --git a/archive/xml/xmlCheck/envVarSubstitute.sh b/archive/xml/xmlCheck/envVarSubstitute.sh deleted file mode 100755 index 9d03f02c9f..0000000000 --- a/archive/xml/xmlCheck/envVarSubstitute.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -# Copyright 2013 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - - - -# ----------------------------------------------------------------------------- -# -# usage -# -function usage() -{ - exitCode=$1 - - spaces=$(echo $0 | tr '0-9a-zA-Z /_.\-' ' ') - - echo $0 "[-u (usage)]" - echo "${spaces} [-v (verbose)]" - echo "${spaces} -f " - - exit $exitCode -} - - - -# ----------------------------------------------------------------------------- -# -# vMsg -# -function vMsg() -{ - if [ "$verbose" = "on" ] - then - echo $* - fi -} - - - -# ------------------------------------------------------------------------------ -# -# command line options -# -verbose="off" -file="" - -while [ "$#" != 0 ] -do - if [ "$1" == "-u" ]; then usage 0; - elif [ "$1" == "-v" ]; then verbose=on; - elif [ "$1" == "-f" ]; then file=$2; shift; - else - echo $0: bad parameter/option: "'"${1}"'" - usage 1 - fi - shift -done - -if [ "$file" == "" ] -then - echo $0: no file specified - exit 2 -fi - - - -# ------------------------------------------------------------------------------ -# -# Just substitute -# -vMsg $file to get variable substitution -unsubstituted=$(cat $file) -vMsg unsubstituted: -vMsg '----------------------------------' -vMsg $unsubstituted -vMsg '----------------------------------' - -substituted1=$(echo $unsubstituted | sed 's/\$SUB_ID/123456789012345678901234/g') -substituted2=$(echo $substituted1 | sed 's/\$SUBID/123456789012345678901234/g') -substituted3=$(echo $substituted2 | sed 's/\${LISTENER_PORT}/9999/g') -substituted4=$(echo $substituted3 | sed 's/\$subscriptionId/123456789012345678901234/g') -substituted5=$(echo $substituted4 | sed 's/\$REG_ID/123456789012345678901234/g') - -substituted=$substituted5 - -vMsg -vMsg '----------------------------------' -vMsg $substituted -vMsg '----------------------------------' - -echo "$substituted" > ${file} diff --git a/archive/xml/xmlCheck/xmlCheck.sh b/archive/xml/xmlCheck/xmlCheck.sh deleted file mode 100755 index 46252d1083..0000000000 --- a/archive/xml/xmlCheck/xmlCheck.sh +++ /dev/null @@ -1,560 +0,0 @@ -#!/bin/bash -# Copyright 2013 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - - - -# ----------------------------------------------------------------------------- -# -# usage -# -function usage() -{ - exitCode=$1 - - spaces=$(echo $0 | tr '0-9a-zA-Z /_.\-' ' ') - - echo $0 "[-u (usage)]" - echo "${spaces} [-v (verbose)]" - echo "${spaces} [-f (file to check)]" - echo "${spaces} [--filter (test filter)]" - echo "${spaces} [--no-harness (no harness files)]" - echo "${spaces} [--dryrun (don't actually execute anything)]" - echo "${spaces} [--keep (do NOT remove harness files at end of processing)]" - echo "${spaces} [--xsd-download (download XSD files for FIWARE subversion)]" - echo "${spaces} [--xsd-dir (directory for XSD files)]" - - exit $exitCode -} - - - -# ----------------------------------------------------------------------------- -# -# vMsg -# -function vMsg() -{ - if [ "$verbose" = "on" ] - then - echo $* - fi -} - - - -# ------------------------------------------------------------------------------ -# -# xsdGet - -# -function xsdGet() -{ - prefix="$1" - xfile="$2" - - if [ "$prefix" = "ngsi9" ] - then - echo $xsdDir/Ngsi9_Operations.xsd - elif [ "$prefix" == "ngsi10" ] - then - echo $xsdDir/Ngsi10_Operations.xsd - elif [ "$prefix" == "ngsi" ] - then - echo $xsdDir/Ngsi9_10_dataStructures.xsd - else - echo "unknown file prefix: '"${prefix}"' for $xfile" - exit 2 - fi -} - - - -# ------------------------------------------------------------------------------ -# -# processFile - -# -function processFile() -{ - xmlFile=$1 - xsdFile=$2 - - if [ "$xmlFile" == "" ] || [ "$xsdFile" == "" ] - then - echo "bad parameters for file processing '"${xmlFile}"', '"${xsdFile}"'" - return; - fi - - N=$N+1 - - if [ "$dryrun" == "on" ] - then - vMsg dryrun: xmllint "$xmlFile" --schema "$xsdFile" - return - fi - - vMsg XSD: $xsdFile - if [ "$verbose2" == "on" ] - then - xmllint $xmlFile --schema $xsdFile - RESULT=$? - else - xmllint $xmlFile --schema $xsdFile > /dev/null 2>&1 - RESULT=$? - fi - - if [ "$RESULT" == "0" ] - then - echo "$xmlFile: ok" - OK=$OK+1 - else - echo "$xmlFile: FAILS (xmllint error: $RESULT)" - ERR=$ERR+1 - failingList=${failingList}" "$xmlFile - fi -} - - - -# ----------------------------------------------------------------------------- -# -# harnessFiles - -# -function harnessFiles() -{ - harnessList="" - TMP_DIR=$(mktemp -d /tmp/xmlCheck.XXXXX) - vMsg TMP_DIR: $TMP_DIR - for FILE in $(find $SRC_TOP/test/functionalTest/cases -name *.test) - do - PREFIX=$(basename ${FILE%.*}) - FILE_DIR=$(dirname $FILE) - LAST_SUBDIR=$(basename $FILE_DIR) - mkdir -p $TMP_DIR/$LAST_SUBDIR - $SRC_TOP/test/xmlCheck/xmlExtractor.py $FILE $TMP_DIR/$LAST_SUBDIR $PREFIX - done - - for FILE in $(find $TMP_DIR -name ngsi*.valid.xml) - do - grep '\$' $FILE - if [ "$?" != "0" ] - then - continue - fi - - $SRC_TOP/test/xmlCheck/envVarSubstitute.sh -f "$FILE" - done - - harnessList=$(find $TMP_DIR -name "ngsi*.valid.xml") - xmlPartsFound=$(find $TMP_DIR -name "ngsi*.xml" | wc -l) - xmlPartsValid=$(find $TMP_DIR -name "ngsi*.valid.xml" | wc -l) - xmlPartsInvalid=$(find $TMP_DIR -name "ngsi*.invalid.xml" | wc -l) - xmlPartsPostponed=$(find $TMP_DIR -name "ngsi*.postponed.xml" | wc -l) - xmlPartsUnknown=$(find $TMP_DIR -name "unknown*.xml" | wc -l) -} - - - -# ------------------------------------------------------------------------------ -# -# command line options -# -verbose="off" -filter="" -file="" -xsdDir="/tmp/xsd" -harness="on" -dryrun="off" -xsdDownload="off" -keep="off" - -while [ "$#" != 0 ] -do - if [ "$1" == "-u" ]; then usage 0; - elif [ "$1" == "-v" ]; then verbose=on; - elif [ "$1" == "-f" ]; then file=$2; shift; - elif [ "$1" == "--filter" ]; then filter=$2; shift; - elif [ "$1" == "--no-harness" ]; then harness="off"; - elif [ "$1" == "--dryrun" ]; then dryrun="on"; - elif [ "$1" == "--keep" ]; then keep="on"; - elif [ "$1" == "--xsd-download" ]; then xsdDownload="on"; - elif [ "$1" == "--xsd-dir" ]; then xsdDir=$2; shift; - else - echo $0: bad parameter/option: "'"${1}"'" - usage 1 - fi - shift -done - - - -# ----------------------------------------------------------------------------- -# -# SRC_TOP - getting the TOP directory -# -dir=$(dirname $0) -SRC_TOP1=${PWD}/${dir}/../.. # if called with a relative path -SRC_TOP2=${dir}/../.. # if called via $PATH or with an absolute path -if [ -d ${SRC_TOP1} ] -then - SRC_TOP=${SRC_TOP1} -else - SRC_TOP=${SRC_TOP2} -fi - -cd $SRC_TOP -SRC_TOP=$(pwd) -cd - > /dev/null -vMsg Git repo home: $SRC_TOP - - - -# ----------------------------------------------------------------------------- -# -# Checking the XSD directory -# -if [ ! -d "$xsdDir" ] -then - echo "$0: error: '"${xsdDir}"': no such directory" - exit 3 -fi - - - - -# ------------------------------------------------------------------------------ -# -# Download XSD files from FIWARE repo? -# -if [ "$xsdDownload" == "on" ] -then - # Get the .xsd files - echo -n "Enter username: " - read USER - - echo -n "Enter password: " - STTY_ORIG=`stty -g` - stty -echo - read PASS - stty $STTY_ORIG - echo - - \rm Ngsi10_Operations.xsd Ngsi9_Operations.xsd Ngsi9_10_dataStructures.xsd 2> /dev/null - wget -q --no-check-certificate --user=$USER --password=$PASS https://forge.fiware.org/scmrepos/svn/iot/trunk/schemes/Ngsi10_Operations.xsd - wget -q --no-check-certificate --user=$USER --password=$PASS https://forge.fiware.org/scmrepos/svn/iot/trunk/schemes/Ngsi9_Operations.xsd - wget -q --no-check-certificate --user=$USER --password=$PASS https://forge.fiware.org/scmrepos/svn/iot/trunk/schemes/Ngsi9_10_dataStructures.xsd - - if [ ! -f Ngsi10_Operations.xsd ] || [ ! -f Ngsi9_Operations.xsd ] || [ ! -f Ngsi9_10_dataStructures.xsd ] - then - echo $0: error: wget failed to download latest XSD files - exit 4 - fi - - mv Ngsi10_Operations.xsd Ngsi9_Operations.xsd Ngsi9_10_dataStructures.xsd $xsdDir - - vMsg "got XSD files" -fi - - - -# ------------------------------------------------------------------------------ -# -# XSD files there? -# -if [ ! -f $xsdDir/Ngsi10_Operations.xsd ] || [ ! -f $xsdDir/Ngsi9_Operations.xsd ] || [ ! -f $xsdDir/Ngsi9_10_dataStructures.xsd ] -then - echo "$0: error: XSD files missing in $xsdDir" - exit 5 -fi - - - -# ------------------------------------------------------------------------------ -# -# variables -# -typeset -i N -typeset -i OK -typeset -i ERR -N=0 -OK=0 -ERR=0 - - - -# ----------------------------------------------------------------------------- -# -# Setting up the file list -# -if [ "$file" != "" ] -then - fileList=$file - verbose2=on -else - if [ "$harness" == "on" ] - then - harnessFiles - fi - - ngsi9List=$(find $SRC_TOP/test -name "ngsi9.*.valid.xml") - ngsi10List=$(find $SRC_TOP/test -name "ngsi10.*.valid.xml") - fileList=${ngsi9List}" "${ngsi10List} - partList=$harnessList -fi - - - -# ------------------------------------------------------------------------------ -# -# Counters -# -typeset -i xmlFilesFound # Total number of XML files under test/ -typeset -i xmlFilesValid # Number of XML files that are valid -typeset -i xmlFilesInvalid # Number of XML files that are invalid -typeset -i xmlFilesPostponed # Number of XML files that are postponed -typeset -i xmlFilesBadName # Number of XML files whose names don't follow the naming convention - should be zero -typeset -i xmlFilesProcessed # Total number of XML files that were tested -typeset -i xmlFilesOK # Number of XML files that passed the test -typeset -i xmlFilesErrors # Number of XML files that did not pass the test - -typeset -i harnessFilesFound # Number of files in the harness directory - to part in many XML files -typeset -i xmlPartsFound # Number of XML parts created from harness directory -typeset -i xmlPartsValid # Number of XML parts that are valid -typeset -i xmlPartsInvalid # Number of XML parts that are invalid -typeset -i xmlPartsPostponed # Number of XML parts that are postponed -typeset -i xmlPartsUnknown # Number of XML parts that are unknown -typeset -i xmlPartsProcessed # Number of XML parts that were tested -typeset -i xmlPartsOK # Number of XML parts that passed the test -typeset -i xmlPartsErrors # Number of XML parts that did not pass the test -typeset -i xmlDocsFound # xmlFilesFound + xmlPartsFound -typeset -i xmlDocsProcessed # xmlFilesProcessed + xmlPartsProcessed -typeset -i xmlDocsPostponed # xmlFilesPostponed + xmlPartsPostponed -typeset -i xmlDocsOk # xmlFilesOK + xmlPartsOK -typeset -i xmlDocsErrors # xmlFilesErrors + xmlPartsErrors - -xmlFilesFound=$(find $SRC_TOP/test -name "*.xml" | wc -l) -xmlFilesValid=$(find $SRC_TOP/test -name "ngsi*.valid.xml" | wc -l) -xmlFilesInvalid=$(find $SRC_TOP/test -name "ngsi*.invalid.xml" | wc -l) -xmlFilesPostponed=$(find $SRC_TOP/test -name "ngsi*.postponed.xml" | wc -l) -xmlFilesMiddle=$(find $SRC_TOP/test -name "ngsi*.middle.xml" | wc -l) -xmlFilesOrion=$(find $SRC_TOP/test -name "orion.*.xml" | wc -l) -xmlFilesBadName=$(find $SRC_TOP/test -name "*.xml" | egrep -v 'ngsi.*\.valid\.xml' | egrep -v 'ngsi.*\.invalid\.xml' | egrep -v 'ngsi.*\.postponed\.xml' | egrep -v 'ngsi.*\.middle\.xml' | egrep -v 'orion\..*\.xml' | wc -l) -xmlFilesProcessed=0 -xmlFilesOK=0 -xmlFilesErrors=0 -harnessFilesFound=$(find $SRC_TOP/test/functionalTest/cases -name "*.test" | wc -l) -xmlPartsFound=$xmlPartsFound # already taken care of by function 'harnessFiles' -xmlPartsValid=$xmlPartsValid # already taken care of by function 'harnessFiles' -xmlPartsInvalid=$xmlPartsInvalid # already taken care of by function 'harnessFiles' -xmlPartsPostponed=$xmlPartsPostponed # already taken care of by function 'harnessFiles' -xmlPartsUnknown=$xmlPartsUnknown # already taken care of by function 'harnessFiles' -xmlPartsProcessed=0 -xmlPartsOK=0 -xmlPartsErrors=0 -xmlDocsFound=$(expr $xmlFilesFound + $xmlPartsFound) -xmlDocsProcessed=0 -xmlDocsOk=0 -xmlDocsErrors=0 - - - -# ------------------------------------------------------------------------------ -# -# Applying filter to the file lists -# -if [ "$filter" != "" ] -then - vMsg appying filter "'"${filter}"'" - newFileList="" - for xfile in $fileList - do - echo $xfile | grep "$filter" > /dev/null 2>&1 - if [ "$?" == 0 ] - then - newFileList=${newFileList}" "${xfile} - xmlFilesProcessed=$xmlFilesProcessed+1 - fi - done - fileList=$newFileList - - newPartList="" - for part in $partList - do - echo $part | grep "$filter" > /dev/null 2>&1 - if [ "$?" == 0 ] - then - newPartList=${newPartList}" "${part} - xmlPartsProcessed=$xmlPartsProcessed+1 - fi - done - partList=$newPartList -fi - - - -# ------------------------------------------------------------------------------ -# -# process the list of files to check -# -OK=0 -ERR=0 -N=0 -for xfile in $fileList -do - fileName=$(basename $xfile) - prefix=$(echo $fileName | cut -d . -f 1) - xsd=$(xsdGet "$prefix" "$xfile") - processFile "$xfile" "$xsd" -done - -xmlFilesProcessed=$N -xmlFilesOK=$OK; -xmlFilesErrors=$ERR - - - -# ------------------------------------------------------------------------------ -# -# process the list of parts to check -# -OK=0 -ERR=0 -N=0 -for part in $partList -do - partName=$(basename $part) - prefix=$(echo $partName | cut -d . -f 1) - xsd=$(xsdGet "$prefix" "$part") - processFile "$part" "$xsd" -done - -xmlPartsProcessed=$N -xmlPartsOK=$OK; -xmlPartsErrors=$ERR - - -xmlDocsProcessed=$(expr $xmlFilesProcessed + $xmlPartsProcessed) -xmlDocsOk=$(expr $xmlFilesOK + $xmlPartsOK) -xmlDocsErrors=$(expr $xmlFilesErrors + $xmlPartsErrors) - - - - - -# ------------------------------------------------------------------------------ -# -# Statistics not shown if the option '-f' is set -# -if [ "$file" != "" ] -then - if [ "$ERR" != "0" ] - then - exit 6 - else - exit 0 - fi -fi - - - -# ------------------------------------------------------------------------------ -# -# Statistics -# -echo "=====================================================================" -if [ "$xmlDocsErrors" != "0" ] -then - echo - echo "XML docs that did not pass the XML Validity test:" - - for item in $failingList - do - echo " o " $item - done - - echo - echo "=====================================================================" -fi - - -echo "Tested ${xmlDocsProcessed} (${xmlFilesProcessed} files + ${xmlPartsProcessed} parts) out of ${xmlDocsFound} (${xmlFilesFound} files + ${xmlPartsFound} parts) XML documents:" -echo " ${xmlDocsOk} documents passed the XML validity test" - -exitCode=7 - -if [ "$xmlDocsErrors" != 0 ] -then - echo " ${xmlDocsErrors} documents did not pass" -else - echo " -----------------------" - echo " ALL documents passed!!!" - echo " -----------------------" - exitCode=0 -fi - -echo -echo "${xmlFilesInvalid} documents were not tested as they on purpose don't follow the XSD" -xmlDocsPostponed=$xmlFilesPostponed+$xmlPartsPostponed -echo "${xmlDocsPostponed} documents were not tested as they still have no XSD" -echo "${xmlFilesMiddle} documents were not tested as they don't start the way the XSD states (middle)" - - -if [ "$xmlFilesBadName" != 0 ] -then - echo - if [ "$xmlFilesBadName" != "0" ] - then - echo "WARNING: $xmlFilesBadName XML files do not conform to the naming convention" - for xfile in $(find $SRC_TOP/test -name "*.xml" | grep -v "ngsi*.valid.xml" | grep -v "ngsi*.invalid.xml" | grep -v "ngsi*.postponed.xml" | grep -v "ngsi*.middle.xml" | grep -v "orion.*.xml") - do - echo " o $xfile" - done - fi - exitCode=8 -fi - - -if [ "$xmlPartsUnknown" != 0 ] -then - echo - echo "WARNING: parts marked as unknown" - for xfile in $(find $TMP_DIR -name "unknown*.xml") - do - echo " o $xfile" - done - exitCode=9 -fi - - - -# ------------------------------------------------------------------------------ -# -# Keep? -# -if [ "$keep" == "off" ] -then - rm -rf $TMP_DIR -fi - -if [ "$exitCode" != "0" ] -then - echo exiting with error code $exitCode -fi - -exit $exitCode diff --git a/archive/xml/xmlCheck/xmlExtractor.py b/archive/xml/xmlCheck/xmlExtractor.py deleted file mode 100755 index f924d4f612..0000000000 --- a/archive/xml/xmlCheck/xmlExtractor.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/python -# -*- coding: latin-1 -*- -# Copyright 2013 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# This script extract XML fragments of a given file (passed as first arguments), -# each fragment put in a separate file in a given directory (passed as second argument). -# This is mainly aimed at processing test harness files (.test files) to pass the -# xmlCheck.sh on then afterwards. Note that the heuristic used by the script is not -# bullet-proof: it assumes that the starting and ending tag of the root element is -# in its own line, without other child starting or ending tags (but it should suffice -# for .test files) - -__author__ = 'fermin' - -from sys import argv -import re - -def write_fragment(lines, filename): - f = open(filename, 'w') - f.writelines(lines) - f.close() - -tokens_map = { - # NGSI9 standard operations - 'registerContextRequest': ['ngsi9', 'valid'], - 'registerContextResponse': ['ngsi9', 'valid'], - 'discoverContextAvailabilityRequest': ['ngsi9', 'valid'], - 'discoverContextAvailabilityResponse': ['ngsi9', 'valid'], - 'subscribeContextAvailabilityRequest': ['ngsi9', 'valid'], - 'subscribeContextAvailabilityResponse': ['ngsi9', 'valid'], - 'updateContextAvailabilitySubscriptionRequest': ['ngsi9', 'valid'], - 'updateContextAvailabilitySubscriptionResponse': ['ngsi9', 'valid'], - 'unsubscribeContextAvailabilityRequest': ['ngsi9', 'valid'], - 'unsubscribeContextAvailabilityResponse': ['ngsi9', 'valid'], - 'notifyContextAvailabilityRequest': ['ngsi9', 'valid'], - 'notifyContextAvailabilityResponse': ['ngsi9', 'valid'], - # NGSI10 standard operatoins - 'queryContextRequest': ['ngsi10', 'valid'], - 'queryContextResponse': ['ngsi10', 'valid'], - 'updateContextRequest': ['ngsi10', 'valid'], - 'updateContextResponse': ['ngsi10', 'valid'], - 'subscribeContextRequest': ['ngsi10', 'valid'], - 'subscribeContextResponse': ['ngsi10', 'valid'], - 'updateContextSubscriptionRequest': ['ngsi10', 'valid'], - 'updateContextSubscriptionResponse': ['ngsi10', 'valid'], - 'unsubscribeContextRequest': ['ngsi10', 'valid'], - 'unsubscribeContextResponse': ['ngsi10', 'valid'], - 'notifyContextRequest': ['ngsi10', 'valid'], - 'notifyContextResponse': ['ngsi10', 'valid'], - # NGSI convenience operations exclusive types - 'registerProviderRequest': ['ngsi9', 'postponed'], - 'updateContextElementRequest': ['ngsi10', 'postponed'], - 'updateContextElementResponse': ['ngsi10', 'postponed'], - 'appendContextElementRequest': ['ngsi10', 'postponed'], - 'appendContextElementResponse': ['ngsi10', 'postponed'], - 'updateContextAttributeRequest': ['ngsi10', 'postponed'], - 'contextElementResponse': ['ngsi10', 'postponed'], - 'contextAttributeResponse': ['ngsi10', 'postponed'], - 'statusCode': ['ngsi', 'valid'], - # New operations - 'entityTypeAttributesResponse': ['ngsi10', 'postponed'], - 'entityTypesResponse': ['ngsi10', 'postponed'], - # Orion own types - 'orion': ['orion', 'invalid'], - 'orionError': ['orion', 'invalid'] -} - -if len (argv) != 4: - print 'Wrong number of arguments' - print argv[0] + ' ' - exit(1) - -file = str(argv[1]) -dir = str(argv[2]) -base_name = str(argv[3]) - -buffer = [] -xml_headers_counter = 0 -xml_headers_correction = 0 -xml_fragments_counter = 0 -search_mode = True -next_xml_invalid = False -buffer = [] - -with open (file, 'r') as f: - for l in f: - line = l - - if re.search('#SORT_START', line) or re.search('#SORT_END', line): - # Just skip these marks (they must not appear in the XML string) - continue - - if search_mode: - # Search mode is on: looking for a root element or for an 'invalid' - if re.search('<\?xml', line): - xml_headers_counter += 1 - elif re.search('User-Agent:', line): - # Heuristic: each time a User-Agent is detected, then a nofityContextRequest or notifyContextAvailabiltiyRequest - # if found in the .test file. These are fragments of code that comes from accumulator-script.py dumps, outside - # the usual "xmllint decoration" that adds the XML preamble in that case. Thus, we have to count then and apply as - # a correction factor in the warning generation - xml_headers_correction += 1 - elif re.search('#INVALID_XML', line): - next_xml_invalid = True - else: - m = re.match('\s*<(.*)>', line) - if m != None: - xml_fragments_counter += 1 - root_element = m.group(1) - - # Add XML header and root element as first elements of the buffer - buffer.append('\n') - buffer.append(line) - - search_mode = False - - else: - # Search mode is off: accumulate each line until the ending tag is found - # Note we remove any heading or trailing ' or ", that may occur in the - # case of using variables to store XMLs, e.g.: - # - # payload = ' - # - # '" - # ... - # "' - # ... - # ' - # - line = line.rstrip("'\"\n") - line = line.lstrip(" '\"") - - # We have found that strings like "http://localhost:'$CP1_PORT'/v1" that breaks validation, - # as they are not anyURI. In order to fix, we replace the "'$...'" part with a number - line = re.sub("'\$.*PORT.*'", "9999", line) - - # We have found that we cannot use things like 'REGEX((PT5S|PT7S))', given that - # duration uses type xs:duration, which has a predefined sytanx incompatible with REGEX(). Thus, we - # change these cases on the fly - if re.match('\s*', line): - line = re.sub("REGEX\(.*\)", "PT1M", line) - - # Similar case with providingApplication - if re.match('\s*', line): - line = re.sub("REGEX\(.*\)", "9997", line) - - buffer.append(line + "\n") - - if re.search('<\/'+ root_element + '>', line): - # We use some tokens in the filename to help xmlCheck.sh script - if tokens_map.has_key(root_element): - family = tokens_map[root_element][0] - vality = tokens_map[root_element][1] - else: - family = 'unknown' - vality = 'invalid' - - # No matter the result of the map, if the invalid mark was used in the .test, then - # file is alawys marked as 'invalid' - if next_xml_invalid: - vality = 'invalid' - - filename = dir + '/' + family + '.' + base_name + '.part_' + str(xml_fragments_counter) + '.' + vality + '.xml' - - write_fragment(buffer, filename) - buffer = [] - search_mode = True - next_xml_invalid = False - -if xml_headers_counter != xml_fragments_counter - xml_headers_correction: - print 'Warning in ' + base_name + ': XML headers (' + str(xml_headers_counter) + ', correction: ' + str(xml_headers_correction) + ') and ' \ - 'generated fragments (' + str(xml_fragments_counter) + ') differ' From e78c240be99cff6b51de6272e60f70cc5e162e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 15:51:10 +0200 Subject: [PATCH 305/390] FIX to reference presentation and version numbers --- README.md | 2 +- doc/manuals.jp/devel/README.md | 2 +- doc/manuals/devel/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9cfac12e21..2606236355 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ recommended to have a look to the brief ### Introductory presentations - Orion Context Broker - [(en)](https://www.slideshare.net/slideshows/orion-context-broker-introduction-20240227/266516824) + [(en)](https://www.slideshare.net/slideshow/orion-context-broker-introduction-20240604/269503234) [(jp)](https://www.slideshare.net/slideshows/orion-context-broker-introduction-20240227/266517846) - NGSIv2 Overview for Developers That Already Know NGSIv1 [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220523) diff --git a/doc/manuals.jp/devel/README.md b/doc/manuals.jp/devel/README.md index 603da17228..bfbf13e1d1 100644 --- a/doc/manuals.jp/devel/README.md +++ b/doc/manuals.jp/devel/README.md @@ -1,6 +1,6 @@ # 開発マニュアル -*注 : このドキュメントでは、リリース 3.12.x の Orion Context Broker について説明しています。* +*注 : このドキュメントでは、リリース 4.0.x の Orion Context Broker について説明しています。* ## 対象読者 diff --git a/doc/manuals/devel/README.md b/doc/manuals/devel/README.md index 87fc4f4e02..dd6d690c79 100644 --- a/doc/manuals/devel/README.md +++ b/doc/manuals/devel/README.md @@ -1,6 +1,6 @@ # Development Manual -*Note: This document describes Orion Context Broker as of release 3.12.x.* +*Note: This document describes Orion Context Broker as of release 4.0.x.* ## Intended audience The intended audience of this manual is developers that need to understand the internals of the Orion Context Broker From 8a43ab04303ef18e124bf0b4151abb3a67e74b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 15:58:10 +0200 Subject: [PATCH 306/390] FIX orion version date --- doc/manuals.jp/deprecated.md | 2 +- doc/manuals/deprecated.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manuals.jp/deprecated.md b/doc/manuals.jp/deprecated.md index 535ebbf998..d535d129c7 100644 --- a/doc/manuals.jp/deprecated.md +++ b/doc/manuals.jp/deprecated.md @@ -8,7 +8,7 @@ 推奨されなくなった機能のリストと、廃止された機能のバージョンは次のとおりです : -* Orion 3.12.0 での CLI パラメータ (および関連する環境変数): `-dbhost`、`-rplSet`、`-dbTimeout`、`-dbuser`、`-dbAuthMech`、`-dbAuthDb`、`-dbSSL`、および `-dbDisableRetryWrites`。MongoDB URI を構築するために必要な情報が必要な場合は、[このセクション](#mapping-to-mongouri-from-old-cli-parameters) をチェックして、代わりに `dbURI` を使用してください (Orion 3.13.0 で削除されました) +* Orion 3.12.0 での CLI パラメータ (および関連する環境変数): `-dbhost`、`-rplSet`、`-dbTimeout`、`-dbuser`、`-dbAuthMech`、`-dbAuthDb`、`-dbSSL`、および `-dbDisableRetryWrites`。MongoDB URI を構築するために必要な情報が必要な場合は、[このセクション](#mapping-to-mongouri-from-old-cli-parameters) をチェックして、代わりに `dbURI` を使用してください (Orion 4.0.0 で削除されました) * Orion 3.10.0 での `geo:point`, `geo:line`, `geo:box` および `geo:polygon` 属性タイプ。代わりに `geo:json` を使用してください * Orion 3.8.0 での `GET /v2` 操作。この操作はかなり役に立たず、実際には使用されません。 * Orion 3.1.0 のサブスクリプションでの初期通知 (`skipInitialNotification` オプションと共に)。(Orion 3.2.0 で削除)。初期通知の diff --git a/doc/manuals/deprecated.md b/doc/manuals/deprecated.md index 5ed729988b..cef27a61bf 100644 --- a/doc/manuals/deprecated.md +++ b/doc/manuals/deprecated.md @@ -17,7 +17,7 @@ A list of deprecated features and the version in which they were deprecated foll * CLI parameters (and associated env vars): `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` in Orion 3.12.0. Use `dbURI` instead, - checking [this section](#mapping-to-mongouri-from-old-cli-parameters) if you need to know hot to build the MongoDB URI (removed in Orion 3.13.0). + checking [this section](#mapping-to-mongouri-from-old-cli-parameters) if you need to know hot to build the MongoDB URI (removed in Orion 4.0.0). * `geo:point`, `geo:line`, `geo:box` and `geo:polygon` attribute types in Orion 3.10.0. Use `geo:json` instead. * `GET /v2` operation in Orion 3.8.0. This operation is pretty useless and not actually used. * Initial notification in subscriptions (along with `skipInitialNotification` option) in Orion 3.1.0. From 7d2e97e9520bc633a2fdc4326e05f1d42d751c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 16:05:47 +0200 Subject: [PATCH 307/390] FIX testHarnes.sh wrongly deleted in previous commit --- test/functionalTest/testHarness.sh | 1243 ++++++++++++++++++++++++++++ 1 file changed, 1243 insertions(+) create mode 100755 test/functionalTest/testHarness.sh diff --git a/test/functionalTest/testHarness.sh b/test/functionalTest/testHarness.sh new file mode 100755 index 0000000000..9b30488480 --- /dev/null +++ b/test/functionalTest/testHarness.sh @@ -0,0 +1,1243 @@ +#!/bin/bash +# -*- coding: latin-1 -*- +# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es +# +# Author: Ken Zangelin + + +date +testStartTime=$(date +%s.%2N) +MAX_TRIES=${CB_MAX_TRIES:-3} +echo $testStartTime > /tmp/brokerStartCounter + + +# ----------------------------------------------------------------------------- +# +# DISABLED - funct tests that are disabled, for some reason +# +DISABLED=('test/functionalTest/cases/0000_bad_requests/exit.test' \ + 'test/functionalTest/cases/0917_queryContext_behaves_differently/query_with_and_without_forwarding.test' \ + 'test/functionalTest/cases/0000_ipv6_support/ipv4_only.test' \ + 'test/functionalTest/cases/0000_ipv6_support/ipv6_only.test' \ + 'test/functionalTest/cases/1310_suspect_200OK/suspect_200OK.test'); + + + +# ------------------------------------------------------------------------------ +# +# Find out in which directory this script resides +# +dirname=$(dirname $0) + +if [ "$dirname" == "" ] +then + dirname="." +fi + +cd $dirname +export SCRIPT_HOME=$(pwd) +cd - > /dev/null 2>&1 + + + +# ------------------------------------------------------------------------------ +# +# Debug mode? +# +if [ "$ORION_FT_DEBUG" == "1" ] +then + _debug='on' +fi + + + +# ----------------------------------------------------------------------------- +# +# Log file for debugging +# +rm -f /tmp/orionFuncTestDebug.log +echo $(date) > /tmp/orionFuncTestDebug.log + + + +# ----------------------------------------------------------------------------- +# +# Env vars +# +export LC_ALL=C +export NAME="testFile" +declare -A testErrorV +typeset -i testError +declare -A okOnSecondV +typeset -i okOnSecond +declare -A okOnThirdV +typeset -i okOnThird +declare -A okOnPlus3V +typeset -i okOnPlus3 +declare -A skipV +typeset -i skips +declare -A disabledTestV +typeset -i disabledTests +declare -A notInFlavourTestV +typeset -i notInFlavourTests + +export DIFF=$SCRIPT_HOME/testDiff.py +testError=0 +okOnSecond=0 +okOnThird=0 +okOnPlus3=0 +skips=0 +disabledTests=0 +notInFlavourTests=0 + + +# ----------------------------------------------------------------------------- +# +# Default value of skipList taken from an env var, to make things a little +# easier in distros with constantly failing tests +# +skipList="$CB_SKIP_LIST" + + + +# ----------------------------------------------------------------------------- +# +# usage +# +function usage() +{ + sfile="Usage: "$(basename $0) + empty=$(echo $sfile | tr 'a-zA-z/0-9.:' ' ') + echo "$sfile [-u (usage)]" + echo "$empty [-v (verbose)]" + echo "$empty [--filter ]" + echo "$empty [--match ]" + echo "$empty [--keep (don't remove output files)]" + echo "$empty [--dryrun (don't execute any tests)]" + echo "$empty [--dir ]" + echo "$empty [--fromIx ]" + echo "$empty [--toIx ]" + echo "$empty [--ixList ]" + echo "$empty [--skipList ]" + echo "$empty [--stopOnError (stop at first error encountered)]" + echo "$empty [--no-duration (removes duration mark on successful tests)]" + echo "$empty [--noCache (force broker to be started with the option --noCache)]" + echo "$empty [--cache (force broker to be started without the option --noCache)]" + echo "$empty [--noThreadpool (do not use a threadpool, unless specified by a test case. If not set, a thread pool of 200:20 is used by default in test cases which do not set notificationMode options)]" + echo "$empty [--xbroker (use external brokers, i.e. this script will *not* start any brokers, including context providers)]" + echo "$empty [ ]*" + echo + echo "* Please note that if a directory is passed as parameter, its entire path must be given, not only the directory-name" + echo "* If a file is passed as parameter, its entire file-name must be given, including '.test'" + echo "" + echo "Env Vars:" + echo "CB_MAX_TRIES: The number of tries before giving up on a failing test case" + echo "CB_SKIP_LIST: Default value for option --skipList" + echo "CB_SKIP_FUNC_TESTS: List of names of func tests to skip" + echo "CB_NO_CACHE: Start the broker without subscription cache (if set to 'ON')" + echo "CB_THREADPOOL: Start the broker without thread pool (if set to 'OFF')" + echo "CB_DIFF_TOOL: To view diff of failing tests with diff/tkdiff/meld/... (e.g. export CB_DIFF_TOOL=tkdiff)" + echo "CB_WITH_EXTERNAL_BROKER: The broker is started externally - not 'automatically' by the test harness (if set to 'ON')" + echo "" + echo "FT_FROM_IX: alternative to commandline parameter 'fromIx', index of test where to start (inclusive) " + echo "FT_TO_IX: alternative to commandline parameter 'toIx', index of test where to end (inclusive)" + echo + echo "Please note that, if using CB_WITH_EXTERNAL_BROKER (or --xbroker, which is the same), only a single test case should be run." + echo + exit $1 +} + + + +# ----------------------------------------------------------------------------- +# +# vMsg +# +function vMsg() +{ + if [ "$verbose" = "on" ] + then + echo $ME: $* + fi +} + + + +# ----------------------------------------------------------------------------- +# +# exitFunction +# +function exitFunction() +{ + exitCode=$1 + errorText=$2 + testFile=$3 + errorString=$4 + errorFile=$5 + forced=$6 + + echo -n "(FAIL $exitCode - $errorText) " + + if [ "$stopOnError" == "on" ] || [ "$forced" == "DIE" ] + then + echo $ME/$NAME: $errorString + + if [ "$errorFile" != "" ] && [ -f "$errorFile" ] + then + cat $errorFile 2> /dev/null + fi + + exit $exitCode + fi + + echo >> /tmp/orionFuncTestLog + echo '----- ' $NAME ' -----' >> /tmp/orionFuncTestLog + echo $errorString >> /tmp/orionFuncTestLog + + if [ "$errorFile" != "" ] && [ -f "$errorFile" ] + then + cat $errorFile >> /tmp/orionFuncTestLog 2> /dev/null + echo >> /tmp/orionFuncTestLog + fi + + echo >> /tmp/orionFuncTestLog + + testErrorV[$testError]=$testFile + testError=$testError+1 + toBeStopped=true; +} + + + +# ------------------------------------------------------------------------------ +# +# ME - name of script, to be used in error and verbose messages +# +ME=$(basename $0) +vMsg "$ME, in directory $SCRIPT_HOME" + + + +# ------------------------------------https://github.com/telefonicaid/fiware-orion/pull/394#discussion_r13321709------------------------------------------ +# +# Argument parsing +# +typeset -i fromIx +typeset -i toIx +verbose=off +dryrun=off +keep=off +stopOnError=off +testFilter=${TEST_FILTER:-"*.test"} +match="" +dir=$SCRIPT_HOME/cases +dirOrFile="" +dirGiven=no +filterGiven=no +showDuration=on +fromIx=0 +toIx=0 +ixList="" +noCache="" +threadpool=ON +xbroker=off + +vMsg "parsing options" +while [ "$#" != 0 ] +do + if [ "$1" == "-u" ]; then usage 0; + elif [ "$1" == "-v" ]; then verbose=on; + elif [ "$1" == "--dryrun" ]; then dryrun=on; + elif [ "$1" == "--keep" ]; then keep=on; + elif [ "$1" == "--stopOnError" ]; then stopOnError=on; + elif [ "$1" == "--filter" ]; then testFilter="$2"; filterGiven=yes; shift; + elif [ "$1" == "--match" ]; then match="$2"; shift; + elif [ "$1" == "--dir" ]; then dir="$2"; dirGiven=yes; shift; + elif [ "$1" == "--fromIx" ]; then fromIx=$2; shift; + elif [ "$1" == "--toIx" ]; then toIx=$2; shift; + elif [ "$1" == "--ixList" ]; then ixList=$2; shift; + elif [ "$1" == "--skipList" ]; then skipList=$2; shift; + elif [ "$1" == "--no-duration" ]; then showDuration=off; + elif [ "$1" == "--noCache" ]; then noCache=ON; + elif [ "$1" == "--cache" ]; then noCache=OFF; + elif [ "$1" == "--noThreadpool" ]; then threadpool=OFF; + elif [ "$1" == "--xbroker" ]; then xbroker=ON; + else + if [ "$dirOrFile" == "" ] + then + dirOrFile="$1" + else + echo $0: bad parameter/option: "'"${1}"'"; + echo + usage 1 + fi + fi + shift +done + +vMsg "options parsed" + +# ----------------------------------------------------------------------------- +# +# The function brokerStart looks at the env var CB_NO_CACHE to decide +# whether to start the broker with the --noCache option or not +# +if [ "$noCache" != "" ] +then + export CB_NO_CACHE=$noCache +fi + +# ----------------------------------------------------------------------------- +# +# The function brokerStart looks at the env var CB_THREADPOOL to decide +# whether to start the broker with pool of threads or not. +# Do not overwrite if a value is passed from environment +# +if [ "$CB_THREADPOOL" == "" ] +then + export CB_THREADPOOL=$threadpool +fi + +# ----------------------------------------------------------------------------- +# +# Check if fromIx is set through an env var and use if nothing +# else is set through commandline parameter +# +if [ "$FT_FROM_IX" != "" ] && [ $fromIx == 0 ] +then + fromIx=$FT_FROM_IX +fi + +# ----------------------------------------------------------------------------- +# +# Check if toIx is set through an env var and use if nothing +# else is set through commandline parameter +# +if [ "$FT_TO_IX" != "" ] && [ $toIx == 0 ] +then + toIx=$FT_TO_IX +fi + +echo "Run tests $fromIx to $toIx" + +# ------------------------------------------------------------------------------ +# +# Check unmatching --dir and 'parameter that is a directory' AND +# unmatching --filter and 'parameter that is a file' +# +# 1. If it is a directory - just change the 'dir' variable and continue +# 2. Else, it must be a file, or a filter. +# If the +# +singleFile=No +if [ "$dirOrFile" != "" ] +then + vMsg dirOrFile: $dirOrFile + vMsg dirGiven: $dirGiven + vMsg filterGiven: $filterGiven + vMsg dir: $dir + vMsg testFilter: $testFilter + + if [ -d "$dirOrFile" ] + then + if [ "$dirGiven" == "yes" ] + then + echo "$0: both '--dir' option and directory parameter given - not allowed" + exit 1 + fi + dir="$dirOrFile" + else + if [ "$filterGiven" == "yes" ] + then + echo "$0: both '--filter' option and file parameter given - not allowed" + exit 1 + fi + + singleFile=Yes + # + # If just a filename is given, keep the directory as is. + # If a whole path is given, use the directory-part as directory and the file-part as filter + # + dirPart=$(dirname $dirOrFile) + filePath=$(basename $dirOrFile) + xdir=$(basename $dirPart); + vMsg "dirPart: $dirPart" + vMsg "filePath: $filePath" + + if [ "$dirPart" != "." ] + then + dir=$(dirname $dirOrFile) + testFilter=$(basename $dirOrFile) + + # Last dir + test file ? + if [ -d test/functionalTest/cases/$dirPart ] + then + dirOrFile=test/functionalTest/cases/$dirPart + fi + else + testFilter=$(basename $dirOrFile) + fi + fi +fi + +# +# The option of running against an external broker "--xbroker" only works (for now, at least) with a single test case. +# Strange things may happen (due to the state inside the broker) if more that one test case are launched. +# This check avoid this situation. +# +# If in the future we want to be able to run more than one test case against an external broker, we'd need to make sure +# that each test case undoes all internal state inside the external broker. E.g. delete subscriptions, entities, etc. +# +if [ "$singleFile" == "No" ] && [ "$xbroker" == "ON" ] +then + echo "External broker can only be used with individual test cases" + exit 1 +fi + +vMsg directory: $dir +vMsg testFilter: $testFilter +vMsg "Script in $SCRIPT_HOME" + + + +# ----------------------------------------------------------------------------- +# +# Other global variables +# +toBeStopped=false + + + +# ------------------------------------------------------------------------------ +# +# xbroker - if this CLI is set, then the broker is not to be started as part of +# the test suite - another broker is assumed to be running already +# +if [ "$xbroker" == "ON" ] +then + export CB_WITH_EXTERNAL_BROKER=ON +fi + + + +# ----------------------------------------------------------------------------- +# +# Init files already sourced? +# +if [ "$CONTEXTBROKER_TESTENV_SOURCED" != "YES" ] +then + if [ -f "$SCRIPT_HOME/testEnv.sh" ] + then + # First, we try with a testEnv.sh file in the script home + vMsg Sourcing $SCRIPT_HOME/testEnv.sh + source $SCRIPT_HOME/testEnv.sh + elif [ -f "$SCRIPT_HOME/../../scripts/testEnv.sh" ] + then + # Second, we try with a testEnv.sh file in the script/testEnv.sh (realtive to git repo home). + # Note that the script home in this case is test/functionalTest + vMsg Sourcing $SCRIPT_HOME/../../scripts/testEnv.sh + source $SCRIPT_HOME/../../scripts/testEnv.sh + else + echo "------------------------------------------------------------------" + echo "Please source testEnv.sh before running the functional test suite." + echo "------------------------------------------------------------------" + exit 1 + fi +fi + +if [ "$CONTEXTBROKER_HARNESS_FUNCTIONS_SOURCED" != "YES" ] +then + if [ -f $SCRIPT_HOME/harnessFunctions.sh ] + then + vMsg Sourcing $SCRIPT_HOME/harnessFunctions.sh + source $SCRIPT_HOME/harnessFunctions.sh + else + echo "--------------------------------------------------------------------------------------------" + echo "Please source $SCRIPT_HOME/harnessFunctions.sh before running the functional test suite." + echo "--------------------------------------------------------------------------------------------" + exit 1 + fi +fi + + + +# ------------------------------------------------------------------------------ +# +# Preparations - cd to the test directory +# +dMsg Functional Tests Starting ... + +if [ "$dirOrFile" != "" ] && [ -d "$dirOrFile" ] +then + cd $dirOrFile +elif [ ! -d "$dir" ] +then + exitFunction 1 "$dir is not a directory" "HARNESS" "$dir" "" DIE +else + cd $dir +fi + +echo "Orion Functional tests starting" > /tmp/orionFuncTestLog +date >> /tmp/orionFuncTestLog + + + +# ------------------------------------------------------------------------------ +# +# Preparations - number of test cases +# +vMsg find in $(pwd), filter: $testFilter +if [ "$match" == "" ] +then + fileList=$(find . -name "$testFilter" | sort | sed 's/^.\///') +else + fileList=$(find . -name "$testFilter" | grep "$match" | sort | sed 's/^.\///') +fi +vMsg "fileList: $fileList" +typeset -i noOfTests +typeset -i testNo + + + +# ------------------------------------------------------------------------------ +# +# Count total number of tests (for progressing info in messages) +# +for i in $fileList +do + noOfTests=$noOfTests+1 +done + + + +# ------------------------------------------------------------------------------ +# +# fileCleanup - +# +function fileCleanup() +{ + filename=$1 + keepOutputFiles=$2 + path=$3 + dir=$(dirname $path) + + vMsg "---------------------------------------------------------" + vMsg "In fileCleanup for $filename in $dir" + vMsg "---------------------------------------------------------" + + if [ "$keepOutputFiles" != "on" ] + then + olddir=$PWD + cd $dir + + rm $filename.name 2> /dev/null + rm $filename.shellInit 2> /dev/null + rm $filename.shellInit.* 2> /dev/null + rm $filename.shell 2> /dev/null + rm $filename.shell.* 2> /dev/null + rm $filename.teardown 2> /dev/null + rm $filename.teardown.* 2> /dev/null + rm $filename.valgrind.out 2> /dev/null + rm $filename.valgrind.stop.out 2> /dev/null + rm $filename.out 2> /dev/null + rm $filename.regexpect 2> /dev/null + rm $filename.out.sorted 2> /dev/null + rm $filename.regexpect.sorted 2> /dev/null + rm $filename.blockSortDiff.out 2> /dev/null + rm $filename.diff 2> /dev/null + + cd $olddir + fi +} + + + +# ------------------------------------------------------------------------------ +# +# fileCreation - create the files for test execution +# +function fileCreation() +{ + path=$1 + filename=$2 + ret=0 + + dirname=$(dirname $path) + filename=$(basename $path .test) + + if [ "$dirname" != "." ] && [ "$dirname" != "" ] + then + pathWithoutExt=$dirname/$filename + vMsg New path: $path + else + pathWithoutExt=$filename + fi + + vMsg Creating test files for $pathWithoutExt + + + # + # Extract the NAME + # + NAME=$(sed -n '/--NAME--/,/^--/p' $path | grep -v "^--") + if [ "$NAME" == "" ] + then + exitFunction 2 "--NAME-- part is missing" "$path" "($path)" "" DIE + exit 2 # Just in case + fi + + # + # Extract the shell init script + # + if [ $(grep "\-\-SHELL\-INIT\-\-" $path | wc -l) -eq 1 ] + then + TEST_SHELL_INIT=${pathWithoutExt}.shellInit + vMsg "Creating $TEST_SHELL_INIT at $PWD" + sed -n '/--SHELL-INIT--/,/^--/p' $path | grep -v "^--" > $TEST_SHELL_INIT + else + exitFunction 3 "--SHELL-INIT-- part is missing" $path "($path)" "" DIE + fi + + # + # Extract the test shell script + # + if [ $(grep "\-\-SHELL\-\-" $path | wc -l) -eq 1 ] + then + TEST_SHELL=${pathWithoutExt}.shell + vMsg "Creating $TEST_SHELL at $PWD" + sed -n '/--SHELL--/,/^--/p' $path | grep -v "^--" > $TEST_SHELL + else + exitFunction 4 "--SHELL-- part is missing" $path "($path)" "" DIE + fi + + # + # Extract the REGEXPECT part + # + if [ $(grep "\-\-REGEXPECT\-\-" $path | wc -l) -eq 1 ] + then + TEST_REGEXPECT=${pathWithoutExt}.regexpect + vMsg "Creating $TEST_REGEXPECT at $PWD" + sed -n '/--REGEXPECT--/,/^--/p' $path | grep -v "^--" | sed '/^##/d' > $TEST_REGEXPECT + else + exitFunction 5 "--REGEXPECT-- part is missing" $path "($path)" "" DIE + fi + + # + # Extract the teardown script + # + if [ $(grep "\-\-TEARDOWN\-\-" $path | wc -l) -eq 1 ] + then + TEST_TEARDOWN=${pathWithoutExt}.teardown + vMsg "Creating $TEST_TEARDOWN at $PWD" + sed -n '/--TEARDOWN--/,/^--/p' $path | grep -v "^--" > $TEST_TEARDOWN + else + exitFunction 6 "--TEARDOWN-- part is missing" $path "($path)" "" DIE + fi +} + + + +# ------------------------------------------------------------------------------ +# +# partExecute +# +function partExecute() +{ + what=$1 + path=$2 + forcedDie=$3 + __tryNo=$4 + + vMsg Executing $what part for $path + dirname=$(dirname $path) + filename=$(basename $path .test) + + if [ "$dirname" != "." ] && [ "$dirname" != "" ] + then + path=$dirname/$filename.test + fi + + # + # Prepare to execute + # + chmod 755 $dirname/$filename.$what + rm -f $dirname/$filename.$what.stderr + rm -f $dirname/$filename.$what.stdout + $dirname/$filename.$what > $dirname/$filename.$what.stdout 2> $dirname/$filename.$what.stderr + exitCode=$? + linesInStderr=$(wc -l $dirname/$filename.$what.stderr | awk '{ print $1}' 2> /dev/null) + + # + # Check that stdout is empty + # + if [ "$linesInStderr" != "" ] && [ "$linesInStderr" != "0" ] + then + if [ $__tryNo == $MAX_TRIES ] + then + exitFunction 7 "$what: output on stderr" $path "($path): $what produced output on stderr" $dirname/$filename.$what.stderr "$forcedDie" + else + echo -n "(ERROR 7 - $what: output on stderr) " + fi + + partExecuteResult=7 + return + fi + + + # + # Check that exit code is ZERO + # + if [ "$exitCode" != "0" ] + then + if [ $__tryNo == $MAX_TRIES ] + then + exitFunction 8 $path "$what exited with code $exitCode" "($path)" $dirname/$filename.$what.stderr "$forcedDie" + else + echo -n "(ERROR 8 - $what: exited with code $exitCode) " + fi + + partExecuteResult=8 + return + fi + + + # + # Compare produced output with expected output + # + if [ "$what" == "shell" ] + then + mv $dirname/$filename.$what.stdout $dirname/$filename.out # We are very much used to this name ... + sed -i 's/[[:space:]]*$//' $dirname/$filename.out # Remove trailing whitespace in .out (reduces diff noise) + + # + # Special sorted diff or normal REGEX diff ? + # + blockDiff='no' + grep '^#SORT_START$' $dirname/$filename.regexpect > /dev/null 2>&1 + if [ $? == 0 ] + then + $SCRIPT_HOME/blockSortDiff.sh --referenceFile $dirname/$filename.regexpect --brokerOutputFile $dirname/$filename.out > $dirname/$filename.blockSortDiff.out + exitCode=$? + blockDiff='yes' + else + PYTHONIOENCODING=utf8 $DIFF -r $dirname/$filename.regexpect -i $dirname/$filename.out > $dirname/$filename.diff + exitCode=$? + fi + + if [ "$exitCode" != "0" ] + then + if [ $__tryNo == $MAX_TRIES ] + then + exitFunction 9 ".out and .regexpect differ" $path "($path) output not as expected" $dirname/$filename.diff + else + echo -n "(ERROR 9 - .out and .regexpect differ) " + fi + + if [ "$CB_DIFF_TOOL" != "" ] && [ $__tryNo == $MAX_TRIES ] + then + endDate=$(date) + if [ $blockDiff == 'yes' ] + then + $CB_DIFF_TOOL $dirname/$filename.regexpect.sorted $dirname/$filename.out.sorted + else + $CB_DIFF_TOOL $dirname/$filename.regexpect $dirname/$filename.out + fi + fi + partExecuteResult=9 + return + fi + fi + + partExecuteResult=0 +} + + + +# ------------------------------------------------------------------------------ +# +# runTest - the function that runs ONE test case +# +# 1. Remove old output files +# 2.1. Create the various test files from '$path' +# 3.1. Run the SHELL-INIT part +# 3.2. If [ $? != 0 ] || [ STDERR != empty ] ERROR +# 4.1. Run the SHELL part +# 4.2. If [ $? != 0 ] || [ STDERR != empty ] ERROR +# 5.1. Run the TEARDOWN part +# 5.2. If [ $? != 0 ] || [ STDERR != empty ] ERROR +# 6.1. Compare output with regexpect (or expect) +# 6.2. Not EQUAL: ERROR +# 7. If [ "$keep" != "yes" ] Remove all output files +# +# +function runTest() +{ + path=$1 + _tryNo=$2 + + runTestStatus="ok" + + vMsg path=$path + dirname=$(dirname $path) + filename=$(basename $path .test) + dir="" + + if [ "$dirname" != "." ] && [ "$dirname" != "" ] + then + path=$dirname/$filename.test + vMsg New path: $path + fi + + vMsg running test $path + + # 1. Remove old output files + fileCleanup $filename removeAll $path + if [ "$toBeStopped" == "yes" ] + then + echo toBeStopped == yes + runTestStatus="stopped" + return + fi + + # 2. Create the various test files from '$path' + fileCreation $path $filename + if [ "$toBeStopped" == "yes" ] + then + runTestStatus="stopped2" + return + fi + + # 3. Run the SHELL-INIT part + vMsg Executing SHELL-INIT part for $path + chmod 755 $dirname/$filename.shellInit + rm -f $dirname/$filename.shellInit.stderr + rm -f $dirname/$filename.shellInit.stdout + $dirname/$filename.shellInit > $dirname/$filename.shellInit.stdout 2> $dirname/$filename.shellInit.stderr + exitCode=$? + linesInStderr=$(wc -l $dirname/$filename.shellInit.stderr | awk '{ print $1}' 2> /dev/null) + + if [ "$linesInStderr" != "" ] && [ "$linesInStderr" != "0" ] + then + exitFunction 10 "SHELL-INIT produced output on stderr" $path "($path)" $dirname/$filename.shellInit.stderr + runTestStatus="shell-init-error" + return + fi + + if [ "$exitCode" != "0" ] + then + + # + # 3.2 Run the SHELL-INIT part AGAIN + # + # This 're-run' of the SHELL-INIT part is due to errors we've seen that seem to be caused by + # a try to start a broker while the old one (from the previous functest) is still running. + # No way to test this, except with some patience. + # + # We have seen 'ERROR 11' around once every 500-1000 functests (the suite is of almost 400 tests) + # and this fix, if working, will make us not see those 'ERROR 11' again. + # If we keep seeing 'ERROR 11' after this change then we will need to investigate further. + # + sleep 1 + rm -f $dirname/$filename.shellInit.stderr + rm -f $dirname/$filename.shellInit.stdout + $dirname/$filename.shellInit > $dirname/$filename.shellInit.stdout 2> $dirname/$filename.shellInit.stderr + exitCode=$? + linesInStderr=$(wc -l $dirname/$filename.shellInit.stderr | awk '{ print $1}' 2> /dev/null) + + if [ "$linesInStderr" != "" ] && [ "$linesInStderr" != "0" ] + then + exitFunction 20 "SHELL-INIT II produced output on stderr" $path "($path)" $dirname/$filename.shellInit.stderr + runTestStatus="shell-init-output-on-stderr" + return + fi + + if [ "$exitCode" != "0" ] + then + exitFunction 11 "SHELL-INIT exited with code $exitCode" $path "($path)" "" DIE + runTestStatus="shell-init-exited-with-"$exitCode + return + fi + fi + + # 4. Run the SHELL part (which also compares - FIXME P2: comparison should be moved to separate function) + partExecute shell $path "DontDie - only for SHELL-INIT" $_tryNo + shellResult=$partExecuteResult + if [ "$toBeStopped" == "yes" ] + then + runTestStatus="shell-failed" + return + fi + + # 5. Run the TEARDOWN part + partExecute teardown $path "DIE" 0 + teardownResult=$partExecuteResult + vMsg "teardownResult: $teardownResult" + vMsg "shellResult: $shellResult" + + if [ "$shellResult" == "0" ] && [ "$teardownResult" == "0" ] + then + # 6. Remove output files + vMsg "Remove output files: fileCleanup $filename $keep" + fileCleanup $filename $keep $path + else + file=$(basename $path .test) + cp /tmp/contextBroker.log $file.contextBroker.log + runTestStatus="test-failed" + fi +} + + + +# ----------------------------------------------------------------------------- +# +# testDisabled +# +function testDisabled +{ + testcase=$1 + typeset -i dIx + dIx=0 + while [ $dIx -lt ${#DISABLED[@]} ] + do + # Comparison is done based in filename, skipping the path (see https://stackdiary.com/tutorials/bash-get-filename-from-path/) + if [ "${testcase##*/}" == "${DISABLED[$dIx]##*/}" ] + then + echo "Disabled" + + # + # NOTE: In a non-disabled test, running inside the valgrind test suite, the function 'localBrokerStart()' (from harnessFunctions.sh) + # redirects the output of "valgrind contextBroker" to the file /tmp/valgrind.out. + # Later, the valgrind test suite uses the existence of this file (/tmp/valgrind.out) to detect errors in the valgrind execution. + # But, in the case of a disabled func test, we will not start the test case. and thus we will not reach 'localBrokerStart()', so the + # file will not be created and an error will be flagged by the valgrind test suite. + # The simplest solution is to simply create the file here, in the case of a disabled test. + # + echo "Disabled" > /tmp/valgrind.out + return + fi + dIx=$dIx+1 + done + echo NOT Disabled +} + + + +# ----------------------------------------------------------------------------- +# +# testMatchExprFlavour +# +function testMatchExprFlavour +{ + testcase=$1 + + if grep -q JEXL_EXPR_FLAVOUR $testcase + then + if $(contextBroker --version | grep -q jexl-expr) + then + echo NOT Disabled + else + echo "Disabled" + echo "Disabled" > /tmp/valgrind.out + fi + else + echo NOT Disabled + fi +} + + + +# ------------------------------------------------------------------------------ +# +# Main loop +# +vMsg Total number of tests: $noOfTests +testNo=0 +for testFile in $fileList +do + if [ -d "$testFile" ] + then + continue + fi + + testNo=$testNo+1 + + if [ $fromIx != 0 ] && [ $testNo -lt $fromIx ] + then + continue; + fi + + if [ $toIx != 0 ] && [ $testNo -gt $toIx ] + then + continue; + fi + + # + # Disabled test? + # + disabled=$(testDisabled $testFile) + if [ "$disabled" == "Disabled" ] + then + disabledTestV[$disabledTests]=$testNo': '$testFile + disabledTests=$disabledTests+1 + continue + fi + + # + # Should the test be skipped due to it doesn't mach in the contextBroker flavour? + # + notInFlavour=$(testMatchExprFlavour $testFile) + if [ "$notInFlavour" == "Disabled" ] + then + notInFlavourTestV[$notInFlavourTests]=$testNo': '$testFile + notInFlavourTests=$notInFlavourTests+1 + continue + fi + + if [ "$ixList" != "" ] + then + hit=$(echo ' '$ixList' ' | grep ' '$testNo' ') + if [ "$hit" == "" ] + then + # Test case not found in ix-list, so it is not executed + continue + fi + fi + + if [ "$CB_SKIP_FUNC_TESTS" != "" ] + then + hit=$(echo ' '$CB_SKIP_FUNC_TESTS' ' | grep ' '$testFile' ') + if [ "$hit" != "" ] + then + # Test case found in skip-list, so it is skipped + skipV[$skips]=$testNo': '$testFile + skips=$skips+1 + continue + fi + fi + + if [ "$skipList" != "" ] + then + hit=$(echo ' '$skipList' ' | grep ' '$testNo' ') + if [ "$hit" != "" ] + then + # Test case found in skip-list, so it is skipped + skipV[$skips]=$testNo': '$testFile + skips=$skips+1 + continue + fi + fi + + startDate=$(date) + start=$(date --date="$startDate" +%s) + endDate="" + typeset -i tryNo + tryNo=1 + + if [ "$dryrun" == "off" ] + then + while [ $tryNo -le $MAX_TRIES ] + do + if [ "$verbose" == "off" ] + then + tryNoInfo="" + if [ $tryNo != "1" ] + then + tryNoInfo="(intent $tryNo)" + fi + + init=$testFile" ................................................................................................................." + init=${init:0:110} + printf "%04d/%d: %s %s " "$testNo" "$noOfTests" "$init" "$tryNoInfo" + else + printf "Running test %04d/%d: %s\n" "$testNo" "$noOfTests" "$testFile" + fi + + runTest $testFile $tryNo + if [ "$shellResult" == "0" ] + then + if [ $tryNo != 1 ] + then + if [ $tryNo == 2 ] + then + okOnSecondV[$okOnSecond]=$testFile + okOnSecond=$okOnSecond+1 + elif [ $tryNo == 3 ] + then + okOnThirdV[$okOnThird]=$testFile + okOnThird=$okOnThird+1 + else + okOnPlus3V[$okOnPlus3]=$testFile + okOnPlus3=$okOnPlus3+1 + fi + echo "OK" + fi + break + else + tryNo=$tryNo+1 + echo + fi + done + else + if [ "$verbose" == "off" ] + then + init=$testFile" ................................................................................................................." + init=${init:0:110} + printf "%04d/%d: %s " "$testNo" "$noOfTests" "$init" + else + printf "Running test %04d/%d: %s\n" "$testNo" "$noOfTests" "$testFile" + fi + fi + + if [ "$endDate" == "" ] # Could have been set in 'partExecute' + then + endDate=$(date) + fi + + end=$(date --date="$endDate" +%s) + typeset -i secs + secs=$end-$start + if [ "$showDuration" == "on" ] + then + if [ $secs -lt 10 ] + then + xsecs=0$secs + else + xsecs=$secs + fi + echo $xsecs seconds + else + echo "SUCCESS" + fi +done + + +testEndTime=$(date +%s.%2N) +testDiffTime=$(echo $testEndTime - $testStartTime | bc)" seconds" +echo Total test time: $testDiffTime + + +typeset -i ix +exitCode=0 +# ------------------------------------------------------------------------------ +# +# Check for errors - if any, print to stdout +# +if [ "$testError" != "0" ] +then + echo + echo "Orion Functional Test Log File:" + echo "================================================================================" + cat /tmp/orionFuncTestLog 2> /dev/null + echo "================================================================================" + echo + echo "----------- Failing tests ------------------" + + ix=0 + while [ $ix -lt $testError ] + do + echo " o " ${testErrorV[$ix]} + ix=$ix+1 + done + exitCode=1 +fi + + + +# ------------------------------------------------------------------------------ +# +# Check for reintents +# +if [ "$okOnSecond" != "0" ] +then + echo + echo "$okOnSecond test cases OK in the second attempt:" + + ix=0 + while [ $ix -lt $okOnSecond ] + do + echo " o " ${okOnSecondV[$ix]} + ix=$ix+1 + done +fi + +if [ "$okOnThird" != "0" ] +then + echo + echo "$okOnThird test cases OK in the third attempt:" + + ix=0 + while [ $ix -lt $okOnThird ] + do + echo " o " ${okOnThirdV[$ix]} + ix=$ix+1 + done +fi + +if [ "$okOnPlus3" != "0" ] +then + echo + echo "$okOnPlus3 test cases OK after three or more failed attempts:" + + ix=0 + while [ $ix -lt $okOnPlus3 ] + do + echo " o " ${okOnPlus3V[$ix]} + ix=$ix+1 + done +fi + +if [ $skips != 0 ] +then + echo + echo WARNING: $skips test cases skipped: + ix=0 + while [ $ix -lt $skips ] + do + echo " o " ${skipV[$ix]} + ix=$ix+1 + done +fi + +if [ $disabledTests != 0 ] +then + echo + echo WARNING: $disabledTests test cases disabled: + ix=0 + while [ $ix -lt $disabledTests ] + do + echo " o " ${disabledTestV[$ix]} + ix=$ix+1 + done +fi + +if [ $notInFlavourTests != 0 ] +then + echo + echo WARNING: $notInFlavourTests test cases were not executed due to contexBroker not matching flavour: + ix=0 + while [ $ix -lt $notInFlavourTests ] + do + echo " o " ${notInFlavourTestV[$ix]} + ix=$ix+1 + done +fi + +exit $exitCode From 945d2ee8da668fb68c03f516267efd8f27d762f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 4 Jun 2024 19:52:06 +0200 Subject: [PATCH 308/390] FIX ftest --- .../log_deprecate_warning.test | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test index a340c8cd99..0d75384406 100644 --- a/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test +++ b/test/functionalTest/cases/0000_deprecated_checkings/log_deprecate_warning.test @@ -331,14 +331,10 @@ Deprecated usage of geo:point detected in attribute location at entity update, p Deprecated usage of legacyForwarding mode in registration creation (regId: REG_ID) Deprecated usage of legacyForwarding mode detected in existing registration (regId: REG_ID) Deprecated usage of legacyForwarding mode in query forwarding operation (regId: REG_ID) -Notification (regId: REG_ID) response NOT OK, http code: 400 -Raising alarm BadInput 127.0.0.1: JSON Parse Error: unknown field: /error -Raising alarm ForwardingError localhost:9801/v2/queryContext: error parsing reply from prov app: {"error":"BadRequest","description":"JSON Parse Error: unknown field: /error"} -Releasing alarm BadInput 127.0.0.1 +Raising alarm ForwardingError localhost:9801/v2/queryContext: forwarding failure for sender-thread: Couldn't connect to server Deprecated usage of legacyForwarding mode in update forwarding operation (regId: REG_ID) -Notification (regId: REG_ID) response NOT OK, http code: 400 -Raising alarm BadInput 127.0.0.1: JSON Parse Error: unknown field: /error -Raising alarm ForwardingError localhost:9801/v2/updateContext: error parsing reply from prov app: {"error":"BadRequest","description":"JSON Parse Error: unknown field: /error"} +Raising alarm ForwardingError localhost:9801/v2/updateContext: forwarding failure for sender-thread: Couldn't connect to server +Raising alarm BadInput 127.0.0.1: The requested entity has not been found. Check type and id 10. Get statistics and see deprecatedFeatures counters From 5ad0f34e728bc9e8ec1a57a0824ea8c99e8d29ae Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Wed, 5 Jun 2024 17:03:50 +0900 Subject: [PATCH 309/390] (JP) FIX to reference presentation (#4566) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2606236355..c3a2834151 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ recommended to have a look to the brief - Orion Context Broker [(en)](https://www.slideshare.net/slideshow/orion-context-broker-introduction-20240604/269503234) - [(jp)](https://www.slideshare.net/slideshows/orion-context-broker-introduction-20240227/266517846) + [(jp)](https://www.slideshare.net/slideshow/orion-context-broker-introduction-20240605/269515246) - NGSIv2 Overview for Developers That Already Know NGSIv1 [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220523) [(jp)](https://www.slideshare.net/fisuda/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220526) From 2aac6b7da546ab0bab0428e4a83825c8c162da8d Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Wed, 5 Jun 2024 19:07:06 +0900 Subject: [PATCH 310/390] (JP) Synchronized with the English version --- doc/manuals.jp/admin/logs.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/manuals.jp/admin/logs.md b/doc/manuals.jp/admin/logs.md index 2023f5adbc..7c06f2dd88 100644 --- a/doc/manuals.jp/admin/logs.md +++ b/doc/manuals.jp/admin/logs.md @@ -346,15 +346,22 @@ time=2020-10-26T15:06:14.642Z | lvl=INFO | corr=c4a3192e-179c-11eb-ac8f-000c29df `-logDeprecate` CLI 設定 (または [ログ管理 REST API](management_api.md#log-configs-and-trace-levels) の `deprecate` パラメータ) が使用されている場合、次の WARN トレースが生成されます: -* NGSIv1 リクエスト (ペイロードありとペイロードなしの両方)。 注: これには、[`"attrsFormat": "legacy"`](../orion-api.md#subscriptionnotification) - を使用した通知や、[`"legacyForwarding": true`](../orion-api.md#registrationprovider) - を使用したレジストレーションに対応する転送リクエストは含まれないことに注意してください)。例えば： +* NGSIv1 リクエスト (ペイロードありとペイロードなしの両方)。例えば： ``` time=2023-05-25T14:27:45.958Z | lvl=WARN | corr=513bd10e-fb08-11ed-8ad7-000c29583ca5 | trans=1685024865-125-00000000001 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[171]:logInfoRequestWithPayload | msg=Deprecated NGSIv1 request received: POST /v1/queryContext, request payload (48 bytes): { "entities": [ { "type": "T1", "id": "E1" } ] }, response code: 200 time=2023-05-25T14:27:46.041Z | lvl=WARN | corr=51490536-fb08-11ed-9782-000c29583ca5 | trans=1685024865-125-00000000002 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[114]:logInfoRequestWithoutPayload | msg=Deprecated NGSIv1 request received: GET /v1/contextEntities/E, response code: 200 ``` +* [`"legacyForwarding": true`](../orion-api.md#registrationprovider)) の使用。例えば： + +``` +time=2024-01-11T13:57:13.537Z | lvl=WARN | corr=527378d8-b089-11ee-875d-080027cd35f1 | trans=1704981432-655-00000000006 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=mongoRegistrationCreate.cpp[235]:mongoRegistrationCreate | msg=Deprecated usage of legacyForwarding mode in registration creation (regId: 659ff3b9691855f16d00ec5a) +time=2024-01-11T13:57:13.565Z | lvl=WARN | corr=52778eaa-b089-11ee-861c-080027cd35f1 | trans=1704981432-655-00000000007 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=mongoRegistrationGet.cpp[93]:setProvider | msg=Deprecated usage of legacyForwarding mode detected in existing registration (regId: 659ff3b9691855f16d00ec5a) +time=2024-01-11T13:57:13.595Z | lvl=WARN | corr=527c0912-b089-11ee-bb8c-080027cd35f1 | trans=1704981432-655-00000000008 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=postQueryContext.cpp[191]:queryForward | msg=Deprecated usage of legacyForwarding mode in query forwarding operation (regId: 659ff3b9691855f16d00ec5a) +time=2024-01-11T13:57:13.624Z | lvl=WARN | corr=52808938-b089-11ee-9835-080027cd35f1 | trans=1704981432-655-00000000010 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=postUpdateContext.cpp[163]:updateForward | msg=Deprecated usage of legacyForwarding mode in update forwarding operation (regId: 659ff3b9691855f16d00ec5a) +``` + * `geo:point`, `geo:line`, `geo:box` また `geo:line` の使用 ``` From b6b86f32b35a9777611bdd337888c7479f512bc1 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Wed, 5 Jun 2024 19:12:50 +0900 Subject: [PATCH 311/390] (JP) REMOVE documentation about subs legacy format (#4564) --- doc/manuals.jp/admin/database_model.md | 2 +- doc/manuals.jp/admin/statistics.md | 1 - doc/manuals.jp/deprecated.md | 1 + doc/manuals.jp/orion-api.md | 37 ++------------------------ 4 files changed, 4 insertions(+), 37 deletions(-) diff --git a/doc/manuals.jp/admin/database_model.md b/doc/manuals.jp/admin/database_model.md index 52081b2be8..7bb39f597e 100644 --- a/doc/manuals.jp/admin/database_model.md +++ b/doc/manuals.jp/admin/database_model.md @@ -225,7 +225,7 @@ Orion Context Broker は、データベース内で次のサブセクション - **conditions** : 通知をトリガーする属性のリストです。 - **expression** : 更新が来たときに通知を送信するかどうかを評価するために使用される式です。 次のフィールドで構成されています : q, mq, georel, geometry and/or coords (オプション) - **count** : サブスクリプションに関連付けられて送信された通知の数です -- **format** : 通知を送信するために使用する形式。可能な値はは **JSON** (NGSIv1 レガシー形式の JSON 通知を意味する)、**normalized**, **keyValues**, **simplifiedNormalized**, **simplifiedKeyValues**, **values** (最後の5つは NGSIv2 形式で使用されます) です +- **format** : 通知を送信するために使用する形式。可能な値は、**normalized**, **keyValues**, **simplifiedNormalized**, **simplifiedKeyValues**, **values** です - **status** : `active` (アクティブなサブスクリプションの場合) または `inactive` (非アクティブなサブスクリプションの場合)、 または `oneshot` ([oneshot サブスクリプション](../orion-api.md#oneshot-subscriptions) の場合) のいずれか。Orion API は追加の状態 (`expired`など) を考慮しますが、DB にヒットすることはありません (Orion によって管理されます) diff --git a/doc/manuals.jp/admin/statistics.md b/doc/manuals.jp/admin/statistics.md index bcf301f0be..2aa42f09e7 100644 --- a/doc/manuals.jp/admin/statistics.md +++ b/doc/manuals.jp/admin/statistics.md @@ -45,7 +45,6 @@ Orion Context broker は、`GET /statistics` と `GET /cache/statistics` を介 "deprecatedFeatures": { "geoFormat": 2, "ngsiv1Forwarding": 4, - "ngsiv1NotifFormat": 4, "ngsiv1Requests": 4 }, "invalidRequests": 2, diff --git a/doc/manuals.jp/deprecated.md b/doc/manuals.jp/deprecated.md index d535d129c7..d22e8711a1 100644 --- a/doc/manuals.jp/deprecated.md +++ b/doc/manuals.jp/deprecated.md @@ -28,6 +28,7 @@ * `POST /NGSI10/updateContext` * `POST /v1/queryContext` * `POST /NGSI10/queryContext` + * サブスクリプション通知の NGSIv1 形式 (`notification.atttrsFormat` が `legacy` に設定) は Orion 4.0.0 で削除されました * `POST /v2/op/query` の `attributes` フィールドは、Orion 1.15.0 にあります。これらの属性を持つエンティティのみを返すためには、クエリに対するレスポンスにどの属性を含めるかを選択する `attrs` と、`expression` 内の `q` の単項属性フィルタ (unary attribute filter) の組み合わせです。それらを代わりに指定していください * Orion 1.14.0 では `POST /v2/op/update` の `APPEND`, `APPEND_STRICT`, `UPDATE`, `DELETE`, `REPLACE` の使用は非推奨です。`append`, `appendStrict`, `update`, `delete`, `replace` を代わりに使ってください * Orion 1.13.0 ではメタデータ ID が推奨されていません (Orion 2.2.0 で削除されました)。一方、この機能は NGSIv2 と互換性がありません。JSON 表現形式の属性名は JSON オブジェクトのキーとして使用されるため、名前を複製することはできません。一方、IDs は、属性名にプレフィックス/サフィックスを使用して簡単に実装することができます。たとえば、`temperature:ground` および `temperature:ceiling` です。 この非推奨の結果、次のオペレーションも非推奨になりました : diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index d815003198..a4891cb925 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -1978,40 +1978,7 @@ TTL モニタスレッドのデフォルトのスリープ間隔は MongoDB で } ``` -`attrsFormat` が `legacy` の場合、サブスクリプション表現は NGSIv1 形式に従います。このようにして、ユーザは、NGSIv1 -レガシー通知レシーバによる Orion サブスクリプションの拡張 (フィルタリングなど) の恩恵を受けることができます。 - -NGSIv1 は非推奨であることに注意してください。したがって、`legacy` 通知形式を使用することはお勧めしません。 - -```json -{ - "subscriptionId": "56e2ad4e8001ff5e0a5260ec", - "originator": "localhost", - "contextResponses": [{ - "contextElement": { - "type": "Car", - "isPattern": "false", - "id": "Car1", - "attributes": [{ - "name": "temperature", - "type": "centigrade", - "value": "26.5", - "metadatas": [{ - "name": "TimeInstant", - "type": "recvTime", - "value": "2015-12-12 11:11:11.123" - }] - }] - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - }] -} -``` - -通知には、関連するサブスクリプションの形式の値を含む `Ngsiv2-AttrsFormat` (`attrsFormat` が `legacy` の場合を想定) HTTP +通知には、関連するサブスクリプションの形式の値を含む `Ngsiv2-AttrsFormat` HTTP ヘッダを含める必要があります。これにより、通知受信者は通知ペイロードを処理しなくても形式を認識できます。 **注:** 通知には必ず1つのエンティティが含まれることに注意してください。そのため、`data` @@ -4838,7 +4805,7 @@ _**レスポンス・ペイロード**_ |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `attrs` または `exceptAttrs` | | array | 両方を同時に使用することはできません。
  • attrs: 通知メッセージに含まれる属性のリスト。また、attrsFormat value が使用されたときに属性が通知に表示される順序も定義します ("通知メッセージ" のセクションを参照)。空のリストは、すべての属性が通知に含まれることを意味します。詳細については、[属性とメタデータのフィルタリング](#filtering-out-attributes-and-metadata)のセクションを参照してください
  • exceptAttrs: 通知メッセージから除外される属性のリスト。つまり、通知メッセージには、このフィールドにリストされているものを除くすべてのエンティティ属性が含まれます。 空でないリストでなければなりません。
  • attrs も exceptAttrs も指定されていない場合、すべての属性が通知に含まれます
| | [`http`](#subscriptionnotificationhttp), [`httpCustom`](#subscriptionnotificationhttpcustom), [`mqtt`](#subscriptionnotificationmqtt) または [`mqttCustom`](#subscriptionnotificationmqttcustom)| ✓ | object | それらの1つが存在する必要がありますが、同時に1つを超えてはなりません。x トランスポート・プロトコルを介して配信される通知のパラメータを伝達するために使用されます | -| `attrsFormat` | ✓ | string | エンティティが通知でどのように表されるかを指定します。受け入れられる値は、`normalized` (デフォルト), `simplifiedNormalized`, `keyValues`, `simplifiedKeyValues`, `values`, または `legacy` です。
`attrsFormat` がそれらとは異なる値をとると、エラーが発生します。 詳細については、[通知メッセージ](#notification-messages)のセクションを参照してください | +| `attrsFormat` | ✓ | string | エンティティが通知でどのように表されるかを指定します。受け入れられる値は、`normalized` (デフォルト), `simplifiedNormalized`, `keyValues`, `simplifiedKeyValues`, または `values` です。
`attrsFormat` がそれらとは異なる値をとると、エラーが発生します。 詳細については、[通知メッセージ](#notification-messages)のセクションを参照してください | | `metadata` | ✓ | string | 通知メッセージに含まれるメタデータのリスト。詳細については、[属性とメタデータの除外](#filtering-out-attributes-and-metadata)のセクションを参照してください | | `onlyChangedAttrs` | ✓ | boolean | `true` の場合、通知には、`attrs` または `exceptAttrs` フィールドと組み合わせて、トリガー更新リクエストで変更された属性のみが含まれます。(フィールドが省略されている場合、デフォルトは `false` です)) | | `covered` | ✓ | boolean | `true` の場合、通知には、エンティティに存在しない場合でも (この場合、`null` 値で) `attrs` フィールドで定義されたすべての属性が含まれます。(デフォルト値は false です)。詳細については、[対象サブスクリプション](#covered-subscriptions)のセクションを参照してください | From f755865c4bfa526d43fb21914be0e6fa844ec327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 5 Jun 2024 12:17:54 +0200 Subject: [PATCH 312/390] FIX issue raised in code review --- .../ngsi10.test.DISABLED | 365 ------------------ .../ngsi9.test.DISABLED | 335 ---------------- .../0968_invalid_regex_subs.test} | 54 +-- 3 files changed, 16 insertions(+), 738 deletions(-) delete mode 100644 test/functionalTest/cases/0585_empty_type_in_response/ngsi10.test.DISABLED delete mode 100644 test/functionalTest/cases/0585_empty_type_in_response/ngsi9.test.DISABLED rename test/functionalTest/cases/{0968_ngsi10_asterisk_subs/0968_ngsi10_asterisk_subs.test.DISABLED => 0968_invalid_regex_subs/0968_invalid_regex_subs.test} (62%) diff --git a/test/functionalTest/cases/0585_empty_type_in_response/ngsi10.test.DISABLED b/test/functionalTest/cases/0585_empty_type_in_response/ngsi10.test.DISABLED deleted file mode 100644 index cd04a058c6..0000000000 --- a/test/functionalTest/cases/0585_empty_type_in_response/ngsi10.test.DISABLED +++ /dev/null @@ -1,365 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -NGSI10 Empty Entity Type Check - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -# -# 01. Update/APPEND an entity with type Room and id Room, Attribute temperature -# 02. Query with convop GET /v1/contextEntities/Room -# 03. Query with convop GET /v1/contextEntities/Room/attributes/temperature -# 04. Query with convop GET /v1/contextEntities/type/Room/id/Room -# 05. Query with convop GET /v1/contextEntities/type/Room/id/Room/attributes/temperature -# 06. Query with convop GET /v1/contextEntityTypes/Room -# 07. Query with convop GET /v1/contextEntityTypes/Room/attributes/temperature -# - -echo "01. Update/APPEND an entity with type Room and id Room, Attribute temperature" -echo "=============================================================================" -payload='{ - "contextElements": [ - { - "type": "Room", - "isPattern": "false", - "id": "Room", - "attributes": [ - { - "name": "temperature", - "type": "degree", - "value": "20", - "metadatas": [ - { - "name": "ID", - "type": "string", - "value": "backup" - } - ] - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url /v1/updateContext --payload "$payload" -echo -echo - - -echo "02. Query with convop GET /v1/contextEntities/Room" -echo "==================================================" -orionCurl --url /v1/contextEntities/Room -echo -echo - - -echo "03. Query with convop GET /v1/contextEntities/Room/attributes/temperature" -echo "=========================================================================" -orionCurl --url /v1/contextEntities/Room/attributes/temperature -echo -echo - - -echo "04. Query with convop GET /v1/contextEntities/type/Room/id/Room" -echo "===============================================================" -orionCurl --url /v1/contextEntities/type/Room/id/Room -echo -echo - - -echo "05. Query with convop GET /v1/contextEntities/type/Room/id/Room/attributes/temperature" -echo "======================================================================================" -orionCurl --url /v1/contextEntities/type/Room/id/Room/attributes/temperature -echo -echo - - -echo "06. Query with convop GET /v1/contextEntityTypes/Room" -echo "=====================================================" -orionCurl --url /v1/contextEntityTypes/Room -echo -echo - - -echo "07. Query with convop GET /v1/contextEntityTypes/Room/attributes/temperature" -echo "============================================================================" -orionCurl --url /v1/contextEntityTypes/Room/attributes/temperature -echo -echo - - ---REGEXPECT-- -01. Update/APPEND an entity with type Room and id Room, Attribute temperature -============================================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 263 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "ID", - "type": "string", - "value": "backup" - } - ], - "name": "temperature", - "type": "degree", - "value": "" - } - ], - "id": "Room", - "isPattern": "false", - "type": "Room" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -02. Query with convop GET /v1/contextEntities/Room -================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 242 - -{ - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "ID", - "type": "string", - "value": "backup" - } - ], - "name": "temperature", - "type": "degree", - "value": "20" - } - ], - "id": "Room", - "isPattern": "false", - "type": "Room" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } -} - - -03. Query with convop GET /v1/contextEntities/Room/attributes/temperature -========================================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 177 - -{ - "attributes": [ - { - "metadatas": [ - { - "name": "ID", - "type": "string", - "value": "backup" - } - ], - "name": "temperature", - "type": "degree", - "value": "20" - } - ], - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } -} - - -04. Query with convop GET /v1/contextEntities/type/Room/id/Room -=============================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 242 - -{ - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "ID", - "type": "string", - "value": "backup" - } - ], - "name": "temperature", - "type": "degree", - "value": "20" - } - ], - "id": "Room", - "isPattern": "false", - "type": "Room" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } -} - - -05. Query with convop GET /v1/contextEntities/type/Room/id/Room/attributes/temperature -====================================================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 177 - -{ - "attributes": [ - { - "metadatas": [ - { - "name": "ID", - "type": "string", - "value": "backup" - } - ], - "name": "temperature", - "type": "degree", - "value": "20" - } - ], - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } -} - - -06. Query with convop GET /v1/contextEntityTypes/Room -===================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 265 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "ID", - "type": "string", - "value": "backup" - } - ], - "name": "temperature", - "type": "degree", - "value": "20" - } - ], - "id": "Room", - "isPattern": "false", - "type": "Room" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - -07. Query with convop GET /v1/contextEntityTypes/Room/attributes/temperature -============================================================================ -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 265 - -{ - "contextResponses": [ - { - "contextElement": { - "attributes": [ - { - "metadatas": [ - { - "name": "ID", - "type": "string", - "value": "backup" - } - ], - "name": "temperature", - "type": "degree", - "value": "20" - } - ], - "id": "Room", - "isPattern": "false", - "type": "Room" - }, - "statusCode": { - "code": "200", - "reasonPhrase": "OK" - } - } - ] -} - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/0585_empty_type_in_response/ngsi9.test.DISABLED b/test/functionalTest/cases/0585_empty_type_in_response/ngsi9.test.DISABLED deleted file mode 100644 index 5136e99181..0000000000 --- a/test/functionalTest/cases/0585_empty_type_in_response/ngsi9.test.DISABLED +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright 2014 Telefonica Investigacion y Desarrollo, S.A.U -# -# This file is part of Orion Context Broker. -# -# Orion Context Broker is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Orion Context Broker is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero -# General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. -# -# For those usages not covered by this license please contact with -# iot_support at tid dot es - -# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh - ---NAME-- -NGSI9 Empty Entity Type Check - ---SHELL-INIT-- -dbInit CB -brokerStart CB - ---SHELL-- - -# -# 01. Register an entity with type Room and id Room -# 02. Discover with convop GET /v1/registry/contextEntities/Room -# 03. Discover with convop GET /v1/registry/contextEntities/Room/attributes/temperature -# 04. Discover with convop GET /v1/registry/contextEntityTypes/Room -# 05. Discover with convop GET /v1/registry/contextEntityTypes/Room/attributes/temperature -# 06. Discover with convop GET /v1/registry/contextEntities/type/Room/id/Room -# 07. Discover with convop GET /v1/registry/contextEntities/type/Room/id/Room/attributes/temperature -# - -echo "01. Register an entity with type Room and id Room" -echo "=================================================" -payload='{ - "contextRegistrations": [ - { - "entities": [ - { - "type": "Room", - "isPattern": "false", - "id": "Room" - }, - { - "type": "Room", - "isPattern": "false", - "id": "OfficeRoom" - } - ], - "attributes": [ - { - "name": "temperature", - "type": "degree" - } - ], - "providingApplication": "http://localhost:1028/application" - } - ], - "duration": "PT1H" -}' -orionCurl --url /v1/registry/registerContext --payload "$payload" -echo -echo - - -echo "02. Discover with convop GET /v1/registry/contextEntities/Room" -echo "==============================================================" -orionCurl --url /v1/registry/contextEntities/Room -echo -echo - - -echo "03. Discover with convop GET /v1/registry/contextEntities/Room/attributes/temperature" -echo "=====================================================================================" -orionCurl --url /v1/registry/contextEntities/Room/attributes/temperature -echo -echo - - -echo "04. Discover with convop GET /v1/registry/contextEntityTypes/Room" -echo "=================================================================" -orionCurl --url /v1/registry/contextEntityTypes/Room -echo -echo - - -echo "05. Discover with convop GET /v1/registry/contextEntityTypes/Room/attributes/temperature" -echo "========================================================================================" -orionCurl --url /v1/registry/contextEntityTypes/Room/attributes/temperature -echo -echo - -echo "06. Discover with convop GET /v1/registry/contextEntities/type/Room/id/Room" -echo "===========================================================================" -orionCurl --url /v1/registry/contextEntities/type/Room/id/Room -echo -echo - - -echo "07. Discover with convop GET /v1/registry/contextEntities/type/Room/id/Room/attributes/temperature" -echo "==================================================================================================" -orionCurl --url /v1/registry/contextEntities/type/Room/id/Room/attributes/temperature -echo -echo - - ---REGEXPECT-- -01. Register an entity with type Room and id Room -================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 63 - -{ - "duration": "PT1H", - "registrationId": "REGEX([0-9a-f]{24})" -} - - -02. Discover with convop GET /v1/registry/contextEntities/Room -============================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 234 - -{ - "contextRegistrationResponses": [ - { - "contextRegistration": { - "attributes": [ - { - "name": "temperature", - "type": "degree" - } - ], - "entities": [ - { - "id": "Room", - "isPattern": "false", - "type": "Room" - } - ], - "providingApplication": "http://localhost:1028/application" - } - } - ] -} - - -03. Discover with convop GET /v1/registry/contextEntities/Room/attributes/temperature -===================================================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 234 - -{ - "contextRegistrationResponses": [ - { - "contextRegistration": { - "attributes": [ - { - "name": "temperature", - "type": "degree" - } - ], - "entities": [ - { - "id": "Room", - "isPattern": "false", - "type": "Room" - } - ], - "providingApplication": "http://localhost:1028/application" - } - } - ] -} - - -04. Discover with convop GET /v1/registry/contextEntityTypes/Room -================================================================= -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 288 - -{ - "contextRegistrationResponses": [ - { - "contextRegistration": { - "attributes": [ - { - "name": "temperature", - "type": "degree" - } - ], - "entities": [ - { - "id": "Room", - "isPattern": "false", - "type": "Room" - }, - { - "id": "OfficeRoom", - "isPattern": "false", - "type": "Room" - } - ], - "providingApplication": "http://localhost:1028/application" - } - } - ] -} - - -05. Discover with convop GET /v1/registry/contextEntityTypes/Room/attributes/temperature -======================================================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 288 - -{ - "contextRegistrationResponses": [ - { - "contextRegistration": { - "attributes": [ - { - "name": "temperature", - "type": "degree" - } - ], - "entities": [ - { - "id": "Room", - "isPattern": "false", - "type": "Room" - }, - { - "id": "OfficeRoom", - "isPattern": "false", - "type": "Room" - } - ], - "providingApplication": "http://localhost:1028/application" - } - } - ] -} - - -06. Discover with convop GET /v1/registry/contextEntities/type/Room/id/Room -=========================================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 234 - -{ - "contextRegistrationResponses": [ - { - "contextRegistration": { - "attributes": [ - { - "name": "temperature", - "type": "degree" - } - ], - "entities": [ - { - "id": "Room", - "isPattern": "false", - "type": "Room" - } - ], - "providingApplication": "http://localhost:1028/application" - } - } - ] -} - - -07. Discover with convop GET /v1/registry/contextEntities/type/Room/id/Room/attributes/temperature -================================================================================================== -HTTP/1.1 200 OK -Date: REGEX(.*) -Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Content-Type: application/json -Content-Length: 234 - -{ - "contextRegistrationResponses": [ - { - "contextRegistration": { - "attributes": [ - { - "name": "temperature", - "type": "degree" - } - ], - "entities": [ - { - "id": "Room", - "isPattern": "false", - "type": "Room" - } - ], - "providingApplication": "http://localhost:1028/application" - } - } - ] -} - - ---TEARDOWN-- -brokerStop CB -dbDrop CB diff --git a/test/functionalTest/cases/0968_ngsi10_asterisk_subs/0968_ngsi10_asterisk_subs.test.DISABLED b/test/functionalTest/cases/0968_invalid_regex_subs/0968_invalid_regex_subs.test similarity index 62% rename from test/functionalTest/cases/0968_ngsi10_asterisk_subs/0968_ngsi10_asterisk_subs.test.DISABLED rename to test/functionalTest/cases/0968_invalid_regex_subs/0968_invalid_regex_subs.test index 51d1ed2ff5..88b6207876 100644 --- a/test/functionalTest/cases/0968_ngsi10_asterisk_subs/0968_ngsi10_asterisk_subs.test.DISABLED +++ b/test/functionalTest/cases/0968_invalid_regex_subs/0968_invalid_regex_subs.test @@ -21,7 +21,7 @@ # VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh --NAME-- -NGSI10 subscription with invalid regex "*" +Subscription with invalid regex "*" --SHELL-INIT-- dbInit CB @@ -37,38 +37,21 @@ echo "01. Create subscription with invalid regex" echo "==========================================" payload=' { + "subject": { "entities": [ { "type": "Patient", - "isPattern": "true", - "id": "*" + "idPattern": "*" } - ], - "attributes": [ - "Blood glucose", - "Body temperature", - "Systolic blood pressure", - "Diastolic blood pressure", - "Respiration rate", - "Heart rate", - "Blood oxygen", - "timestamp" - ], - "reference": "http://A.B.C.D:5999/orion2mysql", - "duration": "P1Y", - "notifyConditions": [ - { - "type": "ONCHANGE", - "condValues": [ - "timestamp" - ] - } - ], - "throttling": "PT1S" -} -' - -orionCurl --url /v1/subscribeContext --payload "$payload" + ] + }, + "notification": { + "http": { + "url": "http://A.B.C.D:5999/orion2mysql" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" echo echo @@ -76,20 +59,15 @@ echo --REGEXPECT-- 01. Create subscription with invalid regex ========================================== -HTTP/1.1 200 OK +HTTP/1.1 400 Bad Request Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 141 +Content-Length: 73 { - "subscribeError": { - "errorCode": { - "code": "400", - "details": "invalid payload: invalid regex for entity id pattern", - "reasonPhrase": "Bad Request" - } - } + "description": "Invalid regex for entity idPattern", + "error": "BadRequest" } From c28b9464d840ca3db48ab8518056afdf38d57718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 5 Jun 2024 13:20:02 +0200 Subject: [PATCH 313/390] ADD jexl extra case --- .../jexl_transformation_extra_case.test | 310 ++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_extra_case.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_extra_case.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_extra_case.test new file mode 100644 index 0000000000..69a93fd03e --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_extra_case.test @@ -0,0 +1,310 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +JEXL basic expression in custom notification (extra case) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with jexl expression +# 02. Create entity E1 (inactive true, status noData, spots 2344) +# 03. Update entity E1 (inactive false, status noData, spots 2344) +# 04. Update entity E1 (inactive false, status ok, spots 2344) +# 05. Dump accumulator and see three expected notifications (-1, -2 and 2344) +# + + +echo "01. Create custom sub with several attributes with transformations" +echo "==================================================================" +# NOTE: '\'' is the way of scaping a ' in the payload variable below (see https://stackoverflow.com/a/1250279/1485926) +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "address": { + "type": "Text", + "value": "${address||'\'''\''}" + }, + "humidity": { + "type": "Float", + "value": "${humidity||0-1}" + }, + "free": { + "type": "Number", + "value": "${(inactive==true)?(0-1) : ((status == '\''noData'\'')?(0-2) : spots)}" + } + } + }, + "attrs": [ "address", "humidity", "free" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 (inactive true, status noData, spots 2344)" +echo "===============================================================" +payload='{ + "id": "E1", + "type": "T", + "spots": { + "type": "Number", + "value": 2344 + }, + "inactive": { + "type": "Boolean", + "value": true + }, + "status": { + "type": "Text", + "value": "noData" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 (inactive false, status noData, spots 2344)" +echo "================================================================" +payload='{ + "address": { + "type": "Text", + "value": "test5" + }, + "spots": { + "type": "Number", + "value": 2344 + }, + "inactive": { + "type": "Boolean", + "value": false + }, + "status": { + "type": "Text", + "value": "noData" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "04. Update entity E1 (inactive true, status ok, spots 2344)" +echo "===========================================================" +payload='{ + "humidity": { + "type": "Number", + "value": 3 + }, + "spots": { + "type": "Number", + "value": 2344 + }, + "inactive": { + "type": "Boolean", + "value": false + }, + "status": { + "type": "Text", + "value": "ok" + } +}' +orionCurl --url /v2/entities/E1/attrs -X POST --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three expected notifications (-1, -2 and 2344)" +echo "===========================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with several attributes with transformations +================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 (inactive true, status noData, spots 2344) +=============================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 (inactive false, status noData, spots 2344) +================================================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Update entity E1 (inactive true, status ok, spots 2344) +=========================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see three expected notifications (-1, -2 and 2344) +=========================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 231 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "address": { + "metadata": {}, + "type": "Text", + "value": "" + }, + "free": { + "metadata": {}, + "type": "Number", + "value": -1 + }, + "humidity": { + "metadata": {}, + "type": "Float", + "value": -1 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 236 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "address": { + "metadata": {}, + "type": "Text", + "value": "test5" + }, + "free": { + "metadata": {}, + "type": "Number", + "value": -2 + }, + "humidity": { + "metadata": {}, + "type": "Float", + "value": -1 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 237 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "address": { + "metadata": {}, + "type": "Text", + "value": "test5" + }, + "free": { + "metadata": {}, + "type": "Number", + "value": 2344 + }, + "humidity": { + "metadata": {}, + "type": "Float", + "value": 3 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 5e09b00fdd9df76b1f1e59bfdd7ded575c0a0574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 5 Jun 2024 13:21:14 +0200 Subject: [PATCH 314/390] FIX docu --- doc/manuals/admin/logs.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index bf711aba94..ae145c427c 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -406,13 +406,6 @@ time=2020-10-26T15:06:14.642Z | lvl=INFO | corr=c4a3192e-179c-11eb-ac8f-000c29df If `-logDeprecate` CLI setting is used (or `deprecate` parameter in the [log admin REST API](management_api.md#log-configs-and-trace-levels)) the following WARN traces are generated: -* NGSIv1 requests (both with and without payload). For instance: - -``` -time=2023-05-25T14:27:45.958Z | lvl=WARN | corr=513bd10e-fb08-11ed-8ad7-000c29583ca5 | trans=1685024865-125-00000000001 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[171]:logInfoRequestWithPayload | msg=Deprecated NGSIv1 request received: POST /v1/queryContext, request payload (48 bytes): { "entities": [ { "type": "T1", "id": "E1" } ] }, response code: 200 -time=2023-05-25T14:27:46.041Z | lvl=WARN | corr=51490536-fb08-11ed-9782-000c29583ca5 | trans=1685024865-125-00000000002 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[114]:logInfoRequestWithoutPayload | msg=Deprecated NGSIv1 request received: GET /v1/contextEntities/E, response code: 200 -``` - * Usages of [`"legacyForwarding": true`](../orion-api.md#registrationprovider)). For instance: ``` From 1edaab51a33a77388048c47625b9681c5ab457fd Mon Sep 17 00:00:00 2001 From: mapedraza <40356341+mapedraza@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:33:18 +0200 Subject: [PATCH 315/390] add-attrname-jexl-limitation --- doc/manuals/orion-api.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 3b2951b08c..493c5ce72a 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -3097,6 +3097,7 @@ As failsafe behaviour, evaluation returns `null` in the following cases: - The unitary minus operator is not working properly, e.g. the following expression doesn't work (it failsafes to `null`): `A||-1`. However, the following alternatives are working: `A||0-1` and `A||'-1'|parseInt)` - Negation operator `!` (supported in original JavaScript JEXL) is not supported +- Attributes names using JEXL operators (I.E: `:` or `-`) may lead to undesired expression results. For example, an atribute named `temperature-1`, when you use it as part of an expression, it would be the result uf substracting 1 to the value of the attribute `temperature`, if it is defined, or `null`, if the attribute `temperature` doesn't exist. ## Oneshot Subscriptions From be1961ea7612317d6e7c3399671da5c1f723e42c Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Wed, 5 Jun 2024 20:36:12 +0900 Subject: [PATCH 316/390] (JP) REMOVE documentation about NGSIv1 lasting API ops (#4563) --- doc/manuals.jp/admin/logs.md | 7 ------- doc/manuals.jp/deprecated.md | 1 + 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/doc/manuals.jp/admin/logs.md b/doc/manuals.jp/admin/logs.md index 7c06f2dd88..37962c15a4 100644 --- a/doc/manuals.jp/admin/logs.md +++ b/doc/manuals.jp/admin/logs.md @@ -346,13 +346,6 @@ time=2020-10-26T15:06:14.642Z | lvl=INFO | corr=c4a3192e-179c-11eb-ac8f-000c29df `-logDeprecate` CLI 設定 (または [ログ管理 REST API](management_api.md#log-configs-and-trace-levels) の `deprecate` パラメータ) が使用されている場合、次の WARN トレースが生成されます: -* NGSIv1 リクエスト (ペイロードありとペイロードなしの両方)。例えば： - -``` -time=2023-05-25T14:27:45.958Z | lvl=WARN | corr=513bd10e-fb08-11ed-8ad7-000c29583ca5 | trans=1685024865-125-00000000001 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[171]:logInfoRequestWithPayload | msg=Deprecated NGSIv1 request received: POST /v1/queryContext, request payload (48 bytes): { "entities": [ { "type": "T1", "id": "E1" } ] }, response code: 200 -time=2023-05-25T14:27:46.041Z | lvl=WARN | corr=51490536-fb08-11ed-9782-000c29583ca5 | trans=1685024865-125-00000000002 | from=127.0.0.1 | srv=s1 | subsrv=/A | comp=Orion | op=logTracing.cpp[114]:logInfoRequestWithoutPayload | msg=Deprecated NGSIv1 request received: GET /v1/contextEntities/E, response code: 200 -``` - * [`"legacyForwarding": true`](../orion-api.md#registrationprovider)) の使用。例えば： ``` diff --git a/doc/manuals.jp/deprecated.md b/doc/manuals.jp/deprecated.md index d22e8711a1..0ad2f73685 100644 --- a/doc/manuals.jp/deprecated.md +++ b/doc/manuals.jp/deprecated.md @@ -29,6 +29,7 @@ * `POST /v1/queryContext` * `POST /NGSI10/queryContext` * サブスクリプション通知の NGSIv1 形式 (`notification.atttrsFormat` が `legacy` に設定) は Orion 4.0.0 で削除されました + * 最後に、Orion 4.0.0 では、残りの NGSIv1 操作が削除されました。 * `POST /v2/op/query` の `attributes` フィールドは、Orion 1.15.0 にあります。これらの属性を持つエンティティのみを返すためには、クエリに対するレスポンスにどの属性を含めるかを選択する `attrs` と、`expression` 内の `q` の単項属性フィルタ (unary attribute filter) の組み合わせです。それらを代わりに指定していください * Orion 1.14.0 では `POST /v2/op/update` の `APPEND`, `APPEND_STRICT`, `UPDATE`, `DELETE`, `REPLACE` の使用は非推奨です。`append`, `appendStrict`, `update`, `delete`, `replace` を代わりに使ってください * Orion 1.13.0 ではメタデータ ID が推奨されていません (Orion 2.2.0 で削除されました)。一方、この機能は NGSIv2 と互換性がありません。JSON 表現形式の属性名は JSON オブジェクトのキーとして使用されるため、名前を複製することはできません。一方、IDs は、属性名にプレフィックス/サフィックスを使用して簡単に実装することができます。たとえば、`temperature:ground` および `temperature:ceiling` です。 この非推奨の結果、次のオペレーションも非推奨になりました : From c10e83c2e4c7f9a798f0afee599cfebe555f13d8 Mon Sep 17 00:00:00 2001 From: mapedraza <40356341+mapedraza@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:47:25 +0200 Subject: [PATCH 317/390] Update doc/manuals/orion-api.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 493c5ce72a..1bcf31e589 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -3097,7 +3097,7 @@ As failsafe behaviour, evaluation returns `null` in the following cases: - The unitary minus operator is not working properly, e.g. the following expression doesn't work (it failsafes to `null`): `A||-1`. However, the following alternatives are working: `A||0-1` and `A||'-1'|parseInt)` - Negation operator `!` (supported in original JavaScript JEXL) is not supported -- Attributes names using JEXL operators (I.E: `:` or `-`) may lead to undesired expression results. For example, an atribute named `temperature-1`, when you use it as part of an expression, it would be the result uf substracting 1 to the value of the attribute `temperature`, if it is defined, or `null`, if the attribute `temperature` doesn't exist. +- Attribute names and entity identifiers using JEXL operators (eg. `:` or `-`) may lead to undesired expression results. For example, an attribute named `temperature-basic`, when you use it as part of an expression, it would be the result of subtracting the value of the attribute `basic` to the value of the attribute `temperature` (or `null`, if `temperature` or `basic` don't exist) instead the value of the attribute `temperature-basic`. ## Oneshot Subscriptions From a415a7fd6ddc3e3ff4ae8e531e7425108a0327e7 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Wed, 5 Jun 2024 20:56:28 +0900 Subject: [PATCH 318/390] (JP) [Doc] Add know JEXL limitation based on attributes names (#4571) --- doc/manuals.jp/orion-api.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index a4891cb925..6a4ff60d45 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -3168,6 +3168,7 @@ c|arrAvg - 単位マイナス演算子が正しく動作しません。たとえば、次の式は動作しません (`null` にフェイルセーフされます): `A||-1`。ただし、次の代替式は動作します: `A||0-1` および `A||'-1'|parseInt)` - 否定演算子 `!` (元の JavaScript JEXL でサポート) はサポートされていません +- JEXL 演算子 (例: `:` または `-`) を使用する属性名とエンティティ識別子は、望ましくない式結果につながる可能性があります。たとえば、`temperature-basic` という名前の属性を式の一部として使用すると、属性 `temperature-basic` の値ではなく、属性 `basic` の値から属性 `temperature` の値 (または、`temperature` または `basic` が存在しない場合は `null`) を減算した結果になります From 759502d38378ae6792b43d8ba9291f9230156ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 5 Jun 2024 14:10:42 +0200 Subject: [PATCH 319/390] FIX typos in ftest --- .../jexl_transformation_binary_map_accum.test | 11 +++++------ .../jexl_transformation_multiple.test | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test index baa991c929..b4a808b868 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_binary_map_accum.test @@ -35,7 +35,7 @@ accumulatorStart --pretty-print # 01. Create custom sub with several attributes with transformations # 02. Create entity E1 # 03. Update entity E1 -# 05. Dump accumulator and see two expected transformations +# 05. Dump accumulator and see two expected notifications # @@ -72,7 +72,6 @@ orionCurl --url /v2/subscriptions --payload "$payload" echo echo -#"value": "${{count:count.count+1,sum:count.sum+speed|split('\'' '\'')[0]|parseInt}}", echo "02. Create entity E1" echo "====================" @@ -114,8 +113,8 @@ echo echo -echo "05. Dump accumulator and see two expected transformations" -echo "=========================================================" +echo "05. Dump accumulator and see two expected notifications" +echo "=======================================================" accumulatorDump echo echo @@ -150,8 +149,8 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -05. Dump accumulator and see two expected transformations -========================================================= +05. Dump accumulator and see two expected notifications +======================================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / Content-Length: 187 diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test index ec3f7f0fff..872195dccb 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_multiple.test @@ -84,7 +84,6 @@ orionCurl --url /v2/subscriptions --payload "$payload" echo echo -#"value": "${{count:count.count+1,sum:count.sum+speed|split('\'' '\'')[0]|parseInt}}", echo "02. Create entity E1" echo "====================" From 4f9f34df07395c54387a53074f98bef00b1130a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 6 Jun 2024 09:30:18 +0200 Subject: [PATCH 320/390] Step: 3.12.0-next -> 4.0.0 --- CHANGES_NEXT_RELEASE | 17 ----------------- Changelog | 20 ++++++++++++++++++++ README.md | 2 +- src/app/contextBroker/version.h | 2 +- src/lib/common/defaultValues.h | 2 +- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index fe1b2f1c22..e69de29bb2 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,17 +0,0 @@ -- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.3.0) (#4004) -- Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) -- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004) -- Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) -- Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) -- Fix: lighter operation to get databases list from MongoDB (#4517) -- Hardening: compile code using C++14 standard -- Remove: `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` CLI parameters along with associated env vars, already deprecated in Orion 3.12.0 (use `-dbURI` instead`) -- Remove: legacy subscription format -- Remove: all the remaining NGSIv1 operations - - PUT /v1/contextEntities/{id} - - DELETE /v1/contextEntities/{id} - - GET /v1/contextEntities/{id}/attributes/{name} - - POST /v1/updateContext - - POST /NGSI10/updateContext - - POST /v1/queryContext - - POST /NGSI10/queryContext diff --git a/Changelog b/Changelog index 91315cc289..5b5d5ef154 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,23 @@ +4.0.0 (June 6th, 2024) + +- Add: JEXL expression support in custom notification macro replacement (using cjexl 0.3.0) (#4004) +- Add: expression context build and evaluation counters in timing section in GET /statistics (#4004) +- Fix: use null for non existing attributes in custom covered notifications macro substitution (instead of empty string) to make behaviour more consistent (#4004) +- Fix: use null for non existing attributes in macro substitution applied to "paylaod" field (instead of empty string) to make behaviour more consistent (#4004) +- Fix: simplified GET /version operation, without including "libversions" field (add ?options=libVersions to get it) +- Fix: lighter operation to get databases list from MongoDB (#4517) +- Hardening: compile code using C++14 standard +- Remove: `-dbhost`, `-rplSet`, `-dbTimeout`, `-dbuser`, `-dbAuthMech`, `-dbAuthDb`, `-dbSSL` and `-dbDisableRetryWrites` CLI parameters along with associated env vars, already deprecated in Orion 3.12.0 (use `-dbURI` instead`) +- Remove: legacy subscription format +- Remove: all the remaining NGSIv1 operations + - PUT /v1/contextEntities/{id} + - DELETE /v1/contextEntities/{id} + - GET /v1/contextEntities/{id}/attributes/{name} + - POST /v1/updateContext + - POST /NGSI10/updateContext + - POST /v1/queryContext + - POST /NGSI10/queryContext + 3.12.0 (February 29th, 2024) - Fix: service path levels with 0 length should not be allowed (#4495) diff --git a/README.md b/README.md index c3a2834151..29a2bf4c13 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![Support badge](https://img.shields.io/badge/tag-fiware--orion-orange.svg?logo=stackoverflow)](http://stackoverflow.com/questions/tagged/fiware-orion) [![NGSI v2](https://img.shields.io/badge/NGSI-V2-red.svg)](doc/manuals/orion-api.md)
-[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion.svg)](https://fiware-orion.rtfd.io) +[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion/4.0.0.svg)](https://fiware-orion.rtfd.io/en/4.0.0/) ![Compliance Tests](https://github.com/telefonicaid/fiware-orion/workflows/Compliance%20Tests/badge.svg) ![Unit Tests](https://github.com/telefonicaid/fiware-orion/workflows/Unit%20Tests/badge.svg) ![Functional Tests](https://github.com/telefonicaid/fiware-orion/workflows/Functional%20Tests/badge.svg) diff --git a/src/app/contextBroker/version.h b/src/app/contextBroker/version.h index 26b18df3d6..417657d705 100644 --- a/src/app/contextBroker/version.h +++ b/src/app/contextBroker/version.h @@ -28,6 +28,6 @@ -#define ORION_VERSION "3.12.0-next" +#define ORION_VERSION "4.0.0" #endif // SRC_APP_CONTEXTBROKER_VERSION_H_ diff --git a/src/lib/common/defaultValues.h b/src/lib/common/defaultValues.h index 95998c82df..6b5cc76424 100644 --- a/src/lib/common/defaultValues.h +++ b/src/lib/common/defaultValues.h @@ -41,6 +41,6 @@ * * API Documentation - The link to the the GEri documentation, either in the gh-pages (.github.io/) inside the fiware organization in GitHub or ReadTheDocs manual. */ -#define API_DOC "https://fiware-orion.rtfd.io/" +#define API_DOC "https://fiware-orion.rtfd.io/en/4.0.0/" #endif // SRC_LIB_COMMON_DEFAULTVALUES_H_ From 82f28d503249312d6ac8adee8669842baca4da4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 6 Jun 2024 10:25:09 +0200 Subject: [PATCH 321/390] FIX notification_different_sizes.test --- .../0000_large_requests/notification_different_sizes.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functionalTest/cases/0000_large_requests/notification_different_sizes.test b/test/functionalTest/cases/0000_large_requests/notification_different_sizes.test index 3c50b6e889..132c36364c 100644 --- a/test/functionalTest/cases/0000_large_requests/notification_different_sizes.test +++ b/test/functionalTest/cases/0000_large_requests/notification_different_sizes.test @@ -294,9 +294,9 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) 08. Check notification sizes ============================ -Sending message 1 to HTTP server: sending message of REGEX((19400|19401|19405|19406)) bytes to HTTP server -Sending message 2 to HTTP server: sending message of REGEX((21400|21401|21405|21406)) bytes to HTTP server -Sending message 3 to HTTP server: sending message of REGEX((8100402|8100403|8100407|8100408)) bytes to HTTP server +Sending message 1 to HTTP server: sending message of REGEX((19394|19399|19395|19400)) bytes to HTTP server +Sending message 2 to HTTP server: sending message of REGEX((21394|21399|21395|21400)) bytes to HTTP server +Sending message 3 to HTTP server: sending message of REGEX((8100396|8100401|8100397|8100402)) bytes to HTTP server --TEARDOWN-- From 47cefaaf38f5fd4a70a1fef994ae566d0c7c9c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 6 Jun 2024 11:40:20 +0200 Subject: [PATCH 322/390] Step: 4.0.0 -> 4.0.0-next --- README.md | 2 +- src/app/contextBroker/version.h | 2 +- src/lib/common/defaultValues.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 29a2bf4c13..c3a2834151 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![Support badge](https://img.shields.io/badge/tag-fiware--orion-orange.svg?logo=stackoverflow)](http://stackoverflow.com/questions/tagged/fiware-orion) [![NGSI v2](https://img.shields.io/badge/NGSI-V2-red.svg)](doc/manuals/orion-api.md)
-[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion/4.0.0.svg)](https://fiware-orion.rtfd.io/en/4.0.0/) +[![Documentation badge](https://img.shields.io/readthedocs/fiware-orion.svg)](https://fiware-orion.rtfd.io) ![Compliance Tests](https://github.com/telefonicaid/fiware-orion/workflows/Compliance%20Tests/badge.svg) ![Unit Tests](https://github.com/telefonicaid/fiware-orion/workflows/Unit%20Tests/badge.svg) ![Functional Tests](https://github.com/telefonicaid/fiware-orion/workflows/Functional%20Tests/badge.svg) diff --git a/src/app/contextBroker/version.h b/src/app/contextBroker/version.h index 417657d705..fd94430114 100644 --- a/src/app/contextBroker/version.h +++ b/src/app/contextBroker/version.h @@ -28,6 +28,6 @@ -#define ORION_VERSION "4.0.0" +#define ORION_VERSION "4.0.0-next" #endif // SRC_APP_CONTEXTBROKER_VERSION_H_ diff --git a/src/lib/common/defaultValues.h b/src/lib/common/defaultValues.h index 6b5cc76424..95998c82df 100644 --- a/src/lib/common/defaultValues.h +++ b/src/lib/common/defaultValues.h @@ -41,6 +41,6 @@ * * API Documentation - The link to the the GEri documentation, either in the gh-pages (.github.io/) inside the fiware organization in GitHub or ReadTheDocs manual. */ -#define API_DOC "https://fiware-orion.rtfd.io/en/4.0.0/" +#define API_DOC "https://fiware-orion.rtfd.io/" #endif // SRC_LIB_COMMON_DEFAULTVALUES_H_ From 1a824bf514b1b75fb43b7320ba893211bfc4666c Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Thu, 6 Jun 2024 17:44:16 +0530 Subject: [PATCH 323/390] updated code and FT for correct no of days in a month --- src/lib/common/globals.cpp | 48 ++- .../wrong_number_of_days_in_day_field.test | 392 ++++++++++++++++++ 2 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_number_of_days_in_day_field.test diff --git a/src/lib/common/globals.cpp b/src/lib/common/globals.cpp index 0dd8b886d1..3bdbb39a93 100644 --- a/src/lib/common/globals.cpp +++ b/src/lib/common/globals.cpp @@ -534,6 +534,52 @@ static int timezoneOffset(const char* tz) +/***************************************************************************** +* +* isLeapYear - +* +* This code will check, if the given year is a leap year or not. +* +*/ +bool isLeapYear(int year) +{ + if (year % 4 != 0) + { + return false; + } + if (year % 100 == 0 && year % 400 != 0) + { + return false; + } + return true; +} + + + +/***************************************************************************** +* +* daysInMonth - +* +* This code will check correct number of days in given month. +* +*/ +int daysInMonth(int year, int month) +{ + if (month == 1) //february + { + return isLeapYear(year) ? 29 : 28; + } + // for April, June, September, November + if (month == 3 || month == 5 || month == 8 || month == 10) + { + return 30; + } + // For all other months (Jan, March, May, July, Aug, October, December) + return 31; +} + + + /***************************************************************************** * * parse8601Time - @@ -610,7 +656,7 @@ double parse8601Time(const std::string& ss) const int minMonth = 0; const int maxMonth = 11; const int minDay = 1; - const int maxDay = 31; + const int maxDay = daysInMonth(y, time.tm_mon);; const int minHour = 0; const int maxHour = 23; const int minMinute = 0; diff --git a/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_number_of_days_in_day_field.test b/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_number_of_days_in_day_field.test new file mode 100644 index 0000000000..cc73b6296b --- /dev/null +++ b/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_number_of_days_in_day_field.test @@ -0,0 +1,392 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Check number of days in 'days' field for a month + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create a subscription with the wrong days value for 'Jan' month, see error +# 02. Create a subscription with the valid days value for 'Jan' month, (success case) +# 03. Create a subscription with the wrong days value for 'Feb' month (Non-leap year), see error +# 04. Create a subscription with the valid days value for 'Feb' month (Non-leap year), (success case) +# 05. Create a subscription with the wrong days value for 'Feb' month (Leap Year), see error +# 06. Create a subscription with the valid days value for 'Feb' month (Leap-Year), (success case) +# 07. Create a subscription with the wrong days value for 'June' month, see error +# 08. Create a subscription with the valid days value for 'June' month, (success case) +# + + +echo "01. Create a subscription with the wrong days value for 'Jan' month, see error" +echo "==============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2022-01-32T17:16:03.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create a subscription with the valid days value for 'Jan' month, (success case)" +echo "===================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2022-01-31T17:15:03.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create a subscription with the wrong days value for 'Feb' month (Non-leap year), see error" +echo "==============================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2024-02-29T25:16:11.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "04. Create a subscription with the valid days value for 'Feb' month (Non-leap year), (success case)" +echo "===================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2022-02-28T17:16:23.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create a subscription with the wrong days value for 'Feb' month (Leap Year), see error" +echo "==========================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2024-02-30T17:16:43.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "06. Create a subscription with the valid days value for 'Feb' month (Leap-Year), (success case)" +echo "===============================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2024-02-29T17:16:23.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "07. Create a subscription with the wrong days value for 'June' month, see error" +echo "===============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2025-06-31T23:59:59.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "08. Create a subscription with the valid days value for 'June' month, (success case)" +echo "====================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2025-06-30T23:59:59.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. Create a subscription with the wrong days value for 'Jan' month, see error +============================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +02. Create a subscription with the valid days value for 'Jan' month, (success case) +=================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Create a subscription with the wrong days value for 'Feb' month (Non-leap year), see error +============================================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +04. Create a subscription with the valid days value for 'Feb' month (Non-leap year), (success case) +=================================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create a subscription with the wrong days value for 'Feb' month (Leap Year), see error +========================================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +06. Create a subscription with the valid days value for 'Feb' month (Leap-Year), (success case) +=============================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +07. Create a subscription with the wrong days value for 'June' month, see error +=============================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +08. Create a subscription with the valid days value for 'June' month, (success case) +==================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB + From 45d4f78e6a9b024e6f5285559f742c8e123e653c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 6 Jun 2024 16:03:31 +0200 Subject: [PATCH 324/390] FIX set basic expr in utest --- test/unittests/common/commonMacroSubstitute_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unittests/common/commonMacroSubstitute_test.cpp b/test/unittests/common/commonMacroSubstitute_test.cpp index be51a1e8af..90011a3708 100644 --- a/test/unittests/common/commonMacroSubstitute_test.cpp +++ b/test/unittests/common/commonMacroSubstitute_test.cpp @@ -49,7 +49,7 @@ TEST(commonMacroSubstitute, simple) const char* correct = "Entity E1/T1, attribute 'attr1'"; std::string result; - ExprContextObject exprContext; + ExprContextObject exprContext(true); exprContext.add("id", en.id); exprContext.add("type", en.type); exprContext.add(caP->name, caP->stringValue); @@ -91,7 +91,7 @@ TEST(commonMacroSubstitute, withRealloc) std::string correct = std::string(base) + "Now, finally something to substitute: Entity E1/T1, attribute 'attr1'"; std::string result; - ExprContextObject exprContext; + ExprContextObject exprContext(true); exprContext.add("id", en.id); exprContext.add("type", en.type); exprContext.add(caP->name, caP->stringValue); @@ -167,7 +167,7 @@ TEST(commonMacroSubstitute, bufferTooBigAfterSubstitution) // correct = std::string(base) + "EntityId000001/EntityType000001"; // > 8MB after substitutions std::string result; - ExprContextObject exprContext; + ExprContextObject exprContext(true); exprContext.add("id", en.id); exprContext.add("type", en.type); exprContext.add(caP->name, caP->stringValue); From d2e95bc86d1155c87e9435c5beeac7d8d22dc64b Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Fri, 7 Jun 2024 16:45:44 +0530 Subject: [PATCH 325/390] updated code as per comment --- src/lib/common/globals.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/common/globals.cpp b/src/lib/common/globals.cpp index 3bdbb39a93..2bc490b004 100644 --- a/src/lib/common/globals.cpp +++ b/src/lib/common/globals.cpp @@ -543,15 +543,16 @@ static int timezoneOffset(const char* tz) */ bool isLeapYear(int year) { - if (year % 4 != 0) + // ref: https://www.geeksforgeeks.org/program-check-given-year-leap-year/ + if (year % 400 == 0) { - return false; + return true; } - if (year % 100 == 0 && year % 400 != 0) + if ((year % 4 == 0) && (year % 100 != 0)) { - return false; + return true; } - return true; + return false; } From 3d2a20ea930247d454c288ce983fdab791fea6bf Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Fri, 7 Jun 2024 18:02:44 +0530 Subject: [PATCH 326/390] updated FT typo --- .../wrong_date_values_in_expires_field.test | 2 +- .../wrong_number_of_days_in_day_field.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_date_values_in_expires_field.test b/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_date_values_in_expires_field.test index a90e17a916..0c566d00e3 100644 --- a/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_date_values_in_expires_field.test +++ b/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_date_values_in_expires_field.test @@ -219,7 +219,7 @@ payload='{ "pressure" ] }, - "expires": "1822-12-21T17:16:23.00Z" + "expires": "1899-12-21T17:16:23.00Z" }' orionCurl --url /v2/subscriptions --payload "$payload" echo diff --git a/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_number_of_days_in_day_field.test b/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_number_of_days_in_day_field.test index cc73b6296b..93656da64e 100644 --- a/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_number_of_days_in_day_field.test +++ b/test/functionalTest/cases/4541_wrong_date_values_in_subscription/wrong_number_of_days_in_day_field.test @@ -127,7 +127,7 @@ payload='{ "pressure" ] }, - "expires": "2024-02-29T25:16:11.00Z" + "expires": "2100-02-29T12:16:11.00Z" }' orionCurl --url /v2/subscriptions --payload "$payload" echo From 8af83f615cbbe6d110b7792a54dbbb2b79dc380d Mon Sep 17 00:00:00 2001 From: mapedraza <40356341+mapedraza@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:50:04 +0200 Subject: [PATCH 327/390] Update roadmap.md --- doc/roadmap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/roadmap.md b/doc/roadmap.md index f2b51bfa69..2fb7f73951 100644 --- a/doc/roadmap.md +++ b/doc/roadmap.md @@ -32,7 +32,6 @@ Disclaimer: The following list of features are planned to be addressed in the short term, and incorporated into the coming release(s) of the product: -- Expression language support (JEXL) ([#4004](https://github.com/telefonicaid/fiware-orion/issues/4004)) - Dynamic / high order attribute values (e.g. an attribute being a sum of two other attributes) based on subscriptions to do internal modifications to CB entities ([#3815](https://github.com/telefonicaid/fiware-orion/issues/3815), [#4513](https://github.com/telefonicaid/fiware-orion/issues/4513)) - MQTT notification retrial ([#4439](https://github.com/telefonicaid/fiware-orion/issues/4439)) @@ -74,6 +73,7 @@ you wish to get involved in the implementation or influence the roadmap The following list contains all features that were in the roadmap and have already been implemented. +- Expression language support (JEXL) ([#4004](https://github.com/telefonicaid/fiware-orion/issues/4004)) ([4.0.0](https://github.com/telefonicaid/fiware-orion/releases/tag/4.0.0)) - MQTT Retain flag ([#4388](https://github.com/telefonicaid/fiware-orion/issues/4388)) ([3.11.0](https://github.com/telefonicaid/fiware-orion/releases/tag/3.11.0)) - Custom notifications: simplifying sending JSON requests ([#2560](https://github.com/telefonicaid/fiware-orion/issues/2560)) ([3.8.0](https://github.com/telefonicaid/fiware-orion/releases/tag/3.8.0)) - New subscripition modes (create only, update only, delete only and combinations) ([#1494](https://github.com/telefonicaid/fiware-orion/issues/1494)) ([3.7.0](https://github.com/telefonicaid/fiware-orion/releases/tag/3.7.0)) From bf84f6922edf100bdd2288c32edcded1316c658d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 11 Jun 2024 09:18:21 +0200 Subject: [PATCH 328/390] FIX do not raise DB alarm in case of wrong GeoJSON --- CHANGES_NEXT_RELEASE | 3 +- src/lib/mongoBackend/MongoCommonUpdate.cpp | 1 + src/lib/mongoBackend/MongoCommonUpdate.h | 3 -- src/lib/mongoDriver/CMakeLists.txt | 1 + src/lib/mongoDriver/DBErrors.h | 34 +++++++++++++++++++ src/lib/mongoDriver/connectionOperations.cpp | 8 ++++- .../createEntity_with_wrong_GeoJSON.test | 12 +++++++ 7 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 src/lib/mongoDriver/DBErrors.h diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 033c046882..4c5c912cb9 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ -- Fix: wrong date values should not allowed in subscription's expires field (#4541) \ No newline at end of file +- Fix: wrong date values should not allowed in subscription's expires field (#4541) +- Fix: do not raise DB alarm in case of wrong GeoJSON in client request \ No newline at end of file diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 03617d9d50..3e5b3b28ff 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -64,6 +64,7 @@ #include "mongoDriver/safeMongo.h" #include "mongoDriver/BSONObjBuilder.h" #include "mongoDriver/BSONArrayBuilder.h" +#include "mongoDriver/DBErrors.h" /* **************************************************************************** diff --git a/src/lib/mongoBackend/MongoCommonUpdate.h b/src/lib/mongoBackend/MongoCommonUpdate.h index 3873b7a4f8..a3d4266dcc 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.h +++ b/src/lib/mongoBackend/MongoCommonUpdate.h @@ -32,9 +32,6 @@ #include "orionTypes/UpdateActionType.h" #include "ngsi10/UpdateContextResponse.h" -#define MONGODB_ERROR_DUPLICATE_KEY "duplicate key" -#define MONGODB_ERROR_WRONGJSON "Can't extract geo keys" - /* **************************************************************************** diff --git a/src/lib/mongoDriver/CMakeLists.txt b/src/lib/mongoDriver/CMakeLists.txt index e5153dd283..dc23230ab8 100644 --- a/src/lib/mongoDriver/CMakeLists.txt +++ b/src/lib/mongoDriver/CMakeLists.txt @@ -47,6 +47,7 @@ SET (HEADERS OID.h DBConnection.h DBCursor.h + DBErrors.h safeMongo.h connectionOperations.h mongoConnectionPool.h diff --git a/src/lib/mongoDriver/DBErrors.h b/src/lib/mongoDriver/DBErrors.h new file mode 100644 index 0000000000..b184d13317 --- /dev/null +++ b/src/lib/mongoDriver/DBErrors.h @@ -0,0 +1,34 @@ +#ifndef SRC_LIB_MONGODRIVER_DBERRORS_H_ +#define SRC_LIB_MONGODRIVER_DBERRORS_H_ + +/* +* +* Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +* +* This file is part of Orion Context Broker. +* +* Orion Context Broker is free software: you can redistribute it and/or +* modify it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* Orion Context Broker is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +* General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +* +* For those usages not covered by this license please contact with +* iot_support at tid dot es +* +* Author: Fermín Galán +*/ + + +#define MONGODB_ERROR_DUPLICATE_KEY "duplicate key" +#define MONGODB_ERROR_WRONGJSON "Can't extract geo keys" + + +#endif // SRC_LIB_MONGODRIVER_DBERRORS_H_ \ No newline at end of file diff --git a/src/lib/mongoDriver/connectionOperations.cpp b/src/lib/mongoDriver/connectionOperations.cpp index d0f5ca0b9f..0499f5114c 100644 --- a/src/lib/mongoDriver/connectionOperations.cpp +++ b/src/lib/mongoDriver/connectionOperations.cpp @@ -37,6 +37,7 @@ #include "mongoDriver/BSONArrayBuilder.h" #include "mongoDriver/mongoConnectionPool.h" #include "mongoDriver/safeMongo.h" +#include "mongoDriver/DBErrors.h" @@ -507,7 +508,12 @@ bool orion::collectionInsert " - exception: " + error.message; *err = "Database Error (" + msg + ")"; - alarmMgr.dbError(msg); + + // MONGODB_ERROR_WRONGJSON doesn't raise alarm (responses 400 Bad Request in this situation) + if (!(msg.find(MONGODB_ERROR_WRONGJSON) != std::string::npos)) + { + alarmMgr.dbError(msg); + } } bson_free(bsonStr); diff --git a/test/functionalTest/cases/3347_createEntity_with_wrong_GeoJSON/createEntity_with_wrong_GeoJSON.test b/test/functionalTest/cases/3347_createEntity_with_wrong_GeoJSON/createEntity_with_wrong_GeoJSON.test index 2a6f4ab382..08af1386c5 100644 --- a/test/functionalTest/cases/3347_createEntity_with_wrong_GeoJSON/createEntity_with_wrong_GeoJSON.test +++ b/test/functionalTest/cases/3347_createEntity_with_wrong_GeoJSON/createEntity_with_wrong_GeoJSON.test @@ -34,6 +34,7 @@ brokerStart CB 0 # 02. GET /v2/entities/K # 03. Create entity with WRONG GeoJSON1 # 04. Create entity with WRONG GeoJSON2 +# 05. Ensure no DatabaseError alarm is raised # echo "01. Create entity with correct GeoJSON" @@ -102,6 +103,12 @@ orionCurl --url '/v2/entities' --payload "${payload}" echo echo +echo "05. Ensure no DatabaseError alarm is raised" +echo "===========================================" +cat /tmp/contextBroker.log | grep DatabaseError | wc -l +echo +echo + --REGEXPECT-- 01. Create entity with correct GeoJSON @@ -167,6 +174,11 @@ Content-Length: 52 } +05. Ensure no DatabaseError alarm is raised +=========================================== +0 + + --TEARDOWN-- brokerStop CB dbDrop CB From 9dc6d38b428f18479b78b0a2e0905af5f2860e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 11 Jun 2024 15:47:39 +0200 Subject: [PATCH 329/390] FIX enable statistics_with_counters.test --- ...DISABLED => statistics_with_counters.test} | 80 ++----------------- 1 file changed, 7 insertions(+), 73 deletions(-) rename test/functionalTest/cases/0000_statistics_operation/{statistics_with_counters.test.DISABLED => statistics_with_counters.test} (83%) diff --git a/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test.DISABLED b/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test similarity index 83% rename from test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test.DISABLED rename to test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test index 26f899f85f..e184f39215 100644 --- a/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test.DISABLED +++ b/test/functionalTest/cases/0000_statistics_operation/statistics_with_counters.test @@ -44,17 +44,15 @@ brokerStart CB 0 IPv4 -statCounters # # All counters has 1 hit. # -# As a kind of checksum note there are 60 orionCurl ocurrences in this file -# (it can be checked with grep) and that: +# As a kind of checksum note there are 53 orionCurl ocurrences in this file +# (it can be checked with grep orionCurl | grep '\-\-url' | wc -l) and that: # -# - jsonRequests: 16 -# - noPayloadRequests: 42 +# - jsonRequests: 11 +# - noPayloadRequests: 40 # - textRequest: 1 # - (comented orionCurl) 1 # -# which sum 131 - -# Not legacy +# which sum 53 orionCurl --url /admin/log > /dev/null @@ -207,46 +205,6 @@ orionCurl --url /v2/types/T -X OPTIONS > /dev/null orionCurl --url /version > /dev/null -# Legacy - -payload='{}' -orionCurl --url /v1/contextEntities/E -X DELETE > /dev/null -orionCurl --url /v1/contextEntities/E -X PUT --payload "$payload" > /dev/null - -orionCurl --url /v1/contextEntities/E/attributes/A > /dev/null - -payload='{ - "entities": [ - { - "type": "T", - "isPattern": "false", - "id": "E" - } - ] -}' -orionCurl --url /ngsi10/queryContext -X POST --payload "$payload" > /dev/null -orionCurl --url /v1/queryContext -X POST --payload "$payload" > /dev/null - -payload='{ - "contextElements": [ - { - "type": "T", - "isPattern": "false", - "id": "E", - "attributes": [ - { - "name": "A", - "type": "Text", - "value": "foo" - } - ] - } - ], - "updateAction": "APPEND" -}' -orionCurl --url /ngsi10/updateContext -X POST --payload "$payload" > /dev/null -orionCurl --url /v1/updateContext -X POST --payload "$payload" > /dev/null - # Get statistics orionCurl --url /statistics @@ -260,11 +218,8 @@ Content-Length: REGEX(\d+) { "counters": { - "deprecatedFeatures": { - "ngsiv1Requests": 7 - }, - "jsonRequests": 16, - "noPayloadRequests": 42, + "jsonRequests": 11, + "noPayloadRequests": 40, "requests": { "/admin/log": { "GET": 1, @@ -359,27 +314,6 @@ Content-Length: REGEX(\d+) "GET": 1 } }, - "requestsLegacy": { - "/ngsi10/queryContext": { - "POST": 1 - }, - "/ngsi10/updateContext": { - "POST": 1 - }, - "/v1/contextEntities/{id}": { - "DELETE": 1, - "PUT": 1 - }, - "/v1/contextEntities/{id}/attributes/{name}": { - "GET": 1 - }, - "/v1/queryContext": { - "POST": 1 - }, - "/v1/updateContext": { - "POST": 1 - } - }, "textRequests": 1 }, "measuring_interval_in_secs": REGEX(\d+), From 76e4f3304bd99bdde5ff31f4119502677a0847e2 Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Thu, 13 Jun 2024 14:13:42 +0530 Subject: [PATCH 330/390] fix issue2303, invalid characters in dattime field --- CHANGES_NEXT_RELEASE | 3 +- src/lib/common/globals.cpp | 11 + ...s_with_invalid_date_values_in_expires.test | 312 ++++++++++++++++++ 3 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 test/functionalTest/cases/2303_subs_with_invalid_expires_dates/subs_with_invalid_date_values_in_expires.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 4c5c912cb9..60db2f151f 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ - Fix: wrong date values should not allowed in subscription's expires field (#4541) -- Fix: do not raise DB alarm in case of wrong GeoJSON in client request \ No newline at end of file +- Fix: do not raise DB alarm in case of wrong GeoJSON in client request +- Fix: invalid date in expires field of subscription (#2303) diff --git a/src/lib/common/globals.cpp b/src/lib/common/globals.cpp index 2bc490b004..50e761cf83 100644 --- a/src/lib/common/globals.cpp +++ b/src/lib/common/globals.cpp @@ -609,6 +609,17 @@ double parse8601Time(const std::string& ss) return -1; } + // The following 'for' loop is implemented to handle a specific datetime case where the datetime string + // is '2016-04-05T14:10:0x.00Z'. This particular case is being incorrectly PASS through the + // sscanf() function i.e. used next to this 'for' loop. + for (char c : ss) + { + if (std::isalpha(c) && c != 'T' && c != 'Z') + { + return -1; + } + } + // According to https://en.wikipedia.org/wiki/ISO_8601#Times, the following formats have to be supported // // hh:mm:ss.sss or hhmmss.sss diff --git a/test/functionalTest/cases/2303_subs_with_invalid_expires_dates/subs_with_invalid_date_values_in_expires.test b/test/functionalTest/cases/2303_subs_with_invalid_expires_dates/subs_with_invalid_date_values_in_expires.test new file mode 100644 index 0000000000..8405f61811 --- /dev/null +++ b/test/functionalTest/cases/2303_subs_with_invalid_expires_dates/subs_with_invalid_date_values_in_expires.test @@ -0,0 +1,312 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Subscriptions with invalid date formats in 'expires' field + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create a subscription with expires date with an invalid character in 'seconds' field, see error +# 02. Create a subscription with expires date with an invalid character(,), see error +# 03. Create a subscription with expires date with an invalid character in 'millisecond' field, see error +# 04. Create a subscription with expires date with an invalid character in 'millisecond' field, see error +# 05. Create a subscription with expires date with an invalid character in 'millisecond' field, see error +# 06. Create a subscription with an valid date value in 'expires' field, (success case) +# + + +echo "01. Create a subscription with expires date with an invalid character in 'seconds' field, see error" +echo "===================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2016-04-05T14:10:0x.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create a subscription with expires date with an invalid character(,), see error" +echo "===================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2016-04-05T14:10:00,00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create a subscription with expires date with an invalid character in 'millisecond' field, see error" +echo "=======================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2016-04-05T14:10:00.h00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "04. Create a subscription with expires date with an invalid character in 'millisecond' field, see error" +echo "=======================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2016-04-05T14:10:00.0h0Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "05. Create a subscription with expires date with an invalid character in 'millisecond' field, see error" +echo "=======================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2016-04-05T14:10:00.,00L" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "06. Create a subscription with an valid date value in 'expires' field, (success case)" +echo "=====================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id": "Room1", + "type": "Room" + } + ], + "condition": { + "attrs": [ + "pressure" + ] + } + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + }, + "attrs": [ + "pressure" + ] + }, + "expires": "2025-12-31T23:59:59.00Z" +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +--REGEXPECT-- +01. Create a subscription with expires date with an invalid character in 'seconds' field, see error +=================================================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +02. Create a subscription with expires date with an invalid character(,), see error +=================================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +03. Create a subscription with expires date with an invalid character in 'millisecond' field, see error +======================================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +04. Create a subscription with expires date with an invalid character in 'millisecond' field, see error +======================================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +05. Create a subscription with expires date with an invalid character in 'millisecond' field, see error +======================================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 68 + +{ + "description": "expires has an invalid format", + "error": "BadRequest" +} + + +06. Create a subscription with an valid date value in 'expires' field, (success case) +===================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +--TEARDOWN-- +brokerStop CB +dbDrop CB + From 0caefe73da4f270970a37142d5f89226b437b7dc Mon Sep 17 00:00:00 2001 From: Daniel Villalba Date: Wed, 19 Jun 2024 09:33:44 +0200 Subject: [PATCH 331/390] Improve doc of thMapper expression --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 1bcf31e589..c4b2e0744d 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2993,7 +2993,7 @@ results in #### thMapper -Returns a value among several choices based in threshold values. This function is based in an array of *values* and an array of *choices* (which length is exactly the same as values plus one). Thus, if the input value is between the *i*-th and the *i+1*-th item of *values*, then *i*+1-th item of *choices* is returned. +Returns a value among several choices based in threshold values. This function is based in an array of *values* and an array of *choices* (which length is exactly the same as values plus one). Thus, if the input value is greater than or equal to the *i*-th and less than the *i+1*-th item of *values*, then *i*+1-th item of *choices* is returned. This transformation returns `null` if some problem with the arguments is found (i.e. choices length is not exacly the same as values plus one, some of the items in the values array is not a number, etc.) From 192392d841102e8e22bddbc677d585b3c10d0af7 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Tue, 25 Jun 2024 05:36:25 +0900 Subject: [PATCH 332/390] (JP) Improve doc of thMapper expression (#4586) --- doc/manuals.jp/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index 6a4ff60d45..9bfc131b41 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -3052,7 +3052,7 @@ c|mapper(values,choices) #### thMapper -しきい値に基づいて、複数の選択肢の中から値を返します。この関数は、値の配列 (*values*) と選択肢の配列 (*choices*) に基づいています (長さは、値に 1 を加えた値とまったく同じです)。したがって、入力値が *values* の *i* 番目と *i+1* 番目の項目の間にある場合は、*choices* の *i*+1 番目の項目が返されます。 +しきい値に基づいて、複数の選択肢の中から値を返します。この関数は、値の配列 (*values*) と選択肢の配列 (*choices*) に基づいています (長さは、値に 1 を加えた値とまったく同じです)。したがって、入力値が *i* 番目の *values* 項目以上で *i+1* 番目の項目より小さい場合、*i*+1 番目の *choices* 項目が返されます。 引数に問題が見つかった場合 (つまり、選択肢の長さが値に 1 を加えた値とまったく同じではない、値配列の一部の項目が数値ではないなど)、この変換は `null` を返します。 From 0fe0be5d33ebcec5daffbd625ecd5311735f9ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 2 Jul 2024 13:01:01 +0200 Subject: [PATCH 333/390] ADD evalPriority feature --- src/lib/common/JsonHelper.cpp | 33 ++- src/lib/common/JsonHelper.h | 6 +- src/lib/common/errorMessages.h | 2 +- src/lib/common/limits.h | 10 + src/lib/expressions/ExprContext.cpp | 11 +- src/lib/jsonParseV2/parseSubscription.cpp | 15 +- src/lib/ngsi/ContextAttribute.cpp | 31 ++- src/lib/ngsi/ContextAttribute.h | 4 +- src/lib/ngsi/Metadata.h | 1 + src/lib/ngsiNotify/Notifier.cpp | 62 ++++- .../eval_priority_in_expressions.test | 262 ++++++++++++++++++ 11 files changed, 402 insertions(+), 35 deletions(-) create mode 100644 test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test diff --git a/src/lib/common/JsonHelper.cpp b/src/lib/common/JsonHelper.cpp index 3bd6fd5d2a..9adc597fc2 100644 --- a/src/lib/common/JsonHelper.cpp +++ b/src/lib/common/JsonHelper.cpp @@ -138,7 +138,7 @@ std::string objectToJson(std::map& list) * * JsonObjectHelper - */ -JsonObjectHelper::JsonObjectHelper(): empty(true), closed(false) +JsonObjectHelper::JsonObjectHelper(): empty(true) { ss += '{'; } @@ -280,15 +280,17 @@ void JsonObjectHelper::addNull(const std::string& key) * * JsonObjectHelper::str - */ -std::string JsonObjectHelper::str() +std::string JsonObjectHelper::str(bool closed) { - // This check allows to call str() several times (needed when this is used in ExprContext) - if (!closed) + // closed == false used in ExprContext logic + if (closed) { - ss += '}'; - closed = true; + return ss + '}'; + } + else + { + return ss; } - return ss; } @@ -297,7 +299,7 @@ std::string JsonObjectHelper::str() * * JsonVectorHelper - */ -JsonVectorHelper::JsonVectorHelper(): empty(true), closed(false) +JsonVectorHelper::JsonVectorHelper(): empty(true) { ss += '['; } @@ -425,14 +427,17 @@ void JsonVectorHelper::addNull(void) /* **************************************************************************** * * JsonVectorHelper::str - +* FIXME PR: bool closed probably unneded in vectors */ -std::string JsonVectorHelper::str() +std::string JsonVectorHelper::str(bool closed) { - // This check allows to call str() several times (needed when this is used in ExprContext) - if (!closed) + // closed == false used in ExprContext logic + if (closed) { - ss += ']'; - closed = true; + return ss + ']'; + } + else + { + return ss; } - return ss; } diff --git a/src/lib/common/JsonHelper.h b/src/lib/common/JsonHelper.h index bcb8aaeaf2..5b2633e0ca 100644 --- a/src/lib/common/JsonHelper.h +++ b/src/lib/common/JsonHelper.h @@ -45,12 +45,11 @@ class JsonObjectHelper void addBool(const std::string& key, bool b); void addNull(const std::string& key); - std::string str(); + std::string str(bool closed = true); private: std::string ss; bool empty; - bool closed; }; @@ -68,12 +67,11 @@ class JsonVectorHelper void addNull(void); - std::string str(); + std::string str(bool closed = true); private: std::string ss; bool empty; - bool closed; }; diff --git a/src/lib/common/errorMessages.h b/src/lib/common/errorMessages.h index d50bf8acf7..a5f36a2f3a 100644 --- a/src/lib/common/errorMessages.h +++ b/src/lib/common/errorMessages.h @@ -87,7 +87,7 @@ #define ERROR_DESC_BAD_REQUEST_FORMAT_INVALID "invalid render format for notifications" #define ERROR_DESC_BAD_REQUEST_SERVICE_NOT_FOUND "Service not found. Check your URL as probably it is wrong." #define ERROR_DESC_BAD_REQUEST_WRONG_GEOJSON "Wrong GeoJson" -#define ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF "metadata are not allowed in ngsi field in custom notifications" +#define ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF "only evalPriority metadata is allowed in ngsi field in custom notifications" #define ERROR_NOT_FOUND "NotFound" #define ERROR_DESC_NOT_FOUND_ENTITY "The requested entity has not been found. Check type and id" diff --git a/src/lib/common/limits.h b/src/lib/common/limits.h index 39295ec64d..9a903f7953 100644 --- a/src/lib/common/limits.h +++ b/src/lib/common/limits.h @@ -229,4 +229,14 @@ +/* **************************************************************************** +* +* MIX_PRIORITY and MAX_PRIORITY +* +*/ +#define MIN_PRIORITY 1 +#define MAX_PRIORITY LLONG_MAX + + + #endif // SRC_LIB_COMMON_LIMITS_H_ diff --git a/src/lib/expressions/ExprContext.cpp b/src/lib/expressions/ExprContext.cpp index 80148399b3..94bf607a74 100644 --- a/src/lib/expressions/ExprContext.cpp +++ b/src/lib/expressions/ExprContext.cpp @@ -47,7 +47,7 @@ ExprContextObject::ExprContextObject(bool _basic) */ std::string ExprContextObject::getJexlContext(void) { - return jh.str(); + return jh.str(false) + '}'; } @@ -83,7 +83,14 @@ void ExprContextObject::add(const std::string &key, const std::string &_value, b else { LM_T(LmtExpr, ("adding to JEXL expression context object (string): %s=%s", key.c_str(), _value.c_str())); - jh.addString(key, _value); + if (raw) + { + jh.addRaw(key, _value); + } + else + { + jh.addString(key, _value); + } } } diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index 593161b202..7dd54c9b65 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -554,10 +554,19 @@ static std::string parseCustomPayload return badInput(ciP, r); } - // metadadata are now allowed in this case - if (caP->metadataVector.size() > 0) + // only evalPriority metadadata is allowed in this case + for (unsigned ix = 0; ix < caP->metadataVector.size(); ix++) { - return badInput(ciP, ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF); + if (caP->metadataVector[ix]->name != NGSI_MD_EVAL_PRIORITY) + { + return badInput(ciP, ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF); + } + else + { + // priority must be a number + // priority must be between MIX_PRIORITY and MAX_PRIORITY + // FIXME: PR (include a .test to asses the checkings works) + } } } } diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 79ce3f57fc..5fa1f9f78c 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -763,6 +763,26 @@ bool ContextAttribute::getLocation(orion::BSONObj* attrsP) const +/* **************************************************************************** +* +* getEvalPriority - +*/ +double ContextAttribute::getEvalPriority(void) +{ + for (unsigned int ix = 0; ix < metadataVector.size(); ix++) + { + if (metadataVector[ix]->name == NGSI_MD_EVAL_PRIORITY) + { + return metadataVector[ix]->numberValue; + } + } + + // if the attribute doesn't have evalPriority metadata, then max priority is assumed + return MAX_PRIORITY; +} + + + /* **************************************************************************** * * toJsonV1AsObject - @@ -1140,9 +1160,10 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi * toJsonValue - * * To be used by options=values and options=unique renderings +* Also used by the ngsi expression logic * */ -std::string ContextAttribute::toJsonValue(void) +std::string ContextAttribute::toJsonValue(ExprContextObject* exprContextObjectP) { if (compoundValueP != NULL) { @@ -1164,10 +1185,10 @@ std::string ContextAttribute::toJsonValue(void) } else if (valueType == orion::ValueTypeString) { - std::string out = "\""; - out += toJsonString(stringValue); - out += '"'; - return out; + //std::string out = "\""; + //out += toJsonString(stringValue); + //out += '"'; + return smartStringValue(stringValue, exprContextObjectP, "null"); } else if (valueType == orion::ValueTypeBoolean) { diff --git a/src/lib/ngsi/ContextAttribute.h b/src/lib/ngsi/ContextAttribute.h index f242bf101d..63e4551690 100644 --- a/src/lib/ngsi/ContextAttribute.h +++ b/src/lib/ngsi/ContextAttribute.h @@ -95,6 +95,8 @@ typedef struct ContextAttribute /* Check if attribute means a location */ bool getLocation(orion::BSONObj* attrsP) const; + double getEvalPriority(void); + std::string toJsonV1(bool asJsonObject, RequestType request, const std::vector& metadataFilter, @@ -110,7 +112,7 @@ typedef struct ContextAttribute std::string toJson(const std::vector& metadataFilter, bool renderNgsiField = false, ExprContextObject* exprContextObjectP = NULL); - std::string toJsonValue(void); + std::string toJsonValue(ExprContextObject* exprContextObjectP = NULL); std::string toJsonAsValue(ApiVersion apiVersion, bool acceptedTextPlain, diff --git a/src/lib/ngsi/Metadata.h b/src/lib/ngsi/Metadata.h index 591df92cb1..e1fcdab163 100644 --- a/src/lib/ngsi/Metadata.h +++ b/src/lib/ngsi/Metadata.h @@ -46,6 +46,7 @@ * * Metadata interpreted by Orion Context Broker, i.e. not custom metadata */ +#define NGSI_MD_EVAL_PRIORITY "evalPriority" #define NGSI_MD_IGNORE_TYPE "ignoreType" #define NGSI_MD_PREVIOUSVALUE "previousValue" // Special metadata #define NGSI_MD_ACTIONTYPE "actionType" // Special metadata diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 14724c9a1e..6fd17245da 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -193,6 +193,51 @@ static bool setJsonPayload +/* **************************************************************************** +* +* orderByPriority - +* +* Return false if some problem occur +*/ +static void orderByPriority +( + const Entity& ngsi, + std::vector* orderedAttrs +) +{ + // Add all the attributes to a temporal vector + std::vector attrs; + + for (unsigned ix = 0; ix < ngsi.attributeVector.size(); ix++) + { + attrs.push_back(ngsi.attributeVector[ix]); + } + + // Pass the content of attrs to orderedAttrs based in the evalPriority element + while (!attrs.empty()) + { + ContextAttribute* selectedAttr = attrs[0]; + unsigned int selectedIx = 0; + double prio = selectedAttr->getEvalPriority(); + + for (unsigned ix = 0; ix < attrs.size(); ix++) + { + double newPrio = attrs[ix]->getEvalPriority(); + if (newPrio < prio) + { + selectedAttr = attrs[ix]; + selectedIx = ix; + prio = newPrio; + } + } + + orderedAttrs->push_back(selectedAttr); + attrs.erase(attrs.begin() + selectedIx); + } +} + + + /* **************************************************************************** * * setNgsiPayload - @@ -209,7 +254,8 @@ static bool setNgsiPayload bool blacklist, const std::vector& metadataFilter, std::string* payloadP, - RenderFormat renderFormat + RenderFormat renderFormat, + bool basic // used by TIME_EXPR_CTXBLD_START/STOP macros ) { NotifyContextRequest ncr; @@ -239,10 +285,16 @@ static bool setNgsiPayload cer.entity.fill(effectiveId, effectiveType, en.isPattern, en.servicePath); - // First we add attributes in the ngsi field - for (unsigned int ix = 0; ix < ngsi.attributeVector.size(); ix++) + // First we add attributes in the ngsi field, adding calculated expressions to context in order of priority + std::vector orderedNgsiAttrs; + orderByPriority(ngsi, &orderedNgsiAttrs); + + for (unsigned int ix = 0; ix < orderedNgsiAttrs.size(); ix++) { - cer.entity.attributeVector.push_back(new ContextAttribute(ngsi.attributeVector[ix], false, true)); + cer.entity.attributeVector.push_back(new ContextAttribute(orderedNgsiAttrs[ix], false, true)); + TIME_EXPR_CTXBLD_START(); + exprContextObjectP->add(orderedNgsiAttrs[ix]->name, orderedNgsiAttrs[ix]->toJsonValue(exprContextObjectP), true); + TIME_EXPR_CTXBLD_STOP(); } // Next, other attributes in the original entity not already added for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) @@ -393,7 +445,7 @@ static SenderThreadParams* buildSenderParamsCustom { // Important to use const& for Entity here. Otherwise problems may occur in the object release logic const Entity& ngsi = (notification.type == ngsiv2::HttpNotification ? notification.httpInfo.ngsi : notification.mqttInfo.ngsi); - if (!setNgsiPayload(ngsi, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat)) + if (!setNgsiPayload(ngsi, subscriptionId, en, &exprContext, attrsFilter, blacklist, metadataFilter, &payload, renderFormat, basic)) { // Warning already logged in macroSubstitute() return NULL; diff --git a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test new file mode 100644 index 0000000000..5df812c7a1 --- /dev/null +++ b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test @@ -0,0 +1,262 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +Evaluation priority in expressions + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified) +# 02. Create entity E1 with A=1 +# 03. Update entity E1 with A=2.1 +# 04. Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1) +# + + +echo "01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified)" +echo "=========================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "P2": { + "value": "${P1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 2, + "type": "Number" + } + } + }, + "P1": { + "value": "${A+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 1, + "type": "Number" + } + } + }, + "P3": { + "value": "${P2+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 3, + "type": "Number" + } + } + }, + "S": { + "value": "${P3+1}", + "type": "Calculated" + } + } + }, + "attrs": [ "P1", "P2", "S" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A=2.1" +echo "===============================" +payload='{ + "A": { + "value": 2.1, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1)" +echo "====================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified) +========================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A=2.1 +=============================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1) +==================================================================================== +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 313 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "P1": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 1 + } + }, + "type": "Calculated", + "value": 2 + }, + "P2": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 2 + } + }, + "type": "Calculated", + "value": 3 + }, + "S": { + "metadata": {}, + "type": "Calculated", + "value": 5 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 319 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "P1": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 1 + } + }, + "type": "Calculated", + "value": 3.1 + }, + "P2": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 2 + } + }, + "type": "Calculated", + "value": 4.1 + }, + "S": { + "metadata": {}, + "type": "Calculated", + "value": 6.1 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 50221f6b3a38a0f78e0098c74b3af0d80262876c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 09:41:47 +0200 Subject: [PATCH 334/390] FIX jexl context addition logic --- src/lib/ngsiNotify/Notifier.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 6fd17245da..fb5070f1ab 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -292,9 +292,14 @@ static bool setNgsiPayload for (unsigned int ix = 0; ix < orderedNgsiAttrs.size(); ix++) { cer.entity.attributeVector.push_back(new ContextAttribute(orderedNgsiAttrs[ix], false, true)); - TIME_EXPR_CTXBLD_START(); - exprContextObjectP->add(orderedNgsiAttrs[ix]->name, orderedNgsiAttrs[ix]->toJsonValue(exprContextObjectP), true); - TIME_EXPR_CTXBLD_STOP(); + + // Avoid to add context if an attribute with the same name exists in the entity + if (cer.entity.attributeVector.get(orderedNgsiAttrs[ix]->name) < 0) + { + TIME_EXPR_CTXBLD_START(); + exprContextObjectP->add(orderedNgsiAttrs[ix]->name, orderedNgsiAttrs[ix]->toJsonValue(exprContextObjectP), true); + TIME_EXPR_CTXBLD_STOP(); + } } // Next, other attributes in the original entity not already added for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) From a67e6f10378bb37517cfb6662ce0cba2d93ede30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 09:49:44 +0200 Subject: [PATCH 335/390] FIX description in 4085 ftest --- .../custom_notification_http_ngsi_errors.test | 4 ++-- .../custom_notification_mqtt_ngsi_errors.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_errors.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_errors.test index 56e39ca359..1406c43de4 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_errors.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_http_ngsi_errors.test @@ -243,10 +243,10 @@ HTTP/1.1 400 Bad Request Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 101 +Content-Length: 114 { - "description": "metadata are not allowed in ngsi field in custom notifications", + "description": "only evalPriority metadata is allowed in ngsi field in custom notifications", "error": "BadRequest" } diff --git a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_errors.test b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_errors.test index 59742603a8..78f1212fa3 100644 --- a/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_errors.test +++ b/test/functionalTest/cases/4085_custom_notifications_ngsi_payload/custom_notification_mqtt_ngsi_errors.test @@ -249,10 +249,10 @@ HTTP/1.1 400 Bad Request Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 101 +Content-Length: 114 { - "description": "metadata are not allowed in ngsi field in custom notifications", + "description": "only evalPriority metadata is allowed in ngsi field in custom notifications", "error": "BadRequest" } From 61a2fa4f0570aad05be0130b1d6c594479ffd526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 10:03:39 +0200 Subject: [PATCH 336/390] FIX jexl context addition logic --- src/lib/ngsiNotify/Notifier.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index fb5070f1ab..0ca4da8582 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -291,15 +291,15 @@ static bool setNgsiPayload for (unsigned int ix = 0; ix < orderedNgsiAttrs.size(); ix++) { - cer.entity.attributeVector.push_back(new ContextAttribute(orderedNgsiAttrs[ix], false, true)); - // Avoid to add context if an attribute with the same name exists in the entity - if (cer.entity.attributeVector.get(orderedNgsiAttrs[ix]->name) < 0) + if (en.attributeVector.get(orderedNgsiAttrs[ix]->name) < 0) { TIME_EXPR_CTXBLD_START(); exprContextObjectP->add(orderedNgsiAttrs[ix]->name, orderedNgsiAttrs[ix]->toJsonValue(exprContextObjectP), true); TIME_EXPR_CTXBLD_STOP(); } + + cer.entity.attributeVector.push_back(new ContextAttribute(orderedNgsiAttrs[ix], false, true)); } // Next, other attributes in the original entity not already added for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) From 3eed5ca99f9d89ed4f40f4b3a6408d3e15cd3447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 10:06:25 +0200 Subject: [PATCH 337/390] FIX expression names as suggested in code review --- .../eval_priority_in_expressions.test | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test index 5df812c7a1..6c40610a65 100644 --- a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test +++ b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_in_expressions.test @@ -32,14 +32,14 @@ accumulatorStart --pretty-print --SHELL-- # -# 01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified) +# 01. Create custom sub with expressions Z1=A+1, Y2=Z1+1, X3=Y2+1, S=X3+1 (X3 not notified) # 02. Create entity E1 with A=1 # 03. Update entity E1 with A=2.1 -# 04. Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1) +# 04. Dump accumulator and see two notifications (Z1:2, Y2:3, S:5) (Z1:3.1, Y2:4.1, S:6.1) # -echo "01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified)" +echo "01. Create custom sub with expressions Z1=A+1, Y2=Z1+1, X3=Y2+1, S=X3+1 (X3 not notified)" echo "=========================================================================================" payload='{ "subject": { @@ -54,8 +54,8 @@ payload='{ "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", "ngsi": { - "P2": { - "value": "${P1+1}", + "Y2": { + "value": "${Z1+1}", "type": "Calculated", "metadata": { "evalPriority": { @@ -64,7 +64,7 @@ payload='{ } } }, - "P1": { + "Z1": { "value": "${A+1}", "type": "Calculated", "metadata": { @@ -74,8 +74,8 @@ payload='{ } } }, - "P3": { - "value": "${P2+1}", + "X3": { + "value": "${Y2+1}", "type": "Calculated", "metadata": { "evalPriority": { @@ -85,12 +85,12 @@ payload='{ } }, "S": { - "value": "${P3+1}", + "value": "${X3+1}", "type": "Calculated" } } }, - "attrs": [ "P1", "P2", "S" ] + "attrs": [ "Z1", "Y2", "S" ] } }' orionCurl --url /v2/subscriptions --payload "$payload" @@ -126,7 +126,7 @@ echo echo -echo "Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1)" +echo "Dump accumulator and see two notifications (Z1:2, Y2:3, S:5) (Z1:3.1, Y2:4.1, S:6.1)" echo "====================================================================================" accumulatorDump echo @@ -134,7 +134,7 @@ echo --REGEXPECT-- -01. Create custom sub with expressions P1=A+1, P2=P1+1, P3=P2+1, S=P3+1 (P3 not notified) +01. Create custom sub with expressions Z1=A+1, Y2=Z1+1, X3=Y2+1, S=X3+1 (X3 not notified) ========================================================================================= HTTP/1.1 201 Created Date: REGEX(.*) @@ -162,7 +162,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -Dump accumulator and see two notifications (P1:2, P2:3, S:5) (P1:3.1, P2:4.1, S:6.1) +Dump accumulator and see two notifications (Z1:2, Y2:3, S:5) (Z1:3.1, Y2:4.1, S:6.1) ==================================================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / @@ -177,30 +177,30 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "P1": { + "S": { + "metadata": {}, + "type": "Calculated", + "value": 5 + }, + "Y2": { "metadata": { "evalPriority": { "type": "Number", - "value": 1 + "value": 2 } }, "type": "Calculated", - "value": 2 + "value": 3 }, - "P2": { + "Z1": { "metadata": { "evalPriority": { "type": "Number", - "value": 2 + "value": 1 } }, "type": "Calculated", - "value": 3 - }, - "S": { - "metadata": {}, - "type": "Calculated", - "value": 5 + "value": 2 }, "id": "E1", "type": "T" @@ -222,30 +222,30 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 { "data": [ { - "P1": { + "S": { + "metadata": {}, + "type": "Calculated", + "value": 6.1 + }, + "Y2": { "metadata": { "evalPriority": { "type": "Number", - "value": 1 + "value": 2 } }, "type": "Calculated", - "value": 3.1 + "value": 4.1 }, - "P2": { + "Z1": { "metadata": { "evalPriority": { "type": "Number", - "value": 2 + "value": 1 } }, "type": "Calculated", - "value": 4.1 - }, - "S": { - "metadata": {}, - "type": "Calculated", - "value": 6.1 + "value": 3.1 }, "id": "E1", "type": "T" From 611ed6ac18238c9d91da0913720c026ae9371ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 10:56:38 +0200 Subject: [PATCH 338/390] FIX evalPriority CRUD --- src/lib/common/errorMessages.h | 4 + src/lib/common/limits.h | 2 +- src/lib/jsonParseV2/parseSubscription.cpp | 15 +- src/lib/ngsi/ContextAttribute.cpp | 7 +- .../eval_priority_crud_and_errors.test | 672 ++++++++++++++++++ 5 files changed, 693 insertions(+), 7 deletions(-) create mode 100644 test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test diff --git a/src/lib/common/errorMessages.h b/src/lib/common/errorMessages.h index a5f36a2f3a..0d170c7ec9 100644 --- a/src/lib/common/errorMessages.h +++ b/src/lib/common/errorMessages.h @@ -89,6 +89,10 @@ #define ERROR_DESC_BAD_REQUEST_WRONG_GEOJSON "Wrong GeoJson" #define ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF "only evalPriority metadata is allowed in ngsi field in custom notifications" +#define ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MUST_BE_A_NUMBER "evalPriority metadata must be a number" +#define ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MIN_ERROR "evalPriority metadata minimum priority is " STR(MIN_PRIORITY) +#define ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MAX_ERROR "evalPriority metadata maximum priority is " STR(MAX_PRIORITY) + #define ERROR_NOT_FOUND "NotFound" #define ERROR_DESC_NOT_FOUND_ENTITY "The requested entity has not been found. Check type and id" #define ERROR_DESC_NOT_FOUND_ENTITY_TYPE "Entity type not found" diff --git a/src/lib/common/limits.h b/src/lib/common/limits.h index 9a903f7953..0fd1283759 100644 --- a/src/lib/common/limits.h +++ b/src/lib/common/limits.h @@ -235,7 +235,7 @@ * */ #define MIN_PRIORITY 1 -#define MAX_PRIORITY LLONG_MAX +#define MAX_PRIORITY 100000 diff --git a/src/lib/jsonParseV2/parseSubscription.cpp b/src/lib/jsonParseV2/parseSubscription.cpp index 7dd54c9b65..77269af0aa 100644 --- a/src/lib/jsonParseV2/parseSubscription.cpp +++ b/src/lib/jsonParseV2/parseSubscription.cpp @@ -563,9 +563,18 @@ static std::string parseCustomPayload } else { - // priority must be a number - // priority must be between MIX_PRIORITY and MAX_PRIORITY - // FIXME: PR (include a .test to asses the checkings works) + if (caP->metadataVector[ix]->valueType != orion::ValueTypeNumber) + { + return badInput(ciP, ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MUST_BE_A_NUMBER); + } + if (caP->metadataVector[ix]->numberValue < MIN_PRIORITY) + { + return badInput(ciP, ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MIN_ERROR); + } + if (caP->metadataVector[ix]->numberValue > MAX_PRIORITY) + { + return badInput(ciP, ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MAX_ERROR); + } } } } diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 5fa1f9f78c..74bebd4e99 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1146,10 +1146,11 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi // // metadata (note that ngsi field in custom notifications doesn't include metadata) // - if (!renderNgsiField) - { + // FIXME PR: cleanup this + //if (!renderNgsiField) + //{ jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); - } + //} return jh.str(); } diff --git a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test new file mode 100644 index 0000000000..741a19dfc8 --- /dev/null +++ b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test @@ -0,0 +1,672 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Evaluation priority in expressions CRUD and errors + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with expressions with evalPriority as string, see error +# 02. Create custom sub with expressions with evalPriority 0, see error +# 03. Create custom sub with expressions with evalPriority 100001, see error +# 04. Create custom sub with expression with evalPriority 1 +# 05. Get sub to see evalPriority 1 +# 06. Update custom sub with expression with evalPriority as string, see error +# 07. Update custom sub with expression with evalPriority 0, see error +# 08. Update custom sub with expressions with evalPriority 100001, see error +# 09. Update custom sub with expression with evalPriority 2 +# 10. Get sub to see evalPriority 2 +# 11. Update custom sub with expression without evalPriority +# 12. Get sub to see no evalPriority +# 13. Update custom sub with expression with evalPriority 3 +# 14. Get sub to see evalPriority 3 +# + + +echo "01. Create custom sub with expressions with evalPriority as string, see error" +echo "=============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": "foo", + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create custom sub with expressions with evalPriority 0, see error" +echo "=====================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 0, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "03. Create custom sub with expressions with evalPriority 100001, see error" +echo "=======================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 100001, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "04. Create custom sub with expression with evalPriority 1" +echo "=========================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 1, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +SUB_ID=$(echo "$_responseHeaders" | grep Location | awk -F/ '{ print $4 }' | tr -d "\r\n") + + +echo "05. Get sub to see evalPriority 1" +echo "=================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "06.Update custom sub with expression with evalPriority as string, see error" +echo "===========================================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": "foo", + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "07. Update custom sub with expression with evalPriority 0, see error" +echo "====================================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 0, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "08. Update custom sub with expressions with evalPriority 100001, see error" +echo "=======================================================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 100001, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "09. Update custom sub with expressions with evalPriority 2" +echo "==========================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 2, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "10. Get sub to see evalPriority 2" +echo "=================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +echo "11. Update custom sub with expression without evalPriority" +echo "==========================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated" + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "12. Get sub to see no evalPriority" +echo "==================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + +echo "13. Update custom sub with expressions with evalPriority 3" +echo "==========================================================" +payload='{ + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${Z1+1}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 3, + "type": "Number" + } + } + } + } + } + } +}' +orionCurl --url /v2/subscriptions/$SUB_ID --payload "$payload" -X PATCH +echo +echo + + +echo "14. Get sub to see evalPriority 3" +echo "=================================" +orionCurl --url /v2/subscriptions/$SUB_ID +echo +echo + + +--REGEXPECT-- +01. Create custom sub with expressions with evalPriority as string, see error +============================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 77 + +{ + "description": "evalPriority metadata must be a number", + "error": "BadRequest" +} + + +02. Create custom sub with expressions with evalPriority 0, see error +===================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 82 + +{ + "description": "evalPriority metadata minimum priority is 1", + "error": "BadRequest" +} + + +03. Create custom sub with expressions with evalPriority 100001, see error +======================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 87 + +{ + "description": "evalPriority metadata maximum priority is 100000", + "error": "BadRequest" +} + + +04. Create custom sub with expression with evalPriority 1 +========================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Get sub to see evalPriority 1 +================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 411 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "ngsi": { + "S": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 1 + } + }, + "type": "Calculated", + "value": "${Z1+1}" + } + }, + "url": "http://127.0.0.1:9997/notify" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "id": "E1", + "type": "T" + } + ] + } +} + + +06.Update custom sub with expression with evalPriority as string, see error +=========================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 77 + +{ + "description": "evalPriority metadata must be a number", + "error": "BadRequest" +} + + +07. Update custom sub with expression with evalPriority 0, see error +==================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 82 + +{ + "description": "evalPriority metadata minimum priority is 1", + "error": "BadRequest" +} + + +08. Update custom sub with expressions with evalPriority 100001, see error +======================================================================================= +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 87 + +{ + "description": "evalPriority metadata maximum priority is 100000", + "error": "BadRequest" +} + + +09. Update custom sub with expressions with evalPriority 2 +========================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +10. Get sub to see evalPriority 2 +================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 411 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "ngsi": { + "S": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 2 + } + }, + "type": "Calculated", + "value": "${Z1+1}" + } + }, + "url": "http://127.0.0.1:9997/notify" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "id": "E1", + "type": "T" + } + ] + } +} + + +11. Update custom sub with expression without evalPriority +========================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +12. Get sub to see no evalPriority +================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 369 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "ngsi": { + "S": { + "metadata": {}, + "type": "Calculated", + "value": "${Z1+1}" + } + }, + "url": "http://127.0.0.1:9997/notify" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "id": "E1", + "type": "T" + } + ] + } +} + + +13. Update custom sub with expressions with evalPriority 3 +========================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +14. Get sub to see evalPriority 3 +================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 411 + +{ + "id": "REGEX([0-9a-f]{24})", + "notification": { + "attrs": [], + "attrsFormat": "normalized", + "covered": false, + "httpCustom": { + "ngsi": { + "S": { + "metadata": { + "evalPriority": { + "type": "Number", + "value": 3 + } + }, + "type": "Calculated", + "value": "${Z1+1}" + } + }, + "url": "http://127.0.0.1:9997/notify" + }, + "onlyChangedAttrs": false + }, + "status": "active", + "subject": { + "condition": { + "attrs": [], + "notifyOnMetadataChange": true + }, + "entities": [ + { + "id": "E1", + "type": "T" + } + ] + } +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From f44ab2adac036e44c66d68c6372413407773b793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 11:26:51 +0200 Subject: [PATCH 339/390] ADD evalPriority to doc --- doc/manuals/orion-api.md | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index c4b2e0744d..4d8fdb382a 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -75,6 +75,7 @@ - [Additional considerations](#additional-considerations) - [JEXL Support](#jexl-support) - [JEXL usage example](#jexl-usage-example) + - [Evaluation priority](#evaluation-priority) - [Available Transformations](#available-transformations) - [`uppercase`](#uppercase) - [`lowercase`](#lowercase) @@ -746,6 +747,8 @@ type has an special semantic for Orion: * `DateTime` * `geo:json` +* `evalPriority`: used by expression evaluation. Have a look to [this specific section](#evaluation-priority) for details. + At the present moment `ignoreType` is supported only for geo-location types, this way allowing a mechanism to overcome the limit of only one geo-location per entity (more details in [Geospatial properties of entities](#geospatial-properties-of-entities) section). Support @@ -2507,6 +2510,64 @@ will trigger a notification like this: ] ``` +### Evaluation priority + +Each time an expression is evaluated, it is added to the expression context, so it can be reused by other expressions. However, by default Orion does not guarantee a given evaluation other. + +Thus, if we have this: + +``` +"httpCustom": { + ... + "ngsi": { + "A": { + "value": "${16|sqrt}", + "type": "Calculated" + }, + "B": { + "value": "${A/100}", + "type": "Calculated" + } + }, + "attrs": [ "B" ] +} +``` + +`B` could be set to the desired `0.04` (if `A` is evaluated before `B`) or to the underised `null` (if `B` is evaluated before `A`), randomly. + +In order to overcome this problem, the `evalPriority` metadata can be used to define the evaluation order. It works this way: + +* `evalPriority` metadata is a number from 1 (first evaluation) to 100000 (last evaluation) +* Expressions are evaluated in incresing order of priority +* In case of ties, Orion does not guarantee a particular evaluation order. Thus, expressions in the same priority level must be considered independent or bad things would happen +* If no `evalPriority` is set, default 100000 is used +* `evalPriority` only has a meaning in `notification.httpCustom.ngsi` in subscriptions. As metadata in regular entities (e.g. an entity created with `POST /v2/entities`) Orion doesn't implements any semantic. + +Using `evalPriority` the above example could be reformuled this way: + +``` +"httpCustom": { + ... + "ngsi": { + "A": { + "value": "${16|sqrt}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 1, + "type": "Number" + } + } + }, + "B": { + "value": "${A/100}", + "type": "Calculated" + } + }, + "attrs": [ "B" ] +} +``` + ### Available Transformations #### `uppercase` From 5a3a04efa46c758c106d31a9d4824a7ff9b8ca29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 11:27:00 +0200 Subject: [PATCH 340/390] ADD Changelog entry --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 4c5c912cb9..fa20464f70 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ +- Fix: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) - Fix: wrong date values should not allowed in subscription's expires field (#4541) - Fix: do not raise DB alarm in case of wrong GeoJSON in client request \ No newline at end of file From 23a905f6c9591472110483cdfa17f1e67612bd12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 11:39:47 +0200 Subject: [PATCH 341/390] FIX metadata rendering logic in custom ngsi case --- src/lib/ngsi/ContextAttribute.cpp | 11 +++++------ .../eval_priority_crud_and_errors.test | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 74bebd4e99..58675b6edd 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1144,13 +1144,12 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi filterAndOrderMetadata(metadataFilter, &orderedMetadata); // - // metadata (note that ngsi field in custom notifications doesn't include metadata) + // metadata (note that ngsi field in custom notifications avoids empety metadata array, i.e. "metadata": {}) // - // FIXME PR: cleanup this - //if (!renderNgsiField) - //{ - jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); - //} + if ((!renderNgsiField) || (metadataVector.size() > 0)) + { + jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); + } return jh.str(); } diff --git a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test index 741a19dfc8..a875c8ee21 100644 --- a/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test +++ b/test/functionalTest/cases/4556_eval_priority_in_expressions/eval_priority_crud_and_errors.test @@ -575,7 +575,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 369 +Content-Length: 355 { "id": "REGEX([0-9a-f]{24})", @@ -586,7 +586,6 @@ Content-Length: 369 "httpCustom": { "ngsi": { "S": { - "metadata": {}, "type": "Calculated", "value": "${Z1+1}" } From 553342ac002b1105fb383c2276c0d2aab9efd06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 12:09:14 +0200 Subject: [PATCH 342/390] FIX refactor --- src/lib/common/JsonHelper.cpp | 15 ++++----------- src/lib/common/JsonHelper.h | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/lib/common/JsonHelper.cpp b/src/lib/common/JsonHelper.cpp index 9adc597fc2..65b0e3be64 100644 --- a/src/lib/common/JsonHelper.cpp +++ b/src/lib/common/JsonHelper.cpp @@ -424,20 +424,13 @@ void JsonVectorHelper::addNull(void) + /* **************************************************************************** * * JsonVectorHelper::str - -* FIXME PR: bool closed probably unneded in vectors */ -std::string JsonVectorHelper::str(bool closed) +std::string JsonVectorHelper::str(void) { - // closed == false used in ExprContext logic - if (closed) - { - return ss + ']'; - } - else - { - return ss; - } + ss += ']'; + return ss; } diff --git a/src/lib/common/JsonHelper.h b/src/lib/common/JsonHelper.h index 5b2633e0ca..8f01adcf7b 100644 --- a/src/lib/common/JsonHelper.h +++ b/src/lib/common/JsonHelper.h @@ -67,7 +67,7 @@ class JsonVectorHelper void addNull(void); - std::string str(bool closed = true); + std::string str(void); private: std::string ss; From 1b9b33225ae4d0fb3eded8676c5a0d691a18d00f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 12:36:35 +0200 Subject: [PATCH 343/390] Apply suggestions from code review --- doc/manuals/orion-api.md | 4 ++-- src/lib/ngsi/ContextAttribute.cpp | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 4d8fdb382a..a32334b2aa 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2533,7 +2533,7 @@ Thus, if we have this: } ``` -`B` could be set to the desired `0.04` (if `A` is evaluated before `B`) or to the underised `null` (if `B` is evaluated before `A`), randomly. +in the resulting notification `B` could be set to the desired `0.04` (if `A` is evaluated before `B`) or to the underised `null` (if `B` is evaluated before `A`), randomly. In order to overcome this problem, the `evalPriority` metadata can be used to define the evaluation order. It works this way: @@ -2541,7 +2541,7 @@ In order to overcome this problem, the `evalPriority` metadata can be used to de * Expressions are evaluated in incresing order of priority * In case of ties, Orion does not guarantee a particular evaluation order. Thus, expressions in the same priority level must be considered independent or bad things would happen * If no `evalPriority` is set, default 100000 is used -* `evalPriority` only has a meaning in `notification.httpCustom.ngsi` in subscriptions. As metadata in regular entities (e.g. an entity created with `POST /v2/entities`) Orion doesn't implements any semantic. +* `evalPriority` only has a meaning in `notification.httpCustom.ngsi` in subscriptions. As metadata in regular entities (e.g. an entity created with `POST /v2/entities`) Orion doesn't implements any semantic for it Using `evalPriority` the above example could be reformuled this way: diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp index 58675b6edd..0fc57e3053 100644 --- a/src/lib/ngsi/ContextAttribute.cpp +++ b/src/lib/ngsi/ContextAttribute.cpp @@ -1144,11 +1144,11 @@ std::string ContextAttribute::toJson(const std::vector& metadataFi filterAndOrderMetadata(metadataFilter, &orderedMetadata); // - // metadata (note that ngsi field in custom notifications avoids empety metadata array, i.e. "metadata": {}) + // metadata (note that ngsi field in custom notifications avoids empty metadata array, i.e. "metadata": {}) // if ((!renderNgsiField) || (metadataVector.size() > 0)) { - jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); + jh.addRaw("metadata", metadataVector.toJson(orderedMetadata)); } return jh.str(); @@ -1185,9 +1185,6 @@ std::string ContextAttribute::toJsonValue(ExprContextObject* exprContextObjectP) } else if (valueType == orion::ValueTypeString) { - //std::string out = "\""; - //out += toJsonString(stringValue); - //out += '"'; return smartStringValue(stringValue, exprContextObjectP, "null"); } else if (valueType == orion::ValueTypeBoolean) From 4698f6474ee1bbce113f4d369ce1cf8707851150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 3 Jul 2024 13:33:04 +0200 Subject: [PATCH 344/390] Update doc/manuals/orion-api.md Co-authored-by: mapedraza <40356341+mapedraza@users.noreply.github.com> --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index a32334b2aa..1c1bbc1961 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2539,7 +2539,7 @@ In order to overcome this problem, the `evalPriority` metadata can be used to de * `evalPriority` metadata is a number from 1 (first evaluation) to 100000 (last evaluation) * Expressions are evaluated in incresing order of priority -* In case of ties, Orion does not guarantee a particular evaluation order. Thus, expressions in the same priority level must be considered independent or bad things would happen +* In case of ties, Orion does not guarantee a particular evaluation order. Thus, expressions in the same priority level must be considered independent, and expressions using other attributes with the same or lower priority would result in unexpected values. * If no `evalPriority` is set, default 100000 is used * `evalPriority` only has a meaning in `notification.httpCustom.ngsi` in subscriptions. As metadata in regular entities (e.g. an entity created with `POST /v2/entities`) Orion doesn't implements any semantic for it From bdb93e8395d1b2ed8c5609081d224fc470fc5519 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Thu, 4 Jul 2024 18:53:09 +0900 Subject: [PATCH 345/390] (JP) ADD doc about evalPriority feature (#4590) --- doc/manuals.jp/orion-api.md | 64 +++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index 9bfc131b41..eb1e656bbc 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -75,6 +75,7 @@ - [その他の考慮事項](#additional-considerations) - [JEXL サポート (JEXL Support)](#jexl-support) - [JEXL 使用例 (JEXL usage example)](#jexl-usage-example) + - [評価の優先順位](#evaluation-priority) - [利用可能な変換 (Available Transformations)](#available-transformations) - [`uppercase`](#uppercase) - [`lowercase`](#lowercase) @@ -716,6 +717,9 @@ Orion は階層スコープをサポートしているため、エンティテ - `DateTime` - `geo:json` +- `evalPriority`: 式の評価で使用されます。詳細については、[この特定のセクション](#evaluation-priority)を参照してください + + 現時点では、'ignoreType' は地理位置情報タイプに対してのみサポートされており、この方法により、エンティティごとに1つの 地理位置情報のみという制限を克服するメカニズムが可能になります (詳細は [エンティティの地理空間プロパティ](#geospatial-properties-of-entities)のセクションを参照)。`DateTime` での @@ -2510,6 +2514,66 @@ Orion は、この機能を提供するために cjexl ライブラリに依存 ] ``` + + +### 評価の優先順位 + +式が評価されるたびに、その式は式のコンテキストに追加され、他の式で再利用できるようになります。ただし、デフォルトでは、Orion は特定の評価を保証しません。 + +したがって、次の式があるとします: + +``` +"httpCustom": { + ... + "ngsi": { + "A": { + "value": "${16|sqrt}", + "type": "Calculated" + }, + "B": { + "value": "${A/100}", + "type": "Calculated" + } + }, + "attrs": [ "B" ] +} +``` + +結果の通知では、`B` は、希望する `0.04` (`A` が `B` の前に評価される場合) または、過小評価された `null` (`B` が `A` の前に評価される場合) にランダムに設定される可能性があります。 + +この問題を克服するために、`evalPriority` メタデータを使用して評価順序を定義できます。これは次のように機能します: + +- `evalPriority` メタデータは、1 (最初の評価) から 100000 (最後の評価) までの数値です +- 式は優先度の昇順で評価されます +- 同順位の場合、Orion は特定の評価順序を保証しません。したがって、同じ優先度レベルの式は独立していると見なす必要があり、同じまたはより低い優先度の他の属性を使用する式は予期しない値になります +- `evalPriority` が設定されていない場合は、デフォルトの 100000 が使用されます +- `evalPriority` は、サブスクリプションの `notification.httpCustom.ngsi` でのみ意味を持ちます。通常のエンティティのメタデータ (`POST /v2/entities` で作成されたエンティティなど) として、Orion はそれに対するセマンティクスを実装しません + +`evalPriority` を使用すると、上記の例は次のように書き直すことができます: + +``` +"httpCustom": { + ... + "ngsi": { + "A": { + "value": "${16|sqrt}", + "type": "Calculated", + "metadata": { + "evalPriority": { + "value": 1, + "type": "Number" + } + } + }, + "B": { + "value": "${A/100}", + "type": "Calculated" + } + }, + "attrs": [ "B" ] +} +``` + ### 利用可能な変換 (Available Transformations) From 6b45bcd49177d1afa82a322c6d76906ffe49f8c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 4 Jul 2024 12:52:54 +0200 Subject: [PATCH 346/390] FIX improve Docker building process --- .github/workflows/publishimage-master.yml | 2 +- .github/workflows/publishimage-tag.yml | 2 +- docker/Dockerfile | 6 ++---- docker/Dockerfile.alpine | 6 ++---- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index 64ceb47bcf..c1d42e8662 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -26,7 +26,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TEF_TOKEN }} - name: Build Docker image - run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --secret id=repo_token,env=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . - name: Push Docker image run: docker push telefonicaiot/fiware-orion:latest diff --git a/.github/workflows/publishimage-tag.yml b/.github/workflows/publishimage-tag.yml index 5dde6e8d27..0218590d03 100644 --- a/.github/workflows/publishimage-tag.yml +++ b/.github/workflows/publishimage-tag.yml @@ -30,7 +30,7 @@ jobs: run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - name: Build Docker image - run: docker build -t telefonicaiot/fiware-orion:${{ env.VERSION }} --build-arg GIT_REV_ORION=${{ env.VERSION }} --build-arg REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + run: docker build -t telefonicaiot/fiware-orion:${{ env.VERSION }} --build-arg GIT_REV_ORION=${{ env.VERSION }} --secret id=repo_token,env=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . - name: Push Docker image run: docker push telefonicaiot/fiware-orion:${{ env.VERSION }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 463a4e3034..0b2ad28fa7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -29,19 +29,17 @@ ARG GITHUB_REPOSITORY=fiware-orion ARG GIT_NAME ARG GIT_REV_ORION ARG CLEAN_DEV_TOOLS -ARG REPO_ACCESS_TOKEN ENV ORION_USER ${ORION_USER:-orion} ENV GIT_NAME ${GIT_NAME:-telefonicaid} ENV GIT_REV_ORION ${GIT_REV_ORION:-master} ENV CLEAN_DEV_TOOLS ${CLEAN_DEV_TOOLS:-1} -ENV REPO_ACCESS_TOKEN ${REPO_ACCESS_TOKEN:-""} SHELL ["/bin/bash", "-o", "pipefail", "-c"] WORKDIR /opt -RUN \ +RUN --mount=type=secret,id=repo_token,dst=/run/secrets/repo_token \ # Install security updates apt-get -y update && \ apt-get -y upgrade && \ @@ -104,7 +102,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.3.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.3.0 $(cat /run/secrets/repo_token) && \ make && \ make install && \ # reduce size of installed binaries diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 736f2dbf3e..552b903108 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -32,19 +32,17 @@ ARG GITHUB_REPOSITORY=fiware-orion ARG GIT_NAME ARG GIT_REV_ORION ARG CLEAN_DEV_TOOLS -ARG REPO_ACCESS_TOKEN ENV ORION_USER ${ORION_USER:-orion} ENV GIT_NAME ${GIT_NAME:-telefonicaid} ENV GIT_REV_ORION ${GIT_REV_ORION:-master} ENV CLEAN_DEV_TOOLS ${CLEAN_DEV_TOOLS:-1} -ENV REPO_ACCESS_TOKEN ${REPO_ACCESS_TOKEN:-""} SHELL ["/bin/ash", "-o", "pipefail", "-c"] WORKDIR /opt -RUN \ +RUN --mount=type=secret,id=repo_token,dst=/run/secrets/repo_token \ # Install dependencies apk add --no-cache \ curl \ @@ -110,7 +108,7 @@ RUN \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.3.0 ${REPO_ACCESS_TOKEN} && \ + bash get_cjexl.sh 0.3.0 $(cat /run/secrets/repo_token) && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ From e60106f807a4b586284bc37b4e601e7c39cad165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 4 Jul 2024 14:10:03 +0200 Subject: [PATCH 347/390] FIX upgrade to debian 12.6 --- CHANGES_NEXT_RELEASE | 3 ++- ci/deb/Dockerfile | 2 +- doc/manuals.jp/admin/build_source.md | 2 +- doc/manuals.jp/admin/install.md | 2 +- doc/manuals/admin/build_source.md | 2 +- doc/manuals/admin/install.md | 2 +- docker/Dockerfile | 2 +- docker/README.jp.md | 14 +++++++------- docker/README.md | 14 +++++++------- docker/docker-compose.yml | 2 +- 10 files changed, 23 insertions(+), 22 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index fa20464f70..656f28c5a6 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ - Fix: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) - Fix: wrong date values should not allowed in subscription's expires field (#4541) -- Fix: do not raise DB alarm in case of wrong GeoJSON in client request \ No newline at end of file +- Fix: do not raise DB alarm in case of wrong GeoJSON in client request +- Upgrade Debian version from 12.4 to 12.6 in Dockerfile \ No newline at end of file diff --git a/ci/deb/Dockerfile b/ci/deb/Dockerfile index 93c38a5cf9..afe1fbe778 100644 --- a/ci/deb/Dockerfile +++ b/ci/deb/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:12.4-slim +FROM debian:12.6-slim ADD build.sh /opt/bin/ ADD build-dep.sh /opt/bin/ diff --git a/doc/manuals.jp/admin/build_source.md b/doc/manuals.jp/admin/build_source.md index b5464ae4ea..54e7c19e36 100644 --- a/doc/manuals.jp/admin/build_source.md +++ b/doc/manuals.jp/admin/build_source.md @@ -13,7 +13,7 @@ Orion Context Broker は、以下のライブラリをビルドの依存関係 * boost: 1.74 * libmicrohttpd: 0.9.76 (ソースから) * libcurl: 7.88.1 -* openssl: 3.0.9 +* openssl: 3.0.13 * libuuid: 2.38.1 * libmosquitto: 2.0.15 (ソースから) * Mongo C driver: 1.24.3 (ソースから) diff --git a/doc/manuals.jp/admin/install.md b/doc/manuals.jp/admin/install.md index 07cfba626f..06a4c2a5c7 100644 --- a/doc/manuals.jp/admin/install.md +++ b/doc/manuals.jp/admin/install.md @@ -26,7 +26,7 @@ Docker hub で公式の Orion docker コンテナを使用してインストー 必要なソースから ビルドした Orion をインストールする場合： -* オペレーティングシステム: Debian。リファレンス・オペレーティングシステムは Debian 12.4 ですが、それ以降の +* オペレーティングシステム: Debian。リファレンス・オペレーティングシステムは Debian 12.6 ですが、それ以降の Debian 12 バージョンでも動作するはずです * データベース: MongoDB は、Orion Context Broker がインストールされるのと同じホストで実行するか、ネットワーク経由で アクセスできる別のホストで実行する必要があります。推奨される MongoDB バージョンは 6.0 です (Orion は古いバージョンで diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index 3292e71d32..e86a146047 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -13,7 +13,7 @@ The Orion Context Broker uses the following libraries as build dependencies: * boost: 1.74 * libmicrohttpd: 0.9.76 (from source) * libcurl: 7.88.1 -* openssl: 3.0.9 +* openssl: 3.0.13 * libuuid: 2.38.1 * libmosquitto: 2.0.15 (from source) * Mongo C driver: 1.24.3 (from source) diff --git a/doc/manuals/admin/install.md b/doc/manuals/admin/install.md index 5efbe8d65c..f8ef9489f5 100644 --- a/doc/manuals/admin/install.md +++ b/doc/manuals/admin/install.md @@ -23,7 +23,7 @@ In the case you install using the official Orion docker container at Dockerhub, In the case you are installing Orion building from sources you need: -* Operating system: Debian. The reference operating system is Debian 12.4 +* Operating system: Debian. The reference operating system is Debian 12.6 but it should work also in any later Debian 12 version. * Database: MongoDB is required to run either in the same host where Orion Context Broker is to be installed or in a different host accessible through the network. The recommended MongoDB version is 6.0 (Orion may work with older versions but we don't recommend it at all!). diff --git a/docker/Dockerfile b/docker/Dockerfile index 0b2ad28fa7..73d0501729 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ # ARG IMAGE_NAME=debian -ARG IMAGE_TAG=12.4-slim +ARG IMAGE_TAG=12.6-slim FROM ${IMAGE_NAME}:${IMAGE_TAG} ARG GITHUB_ACCOUNT=telefonicaid diff --git a/docker/README.jp.md b/docker/README.jp.md index e1080f2888..bd77c4f967 100644 --- a/docker/README.jp.md +++ b/docker/README.jp.md @@ -28,7 +28,7 @@ Orion Context Broker を試してみたいし、データベースについて services: orion: - image: fiware/orion + image: telefonicaiot/fiware-orion ports: - "1026:1026" depends_on: @@ -49,7 +49,7 @@ Orion Context Broker を試してみたいし、データベースについて curl localhost:1026/version -この方法で行ったことは、[Docker Hub](https://hub.docker.com/) というイメージの公開リポジトリから [Orion Context Broker](https://hub.docker.com/r/fiware/orion/) と [MongoDB](https://hub.docker.com/_/mongo/) 用のイメージをダウンロードすることです。次に、両方のイメージに基づいて2つのコンテナを作成しました。 +この方法で行ったことは、[Docker Hub](https://hub.docker.com/) というイメージの公開リポジトリから [Orion Context Broker](https://hub.docker.com/r/telefonicaiot/fiware-orion/) と [MongoDB](https://hub.docker.com/_/mongo/) 用のイメージをダウンロードすることです。次に、両方のイメージに基づいて2つのコンテナを作成しました。 このシナリオを停止したい場合は、docker-compose が実行されているターミナルで Control+C を押す必要があります。このメソッドを使用して Orion で使用されていたデータはすべて失われます。 @@ -60,13 +60,13 @@ Orion Context Broker を試してみたいし、データベースについて > ヒント : これらの方法を試しているか、複数回実行していて、コンテナがすでに存在しているというエラーが表示された場合は、`docker rm orion1` で削除できます。コンテナを停止しなければならない場合は、まず `docker stop orion1` を実行して、止めてください。 -これらのコマンドを使用すると、Orion のタグと特定のバージョンにアクセスできます。たとえば、特定のバージョンが必要な場合は、次のコマンドで `fiware/orion` の代わりに `fiware/orion:0.22` を使用できます。バージョンを指定しない場合は、デフォルトで ` 最新 ` のものから取得します。 +これらのコマンドを使用すると、Orion のタグと特定のバージョンにアクセスできます。たとえば、特定のバージョンが必要な場合は、次のコマンドで `telefonicaiot/fiware-orion` の代わりに `telefonicaiot/fiware-orion:4.0.0` を使用できます。バージョンを指定しない場合は、デフォルトで ` 最新 ` のものから取得します。 ### 2A. MongoDB はローカルホスト上にある場合 これを実行するには、このコマンドを実行します。 - sudo docker run -d --name orion1 -p 1026:1026 fiware/orion + sudo docker run -d --name orion1 -p 1026:1026 telefonicaiot/fiware-orion すべてが動作することを確認します。 @@ -79,7 +79,7 @@ Orion Context Broker を試してみたいし、データベースについて そして、このコマンドで Orion を実行します - sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 fiware/orion -dbURI mongodb://mongodb + sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 telefonicaiot/fiware-orion -dbURI mongodb://mongodb すべてが動作することを確認します。 @@ -91,7 +91,7 @@ Orion Context Broker を試してみたいし、データベースについて 別の MongoDB インスタンスに接続する場合は、前のコマンドの**代わりに**、次のコマンドを実行します - sudo docker run -d --name orion1 -p 1026:1026 fiware/orion -dbURI mongodb:// + sudo docker run -d --name orion1 -p 1026:1026 telefonicaiot/fiware-orion -dbURI mongodb:// すべてが動作することを確認します。 @@ -186,6 +186,6 @@ sudo を使用したくない場合は、[以下の手順](http://askubuntu.com/ ### 4.4 Orion の追加パラメータ -コンテナイメージの名前 (ビルドしている場合は `orion`、リポジトリからプルしている場合は `fiware/orion`) の後ろは、Orion Context Broker のパラメータとして解釈されます。ここでは、MongoDB ホストがどこにあるかを broker に伝えています。これは、他の MongoDB コンテナの名前で表されます。他のコマンドライン・オプションの [ドキュメント](https://github.com/telefonicaid/fiware-orion)を見てください。 +コンテナイメージの名前 (ビルドしている場合は `orion`、リポジトリからプルしている場合は `telefonicaiot/fiware-orion`) の後ろは、Orion Context Broker のパラメータとして解釈されます。ここでは、MongoDB ホストがどこにあるかを broker に伝えています。これは、他の MongoDB コンテナの名前で表されます。他のコマンドライン・オプションの [ドキュメント](https://github.com/telefonicaid/fiware-orion)を見てください。 Orion は [マルチテナントモード](https://fiware-orion.readthedocs.io/en/master/user/multitenancy/index.html)で動作します。 diff --git a/docker/README.md b/docker/README.md index 31f7ccf9a1..0a3e30fdf6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -30,7 +30,7 @@ Follow these steps: services: orion: - image: fiware/orion + image: telefonicaiot/fiware-orion ports: - "1026:1026" depends_on: @@ -51,7 +51,7 @@ Check that everything works with curl localhost:1026/version -What you have done with this method is download images for [Orion Context Broker](https://hub.docker.com/r/fiware/orion/) and [MongoDB](https://hub.docker.com/_/mongo/) from the public repository of images called [Docker Hub](https://hub.docker.com/). Then you have created two containers based on both images. +What you have done with this method is download images for [Orion Context Broker](https://hub.docker.com/r/telefonicaiot/fiware-orion/) and [MongoDB](https://hub.docker.com/_/mongo/) from the public repository of images called [Docker Hub](https://hub.docker.com/). Then you have created two containers based on both images. If you want to stop the scenario you have to press Control+C on the terminal where docker-compose is running. Note that you will lose any data that was being used in Orion using this method. @@ -62,13 +62,13 @@ This method will launch a container running Orion Context Broker, but it is up t > TIP: If you are trying these methods or run them more than once and come across an error saying that the container already exists you can delete it with `docker rm orion1`. If you have to stop it first do `docker stop orion1`. -Keep in mind that if you use these commands you get access to the tags and specific versions of Orion. For example, you may use `fiware/orion:0.22` instead of `fiware/orion` in the following commands if you need that particular version. If you do not specify a version you are pulling from `latest` by default. +Keep in mind that if you use these commands you get access to the tags and specific versions of Orion. For example, you may use `telefonicaiot/fiware-orion:4.0.0` instead of `telefonicaiot/fiware-orion` in the following commands if you need that particular version. If you do not specify a version you are pulling from `latest` by default. ### 2A. MongoDB is on localhost To do this run this command - sudo docker run -d --name orion1 -p 1026:1026 fiware/orion + sudo docker run -d --name orion1 -p 1026:1026 telefonicaiot/fiware-orion Check that everything works with @@ -81,7 +81,7 @@ In case you want to run MongoDB on another container you can launch it like this And then run Orion with this command - sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 fiware/orion -dbURI mongodb://mongodb + sudo docker run -d --name orion1 --link mongodb:mongodb -p 1026:1026 telefonicaiot/fiware-orion -dbURI mongodb://mongodb Check that everything works with @@ -93,7 +93,7 @@ This method is functionally equivalent as the one described in section 1, but do If you want to connect to a different MongoDB instance do the following command **instead of** the previous one - sudo docker run -d --name orion1 -p 1026:1026 fiware/orion -dbURI mongodb:// + sudo docker run -d --name orion1 -p 1026:1026 telefonicaiot/fiware-orion -dbURI mongodb:// Check that everything works with @@ -189,7 +189,7 @@ on your machine you should change this value to something else, for example `-p ### 4.4 Extra parameters for Orion -Anything after the name of the container image (`orion` if you are building, or `fiware/orion` if you are pulling from the repository) is interpreted as a parameter for the Orion Context Broker. In this case we are telling the broker where the MongoDB host is, represented by the name of our other MongoDB container. Take a look at the [documentation](https://github.com/telefonicaid/fiware-orion) for other command-line options. +Anything after the name of the container image (`orion` if you are building, or `telefonicaiot/fiware-orion` if you are pulling from the repository) is interpreted as a parameter for the Orion Context Broker. In this case we are telling the broker where the MongoDB host is, represented by the name of our other MongoDB container. Take a look at the [documentation](https://github.com/telefonicaid/fiware-orion) for other command-line options. Orion will be running on [multi-tenant](https://fiware-orion.readthedocs.io/en/master/user/multitenancy/index.html) mode. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 81b78922e7..ccfbda083b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: orion: - image: fiware/orion + image: telefonicaiot/fiware-orion ports: - "1026:1026" depends_on: From 6c079951887be5435e584f117b8aa937e126aaf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 10 Jul 2024 11:53:44 +0200 Subject: [PATCH 348/390] ADD jexl_transformation_substring.test test case --- .../jexl_transformation_substring.test | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_substring.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_substring.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_substring.test new file mode 100644 index 0000000000..bd9f96db3e --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_substring.test @@ -0,0 +1,234 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +JEXL expression substring specific cases + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression A=S|substring(0,2), B=S|substring(3,5), C=S|substring(3,50), D=S|substring(5,3) +# 02. Create entity E with S=X1:X2 +# 03. Update entity E with S=Y1:Y2 +# 04. Dump accumulator and see notifications (A=X1,B=X2,C=null,D=null) (A=Y1,B=Y2,C=null,D=null) +# + + +echo "01. Create custom sub with custom expression A=S|substring(0,2), B=S|substring(3,5), C=S|substring(3,50), D=S|substring(5,3)" +echo "============================================================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "A": { + "value": "${S|substring(0,2)}", + "type": "Calculated" + }, + "B": { + "value": "${S|substring(3,5)}", + "type": "Calculated" + }, + "C": { + "value": "${S|substring(3,50)}", + "type": "Calculated" + }, + "D": { + "value": "${S|substring(5,3)}", + "type": "Calculated" + } + } + }, + "attrs": [ "A", "B", "C", "D" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E with S=X1:X2" +echo "================================" +payload='{ + "id": "E", + "type": "T", + "S": { + "value": "X1:X2", + "type": "Text" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E with S=Y1:Y2" +echo "================================" +payload='{ + "S": { + "value": "Y1:Y2", + "type": "Text" + } +}' +orionCurl --url /v2/entities/E/attrs --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2)" +echo "=================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression A=S|substring(0,2), B=S|substring(3,5), C=S|substring(3,50), D=S|substring(5,3) +============================================================================================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E with S=X1:X2 +================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. Update entity E with S=Y1:Y2 +================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (E1/T1 A:1, E2/T2 A:2) +================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: x +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Calculated", + "value": "X1" + }, + "B": { + "metadata": {}, + "type": "Calculated", + "value": "X2" + }, + "C": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "D": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: x +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Calculated", + "value": "Y1" + }, + "B": { + "metadata": {}, + "type": "Calculated", + "value": "Y2" + }, + "C": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "D": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 7280faa5faa2e8aefa7615bdbd6f481cce23201b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 10 Jul 2024 13:11:56 +0200 Subject: [PATCH 349/390] FIX substring transformation docu --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 1c1bbc1961..b1431b8e93 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -2684,7 +2684,7 @@ foo bar #### substring -Returns a substring between two positions. +Returns a substring between two positions or `null` in case of wrong parameters (eg. final position is longer than string, final position is leeser than initial position, etc.) Extra arguments: * Initial position From e430828a1d621d2aa7f6cf9554a3bfeb5e2c1bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 10 Jul 2024 13:12:16 +0200 Subject: [PATCH 350/390] FIX step cjexl version --- ci/deb/build.sh | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.alpine | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/deb/build.sh b/ci/deb/build.sh index 6202177dbb..1bc9150e16 100755 --- a/ci/deb/build.sh +++ b/ci/deb/build.sh @@ -175,7 +175,7 @@ rm -Rf /tmp/builder || true && mkdir -p /tmp/builder/{db1,db2,db,bu} if [ -z "${REPO_ACCESS_TOKEN}" ]; then echo "Builder: no REPO_ACCESS_TOKEN, skipping cjexl lib download" else - bash /opt/fiware-orion/get_cjexl.sh 0.3.0 $REPO_ACCESS_TOKEN + bash /opt/fiware-orion/get_cjexl.sh 0.4.0 $REPO_ACCESS_TOKEN fi if [ -n "${branch}" ]; then diff --git a/docker/Dockerfile b/docker/Dockerfile index 73d0501729..aac4d0b52f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -102,7 +102,7 @@ RUN --mount=type=secret,id=repo_token,dst=/run/secrets/repo_token \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.3.0 $(cat /run/secrets/repo_token) && \ + bash get_cjexl.sh 0.4.0 $(cat /run/secrets/repo_token) && \ make && \ make install && \ # reduce size of installed binaries diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 552b903108..884c9cf1ef 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -108,7 +108,7 @@ RUN --mount=type=secret,id=repo_token,dst=/run/secrets/repo_token \ git clone https://github.com/${GIT_NAME}/fiware-orion && \ cd fiware-orion && \ git checkout ${GIT_REV_ORION} && \ - bash get_cjexl.sh 0.3.0 $(cat /run/secrets/repo_token) && \ + bash get_cjexl.sh 0.4.0 $(cat /run/secrets/repo_token) && \ # patch bash and mktemp statement in build script, as in alpine is slightly different sed -i 's/mktemp \/tmp\/compileInfo.h.XXXX/mktemp/g' scripts/build/compileInfo.sh && \ sed -i 's/bash/ash/g' scripts/build/compileInfo.sh && \ From bdf3d43de38dda99849f29b8be65230c95e32dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 19 Jul 2024 09:34:13 +0200 Subject: [PATCH 351/390] FIX image build gitactions --- .github/workflows/publishimage-master.yml | 4 +++- .github/workflows/publishimage-tag.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index c1d42e8662..0be42eef6e 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -26,7 +26,9 @@ jobs: password: ${{ secrets.DOCKERHUB_TEF_TOKEN }} - name: Build Docker image - run: docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --secret id=repo_token,env=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + run: + export TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} + docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --secret id=repo_token,env=TOKEN --no-cache -f docker/Dockerfile . - name: Push Docker image run: docker push telefonicaiot/fiware-orion:latest diff --git a/.github/workflows/publishimage-tag.yml b/.github/workflows/publishimage-tag.yml index 0218590d03..4ff10ad638 100644 --- a/.github/workflows/publishimage-tag.yml +++ b/.github/workflows/publishimage-tag.yml @@ -30,7 +30,9 @@ jobs: run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - name: Build Docker image - run: docker build -t telefonicaiot/fiware-orion:${{ env.VERSION }} --build-arg GIT_REV_ORION=${{ env.VERSION }} --secret id=repo_token,env=${{ secrets.REPO_ACCESS_TOKEN }} --no-cache -f docker/Dockerfile . + run: + export TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} + docker build -t telefonicaiot/fiware-orion:${{ env.VERSION }} --build-arg GIT_REV_ORION=${{ env.VERSION }} --secret id=repo_token,env=TOKEN --no-cache -f docker/Dockerfile . - name: Push Docker image run: docker push telefonicaiot/fiware-orion:${{ env.VERSION }} From bcad605ce7f1ddeb9eb1f1e04b28d8e9b3607443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 19 Jul 2024 10:46:00 +0200 Subject: [PATCH 352/390] FIX temporal test --- .github/workflows/publishimage-master.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index 0be42eef6e..c28487ce23 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -6,7 +6,8 @@ name: Publish Docker image (master) on: push: branches: - - master + # FIXME: temporal test + - hardening/build-gitactions jobs: build-and-push: From 522a429b8e7a4b016ec8733392e00774fee2839f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 19 Jul 2024 10:51:27 +0200 Subject: [PATCH 353/390] FIX gitactions syntax --- .github/workflows/publishimage-master.yml | 2 +- .github/workflows/publishimage-tag.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index c28487ce23..5e7ada2398 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -27,7 +27,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TEF_TOKEN }} - name: Build Docker image - run: + run: | export TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} docker build -t telefonicaiot/fiware-orion:latest --build-arg GIT_REV_ORION=master --secret id=repo_token,env=TOKEN --no-cache -f docker/Dockerfile . diff --git a/.github/workflows/publishimage-tag.yml b/.github/workflows/publishimage-tag.yml index 4ff10ad638..1dd80f8f95 100644 --- a/.github/workflows/publishimage-tag.yml +++ b/.github/workflows/publishimage-tag.yml @@ -30,7 +30,7 @@ jobs: run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - name: Build Docker image - run: + run: | export TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} docker build -t telefonicaiot/fiware-orion:${{ env.VERSION }} --build-arg GIT_REV_ORION=${{ env.VERSION }} --secret id=repo_token,env=TOKEN --no-cache -f docker/Dockerfile . From 049b0c9f3be9c76c7cd265bb6dfe34f5528a61f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 19 Jul 2024 14:44:22 +0200 Subject: [PATCH 354/390] Update publishimage-master.yml --- .github/workflows/publishimage-master.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index 5e7ada2398..1e056970fb 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -5,9 +5,8 @@ name: Publish Docker image (master) on: push: - branches: - # FIXME: temporal test - - hardening/build-gitactions + branches: + - master jobs: build-and-push: From 7e0af0633161ba3333003ee0201a29a6a9266e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 19 Jul 2024 14:44:37 +0200 Subject: [PATCH 355/390] Update publishimage-master.yml --- .github/workflows/publishimage-master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publishimage-master.yml b/.github/workflows/publishimage-master.yml index 1e056970fb..df3d93908d 100644 --- a/.github/workflows/publishimage-master.yml +++ b/.github/workflows/publishimage-master.yml @@ -5,7 +5,7 @@ name: Publish Docker image (master) on: push: - branches: + branches: - master jobs: From 132e364f4d291366bfd80c8823adbbae54513904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 10:43:01 +0200 Subject: [PATCH 356/390] ADD 4004 new cases --- .../jexl_transformation_full.test | 46 +++- .../jexl_transformation_substring.test | 4 +- ...l_transformation_time_transformations.test | 214 +++++++++++++++++ ...mation_time_transformations_datetimes.test | 215 ++++++++++++++++++ 4 files changed, 475 insertions(+), 4 deletions(-) create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_time_transformations.test create mode 100644 test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_time_transformations_datetimes.test diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test index 12646463e7..10ba03a673 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_full.test @@ -81,7 +81,10 @@ payload='{ "AA", "AB", "AC", - "AD" + "AD", + "AE", + "AF", + "AG" ], "httpCustom": { "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", @@ -205,6 +208,18 @@ payload='{ "AD": { "type": "Number", "value": "${AD|len}" + }, + "AE": { + "type": "Number", + "value": "${AE|now}" + }, + "AF": { + "type": "Text", + "value": "${AF|toIsoString}" + }, + "AG": { + "type": "Number", + "value": "${AG|getTime}" } } } @@ -344,6 +359,18 @@ payload='{ "AD": { "type": "StructuredValue", "value": [1, 2, 3] + }, + "AE": { + "type": "Number", + "value": 0 + }, + "AF": { + "type": "Number", + "value": 1722500129 + }, + "AG": { + "type": "Number", + "value": "1979-11-13T18:01:14+00:00" } }' orionCurl --url /v2/entities --payload "$payload" @@ -383,7 +410,7 @@ Content-Length: 0 ================================================== POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: 1627 +Content-Length: 1809 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -422,6 +449,21 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 "type": "Number", "value": 3 }, + "AE": { + "metadata": {}, + "type": "Number", + "value": REGEX(\d+) + }, + "AF": { + "metadata": {}, + "type": "Text", + "value": "2024-08-01T08:15:29+00:00" + }, + "AG": { + "metadata": {}, + "type": "Number", + "value": 311364074 + }, "B": { "metadata": {}, "type": "Text", diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_substring.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_substring.test index bd9f96db3e..d2e40950f3 100644 --- a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_substring.test +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_substring.test @@ -148,7 +148,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) ================================================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: x +Content-Length: 288 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) @@ -188,7 +188,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 ======================================= POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / -Content-Length: x +Content-Length: 288 User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) Ngsiv2-Attrsformat: normalized Host: 127.0.0.1:REGEX(\d+) diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_time_transformations.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_time_transformations.test new file mode 100644 index 0000000000..e5aedcdf6f --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_time_transformations.test @@ -0,0 +1,214 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +JEXL expression time transformation specific cases + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression S=A|getTime|toIsoString, T=B|toIsoString|getTime +# 02. Create entity E with A=2060-02-04T23:28:10+00:00, B=1722501038 +# 03. Update entity E with A=2040-03-12T21:04:13+00:00, B=122501038 +# 04. Dump accumulator and see notifications (S=2060-02-04T23:28:10+00:00, T=1722501038) (S=2040-03-12T21:04:13+00:00, T=122501038) +# + + +echo "01. Create custom sub with custom expression S=A|getTime|toIsoString, T=B|toIsoString|getTime" +echo "=============================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${A|getTime|toIsoString}", + "type": "Calculated" + }, + "T": { + "value": "${B|toIsoString|getTime}", + "type": "Calculated" + } + } + }, + "attrs": [ "S", "T" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E with A=2060-02-04T23:28:10+00:00, B=1722501038" +echo "==================================================================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": "2060-02-04T23:28:10+00:00", + "type": "Text" + }, + "B": { + "value": 1722501038, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E with A=2040-03-12T21:04:13+00:00, B=122501038" +echo "=================================================================" +payload='{ + "A": { + "value": "2040-03-12T21:04:13+00:00", + "type": "Text" + }, + "B": { + "value": 122501038, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (S=2060-02-04T23:28:10+00:00, T=1722501038) (S=2040-03-12T21:04:13+00:00, T=122501038)" +echo "=================================================================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression S=A|getTime|toIsoString, T=B|toIsoString|getTime +============================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E with A=2060-02-04T23:28:10+00:00, B=1722501038 +================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. Update entity E with A=2040-03-12T21:04:13+00:00, B=122501038 +================================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (S=2060-02-04T23:28:10+00:00, T=1722501038) (S=2040-03-12T21:04:13+00:00, T=122501038) +================================================================================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 211 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "S": { + "metadata": {}, + "type": "Calculated", + "value": "2060-02-04T23:28:10+00:00" + }, + "T": { + "metadata": {}, + "type": "Calculated", + "value": 1722501038 + }, + "id": "E", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 210 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "S": { + "metadata": {}, + "type": "Calculated", + "value": "2040-03-12T21:04:13+00:00" + }, + "T": { + "metadata": {}, + "type": "Calculated", + "value": 122501038 + }, + "id": "E", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_time_transformations_datetimes.test b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_time_transformations_datetimes.test new file mode 100644 index 0000000000..4166851546 --- /dev/null +++ b/test/functionalTest/cases/4004_jexl_expressions_in_subs/jexl_transformation_time_transformations_datetimes.test @@ -0,0 +1,215 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +JEXL expression time transformation specific cases + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# Same test as jexl_transformation_time_transformations but using "DateTime" instead of "Text" for A attribute +# +# 01. Create custom sub with custom expression S=A|getTime|toIsoString, T=B|toIsoString|getTime +# 02. Create entity E with A=2060-02-04T23:28:10+00:00, B=1722501038 +# 03. Update entity E with A=2040-03-12T21:04:13+00:00, B=122501038 +# 04. Dump accumulator and see notifications (S=2060-02-04T23:28:10+00:00, T=1722501038) (S=2040-03-12T21:04:13+00:00, T=122501038) +# + + +echo "01. Create custom sub with custom expression S=A|getTime|toIsoString, T=B|toIsoString|getTime" +echo "=============================================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "S": { + "value": "${A|getTime|toIsoString}", + "type": "Calculated" + }, + "T": { + "value": "${B|toIsoString|getTime}", + "type": "Calculated" + } + } + }, + "attrs": [ "S", "T" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E with A=2060-02-04T23:28:10+00:00, B=1722501038" +echo "==================================================================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": "2060-02-04T23:28:10+00:00", + "type": "DateTime" + }, + "B": { + "value": 1722501038, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E with A=2040-03-12T21:04:13+00:00, B=122501038" +echo "=================================================================" +payload='{ + "A": { + "value": "2040-03-12T21:04:13+00:00", + "type": "DateTime" + }, + "B": { + "value": 122501038, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E/attrs --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see notifications (S=2060-02-04T23:28:10+00:00, T=1722501038) (S=2040-03-12T21:04:13+00:00, T=122501038)" +echo "=================================================================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression S=A|getTime|toIsoString, T=B|toIsoString|getTime +============================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E with A=2060-02-04T23:28:10+00:00, B=1722501038 +================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +03. Update entity E with A=2040-03-12T21:04:13+00:00, B=122501038 +================================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see notifications (S=2060-02-04T23:28:10+00:00, T=1722501038) (S=2040-03-12T21:04:13+00:00, T=122501038) +================================================================================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 211 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "S": { + "metadata": {}, + "type": "Calculated", + "value": "2060-02-04T23:28:10+00:00" + }, + "T": { + "metadata": {}, + "type": "Calculated", + "value": 1722501038 + }, + "id": "E", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 210 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "S": { + "metadata": {}, + "type": "Calculated", + "value": "2040-03-12T21:04:13+00:00" + }, + "T": { + "metadata": {}, + "type": "Calculated", + "value": 122501038 + }, + "id": "E", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 33966a9c27dc72a50c5ef367930beeba97d089ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 11:29:06 +0200 Subject: [PATCH 357/390] ADD Changelog entry --- Changelog | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Changelog b/Changelog index 5b5d5ef154..9ba2453185 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,5 @@ +- Upgrafe cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString) + 4.0.0 (June 6th, 2024) - Add: JEXL expression support in custom notification macro replacement (using cjexl 0.3.0) (#4004) @@ -148,11 +150,11 @@ 3.5.0 (January 28th, 2022) -- Fix: avoid duplicated attributes and metadata in arrays in POST /v2/subscriptions and PATCH /v2/subscriptions/{subId} (#2101) -- Fix: $each usage with $push operator to add several items to array (#744) -- Fix: allow HTTPS certificates larger than 2048 bytes (#4012) -- Fix: DateTime attribute macro rendering in custom notifications was numeric instead of ISO8601 (#3894) -- Fix: buffer overflow problem in logs subsystem for from= field, causing crash in some cases (#3884) +- Fix: avoid duplicated attributes and metadata in arrays in POST /v2/subscriptions and PATCH /v2/subscriptions/{subId} (#2101) +- Fix: $each usage with $push operator to add several items to array (#744) +- Fix: allow HTTPS certificates larger than 2048 bytes (#4012) +- Fix: DateTime attribute macro rendering in custom notifications was numeric instead of ISO8601 (#3894) +- Fix: buffer overflow problem in logs subsystem for from= field, causing crash in some cases (#3884) 3.4.0 (December 14th, 2021) From 862804e36826d4379551e88f84f140854bc3ba46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 12:15:37 +0200 Subject: [PATCH 358/390] ADD doc about new transformations --- Changelog | 2 +- doc/manuals/orion-api.md | 64 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/Changelog b/Changelog index 9ba2453185..01d1808f6d 100644 --- a/Changelog +++ b/Changelog @@ -1,4 +1,4 @@ -- Upgrafe cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString) +- Upgrade cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString) 4.0.0 (June 6th, 2024) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index b1431b8e93..fe6cd30b3e 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -108,6 +108,9 @@ - [`keys`](#keys) - [`arrSum`](#arrsum) - [`arrAvg`](#arravg) + - [`now`](#now) + - [`toIsoString`](#toisostring) + - [`getTime](#gettime) - [Failsafe cases](#failsafe-cases) - [Known limitations](#known-limitations) - [Oneshot Subscriptions](#oneshot-subscriptions) @@ -3146,6 +3149,67 @@ results in 3 ``` +#### now + +Returns the current time (plus a number of seconds specified as argument) as seconds since Unix epoch time. + +Extra arguments: none + +Example (being current time August 1st, 2024 at 9:31:02): + +``` +0|new +``` + +results in + +``` +1722504662 +``` + +It can be checked at https://www.epochconverter.com that time corresponds to August 1st, 2024 at 9:31:02 + +#### toIsoString + +Returns the [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) corresponding to a given timestamp (as seconds since Unix epoch time) passed as argument. + +Extra arguments: none + +Example (being context `{"c": 1720606949}`): + +``` +c|toIsoString +``` + +results in + +``` +"2024-07-10T10:22:29+00:00" +``` + +It can be checked at https://www.epochconverter.com that 1720606949 corresponds to 2024-07-10T10:22:29+00:00 + + +#### getTime + +Returns the timestamp (as seconds since Unix epoch time) correspoding to the [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) passed as argument. + +Extra arguments: none + +Example (being context `{"c": "2024-07-10T10:22:29+00:00"}`): + +``` +c|getTime +``` + +results in + +``` +1720606949 +``` + +It can be checked at https://www.epochconverter.com that 1720606949 corresponds to 2024-07-10T10:22:29+00:00 + ### Failsafe cases As failsafe behaviour, evaluation returns `null` in the following cases: From 3446a1ad828c05946c66791bcc72da8e2c348ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 12:16:01 +0200 Subject: [PATCH 359/390] Update doc/manuals/orion-api.md --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index fe6cd30b3e..0a49543307 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -110,7 +110,7 @@ - [`arrAvg`](#arravg) - [`now`](#now) - [`toIsoString`](#toisostring) - - [`getTime](#gettime) + - [`getTime`](#gettime) - [Failsafe cases](#failsafe-cases) - [Known limitations](#known-limitations) - [Oneshot Subscriptions](#oneshot-subscriptions) From b4156bfac6fb9a0bcc078991e8a3878a0641f941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 12:18:13 +0200 Subject: [PATCH 360/390] FIX changelog --- CHANGES_NEXT_RELEASE | 1 + Changelog | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 656f28c5a6..1ba267525a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,5 @@ - Fix: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) - Fix: wrong date values should not allowed in subscription's expires field (#4541) - Fix: do not raise DB alarm in case of wrong GeoJSON in client request +- Upgrade cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString) - Upgrade Debian version from 12.4 to 12.6 in Dockerfile \ No newline at end of file diff --git a/Changelog b/Changelog index 01d1808f6d..4e6d6ab5bf 100644 --- a/Changelog +++ b/Changelog @@ -1,5 +1,3 @@ -- Upgrade cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString) - 4.0.0 (June 6th, 2024) - Add: JEXL expression support in custom notification macro replacement (using cjexl 0.3.0) (#4004) From 99a2752ce31e672dd36af6ef089cced9acb141d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 12:19:05 +0200 Subject: [PATCH 361/390] FIX Changelog file encoding --- Changelog | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Changelog b/Changelog index 4e6d6ab5bf..5b5d5ef154 100644 --- a/Changelog +++ b/Changelog @@ -148,11 +148,11 @@ 3.5.0 (January 28th, 2022) -- Fix: avoid duplicated attributes and metadata in arrays in POST /v2/subscriptions and PATCH /v2/subscriptions/{subId} (#2101) -- Fix: $each usage with $push operator to add several items to array (#744) -- Fix: allow HTTPS certificates larger than 2048 bytes (#4012) -- Fix: DateTime attribute macro rendering in custom notifications was numeric instead of ISO8601 (#3894) -- Fix: buffer overflow problem in logs subsystem for from= field, causing crash in some cases (#3884) +- Fix: avoid duplicated attributes and metadata in arrays in POST /v2/subscriptions and PATCH /v2/subscriptions/{subId} (#2101) +- Fix: $each usage with $push operator to add several items to array (#744) +- Fix: allow HTTPS certificates larger than 2048 bytes (#4012) +- Fix: DateTime attribute macro rendering in custom notifications was numeric instead of ISO8601 (#3894) +- Fix: buffer overflow problem in logs subsystem for from= field, causing crash in some cases (#3884) 3.4.0 (December 14th, 2021) From 32703aa6040c9af5bb68eba4f5b8b0ab77f86aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 12:25:44 +0200 Subject: [PATCH 362/390] Update doc/manuals/orion-api.md Co-authored-by: mapedraza <40356341+mapedraza@users.noreply.github.com> --- doc/manuals/orion-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 0a49543307..374da2024a 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -3158,7 +3158,7 @@ Extra arguments: none Example (being current time August 1st, 2024 at 9:31:02): ``` -0|new +0|now ``` results in From c622a60315517313f8dfd5ee5e7de5cfbb7bad7d Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Thu, 1 Aug 2024 19:53:03 +0900 Subject: [PATCH 363/390] (JP) ADD doc about new transformations (#4595) --- doc/manuals.jp/orion-api.md | 71 ++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index eb1e656bbc..4b81e53973 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -108,6 +108,9 @@ - [`keys`](#keys) - [`arrSum`](#arrsum) - [`arrAvg`](#arravg) + - [`now`](#now) + - [`toIsoString`](#toisostring) + - [`getTime`](#gettime) - [フェイルセーフ・ケース (Failsafe cases)](#failsafe-cases) - [既知の制限 (Known limitations)](#known-limitations) - [Oneshot サブスクリプション (Oneshot Subscriptions)](#oneshot-subscriptions) @@ -2706,7 +2709,7 @@ foo bar #### substring -2つの位置の間の部分文字列を返します。 +2つの位置の間の部分文字列を返します。パラメータが間違っている場合は `null` を返します (例: 最終位置が文字列より長い、最終位置が初期位置より短いなど)。 追加引数: * 開始位置 @@ -3216,6 +3219,72 @@ c|arrAvg 3 ``` + + +#### now + +現在の時刻（引数として指定された秒数を加えたもの）を Unix エポック時間からの秒数として返します。 + +追加の引数: なし + +例 (現在の時刻が 2024 年 8 月 1 日 9:31:02 の場合): + +``` +0|now +``` + +結果 + +``` +1722504662 +``` + +https://www.epochconverter.com で確認すると、その時間は 2024 年 8 月 1 日 9:31:02 に相当します。 + + + +#### toIsoString + +引数として渡された特定のタイムスタンプ (Unix エポック時間からの秒数) に対応する [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) を返します。 + +追加の引数: なし + +例 (コンテキスト `{"c": 1720606949}`): + +``` +c|toIsoString +``` + +結果 + +``` +"2024-07-10T10:22:29+00:00" +``` + +https://www.epochconverter.com で、1720606949 が 2024-07-10T10:22:29+00:00 に対応していることを確認できます。 + + + +#### getTime + +引数として渡された [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) に対応するタイムスタンプ (Unix エポック時間からの秒数) を返します。 + +追加引数: なし + +例 (コンテキスト `{"c": "2024-07-10T10:22:29+00:00"}`): + +``` +c|getTime +``` + +結果 + +``` +1720606949 +``` + +https://www.epochconverter.com で、1720606949 が 2024-07-10T10:22:29+00:00 に対応していることを確認できます。 + ### フェイルセーフ・ケース (Failsafe cases) From 37c7649f10cec778954b8df2008e67f8a69fffa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 13:55:24 +0200 Subject: [PATCH 364/390] FIX DateTime with min and max operators --- CHANGES_NEXT_RELEASE | 3 +- doc/manuals/orion-api.md | 3 +- src/lib/jsonParseV2/parseContextAttribute.cpp | 40 ++- .../datetime_with_min_max_operators.test | 253 ++++++++++++++++++ ...atetime_with_min_max_operators_create.test | 99 +++++++ ...atetime_with_min_max_operators_errors.test | 205 ++++++++++++++ 6 files changed, 594 insertions(+), 9 deletions(-) create mode 100644 test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators.test create mode 100644 test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_create.test create mode 100644 test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_errors.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 1ba267525a..8663908931 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,5 @@ -- Fix: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) +- Add: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) +- Fix: $max and $min operators were not supported with DateTime attributes (#4585) - Fix: wrong date values should not allowed in subscription's expires field (#4541) - Fix: do not raise DB alarm in case of wrong GeoJSON in client request - Upgrade cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 374da2024a..01ab451ca1 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -1618,7 +1618,8 @@ The only exception to "use only one operator" rule is the case of `$set` and Update operators can be used in entity creation or replace operations. In particular: * Numeric operators takes 0 as reference. For instance, `{"$inc": 4}` results in 4, - `{$mul: 1000}` results in 0, etc. + `{"$mul": 1000}` results in 0, etc. +* Epoch time (`"1970-01-01T00:00:00Z"`) is used as reference for `DateTime` when `$min` or `$max` are used. * `$set` takes the empty object (`{}`) as reference. For instance, `"$set": {"X": 1}` results in just `{"X": 1}` * `$push` and `$addToSet` take the empty array (`[]`) as reference. For instance, `{"$push": 4}` results in `[ 4 ]`. diff --git a/src/lib/jsonParseV2/parseContextAttribute.cpp b/src/lib/jsonParseV2/parseContextAttribute.cpp index caf6ac190a..4d4ab692a5 100644 --- a/src/lib/jsonParseV2/parseContextAttribute.cpp +++ b/src/lib/jsonParseV2/parseContextAttribute.cpp @@ -218,16 +218,42 @@ static std::string parseContextAttributeObject // Is it a (not null) date? if (checkAttrSpecialTypes && ((caP->type == DATE_TYPE) || (caP->type == DATE_TYPE_ALT)) && (caP->valueType != orion::ValueTypeNull)) { - caP->numberValue = parse8601Time(caP->stringValue); - - if (caP->numberValue == -1) + if (caP->valueType == orion::ValueTypeObject) { - return "date has invalid format"; + // check if $max or $min operators are being used. Other usages of object values result in parsing error + if ((caP->compoundValueP != NULL) && (caP->compoundValueP->valueType == orion::ValueTypeObject) + && (caP->compoundValueP->childV.size() == 1) && ((caP->compoundValueP->childV[0]->name == "$max") || (caP->compoundValueP->childV[0]->name == "$min"))) + { + orion::CompoundValueNode* upOp = caP->compoundValueP->childV[0]; + upOp->numberValue = parse8601Time(upOp->stringValue); + + if (upOp->numberValue == -1) + { + return "date has invalid format"; + } + + // Probably reseting stringValue is not needed, but let's do it for cleanliness + upOp->stringValue = ""; + upOp->valueType = orion::ValueTypeNumber; + } + else + { + return "date has invalid format"; + } } + else + { + caP->numberValue = parse8601Time(caP->stringValue); + + if (caP->numberValue == -1) + { + return "date has invalid format"; + } - // Probably reseting stringValue is not needed, but let's do it for cleanliness - caP->stringValue = ""; - caP->valueType = orion::ValueTypeNumber; + // Probably reseting stringValue is not needed, but let's do it for cleanliness + caP->stringValue = ""; + caP->valueType = orion::ValueTypeNumber; + } } // It is a safe GeoJSON? diff --git a/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators.test b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators.test new file mode 100644 index 0000000000..49ab277624 --- /dev/null +++ b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators.test @@ -0,0 +1,253 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +DateTime with min and max operators + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z +# 02. Update entity with {$min: 2025-04-15T10:59:59.000Z} +# 03. Get entity see T=2024-04-15T10:59:59.000Z +# 04. Update entity with {$min: 2023-04-15T10:59:59.000Z} +# 05. Get entity see T=2023-04-15T10:59:59.000Z +# 06. Update entity with {$max: 2022-04-15T10:59:59.000Z} +# 07. Get entity see T=2023-04-15T10:59:59.000Z +# 08. Update entity with {$max: 2026-04-15T10:59:59.000Z} +# 09. Get entity see T=2026-04-15T10:59:59.000Z +# + + +echo '01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z' +echo '=============================================================' +payload='{ + "id": "E1", + "type": "T", + "T": { + "value": "2024-04-15T10:59:59.000Z", + "type": "DateTime" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo '02. Update entity with {$min: 2025-04-15T10:59:59.000Z}' +echo "=======================================================" +payload='{ + "T": { + "value": {"$min": "2025-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "03. Get entity see T=2024-04-15T10:59:59.000Z" +echo "=============================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +echo '04. Update entity with {$min: 2023-04-15T10:59:59.000Z}' +echo "=======================================================" +payload='{ + "T": { + "value": {"$min": "2023-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "05. Get entity see T=2023-04-15T10:59:59.000Z" +echo "=============================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +echo '06. Update entity with {$max: 2022-04-15T10:59:59.000Z}' +echo "=======================================================" +payload='{ + "T": { + "value": {"$max": "2022-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "07. Get entity see T=2023-04-15T10:59:59.000Z" +echo "=============================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +echo '08. Update entity with {$max: 2026-04-15T10:59:59.000Z}' +echo "=======================================================" +payload='{ + "T": { + "value": {"$max": "2026-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "09. Get entity see T=2026-04-15T10:59:59.000Z" +echo "=============================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +--REGEXPECT-- +01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z +============================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +02. Update entity with {$min: 2025-04-15T10:59:59.000Z} +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +03. Get entity see T=2024-04-15T10:59:59.000Z +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 74 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2024-04-15T10:59:59.000Z" + } +} + + +04. Update entity with {$min: 2023-04-15T10:59:59.000Z} +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Get entity see T=2023-04-15T10:59:59.000Z +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 74 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2023-04-15T10:59:59.000Z" + } +} + + +06. Update entity with {$max: 2022-04-15T10:59:59.000Z} +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +07. Get entity see T=2023-04-15T10:59:59.000Z +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 74 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2023-04-15T10:59:59.000Z" + } +} + + +08. Update entity with {$max: 2026-04-15T10:59:59.000Z} +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +09. Get entity see T=2026-04-15T10:59:59.000Z +============================================= +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 74 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2026-04-15T10:59:59.000Z" + } +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_create.test b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_create.test new file mode 100644 index 0000000000..5206388283 --- /dev/null +++ b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_create.test @@ -0,0 +1,99 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +DateTime with min and max operators creating entities + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity with T1={$min: 2025-04-15T10:59:59.000Z}, T2={$max: 2024-04-15T10:59:59.000Z} +# 02. Get entity see T1=1970-01-01T00:00:00.000Z, T2=2024-04-15T10:59:59.000Z +# + + +echo '01. Create entity with T1={$min: 2025-04-15T10:59:59.000Z}, T2={$max: 2024-04-15T10:59:59.000Z}' +echo '===============================================================================================' +payload='{ + "id": "E1", + "type": "T", + "T1": { + "value": {"$min": "2025-04-15T10:59:59.000Z"}, + "type": "DateTime" + }, + "T2": { + "value": {"$max": "2024-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "02. Get entity see T1=1970-01-01T00:00:00.000Z, T2=2024-04-15T10:59:59.000Z" +echo "===========================================================================" +orionCurl --url /v2/entities/E1/attrs +echo +echo + + +--REGEXPECT-- +01. Create entity with T1={$min: 2025-04-15T10:59:59.000Z}, T2={$max: 2024-04-15T10:59:59.000Z} +=============================================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +02. Get entity see T1=1970-01-01T00:00:00.000Z, T2=2024-04-15T10:59:59.000Z +=========================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 149 + +{ + "T1": { + "metadata": {}, + "type": "DateTime", + "value": "1970-01-01T00:00:00.000Z" + }, + "T2": { + "metadata": {}, + "type": "DateTime", + "value": "2024-04-15T10:59:59.000Z" + } +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file diff --git a/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_errors.test b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_errors.test new file mode 100644 index 0000000000..05fdd8de7f --- /dev/null +++ b/test/functionalTest/cases/4585_datetime_with_min_max_operators/datetime_with_min_max_operators_errors.test @@ -0,0 +1,205 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +DateTime with min and max operators errors + +--SHELL-INIT-- +dbInit CB +brokerStart CB + +--SHELL-- + +# +# 01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z +# 02. Update entity with {$min: 2025x-04-15T10:59:59.000Z}, see error +# 03. Update entity with {$max: 2023x-04-15T10:59:59.000Z, see error +# 04. Update entity with {$mxn: 2025-04-15T10:59:59.000Z}, see error +# 05. Update entity with {$max: 2025-04-15T10:59:59.000Z, foo: 1}, see error +# 06. Get entity see T=2024-04-15T10:59:59.000Z (entity not modified) +# + + +echo '01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z' +echo '=============================================================' +payload='{ + "id": "E1", + "type": "T", + "T": { + "value": "2024-04-15T10:59:59.000Z", + "type": "DateTime" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo '02. Update entity with {$min: 2025x-04-15T10:59:59.000Z}, see error' +echo "===================================================================" +payload='{ + "T": { + "value": {"$min": "2025x-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo '03. Update entity with {$max: 2023x-04-15T10:59:59.000Z, see error' +echo "==================================================================" +payload='{ + "T": { + "value": {"$max": "2023x-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo '04. Update entity with {$mxn: 2025-04-15T10:59:59.000Z}, see error' +echo "==================================================================" +payload='{ + "T": { + "value": {"$mxn": "2025-04-15T10:59:59.000Z"}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo '05. Update entity with {$max: 2025-04-15T10:59:59.000Z, foo: 1}, see error' +echo "==========================================================================" +payload='{ + "T": { + "value": {"$max": "2025-04-15T10:59:59.000Z", "foo": 1}, + "type": "DateTime" + } +}' +orionCurl --url /v2/entities/E1/attrs --payload "$payload" -X PATCH +echo +echo + + +echo "06. Get entity see T=2024-04-15T10:59:59.000Z (entity not modified)" +echo "===================================================================" +orionCurl --url /v2/entities/E1 +echo +echo + + + +--REGEXPECT-- +01. Create entity E1 with DateTime T=2024-04-15T10:59:59.000Z +============================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +02. Update entity with {$min: 2025x-04-15T10:59:59.000Z}, see error +=================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 62 + +{ + "description": "date has invalid format", + "error": "BadRequest" +} + + +03. Update entity with {$max: 2023x-04-15T10:59:59.000Z, see error +================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 62 + +{ + "description": "date has invalid format", + "error": "BadRequest" +} + + +04. Update entity with {$mxn: 2025-04-15T10:59:59.000Z}, see error +================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 62 + +{ + "description": "date has invalid format", + "error": "BadRequest" +} + + +05. Update entity with {$max: 2025-04-15T10:59:59.000Z, foo: 1}, see error +========================================================================== +HTTP/1.1 400 Bad Request +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 62 + +{ + "description": "date has invalid format", + "error": "BadRequest" +} + + +06. Get entity see T=2024-04-15T10:59:59.000Z (entity not modified) +=================================================================== +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 95 + +{ + "T": { + "metadata": {}, + "type": "DateTime", + "value": "2024-04-15T10:59:59.000Z" + }, + "id": "E1", + "type": "T" +} + + +--TEARDOWN-- +brokerStop CB +dbDrop CB \ No newline at end of file From 900d9860fef27f072ac5db331bba44a232c92df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 1 Aug 2024 14:59:42 +0200 Subject: [PATCH 365/390] FIX comment raised in code review --- doc/manuals/orion-api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 01ab451ca1..0b12ed9df8 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -1332,7 +1332,7 @@ PUT /v2/entities/E/attrs/A would not change attribute value. -Apart from numbers, other value types are supported (eg, strings). +Apart from numbers, other value types are supported (eg, strings or `DateTime`). #### `$max` @@ -1360,7 +1360,7 @@ PUT /v2/entities/E/attrs/A would not change attribute value. -Apart from numbers, other value types are supported (eg, strings). +Apart from numbers, other value types are supported (eg, strings or `DateTime`). #### `$push` From 4b9d92a388f0044dfb4896b0c035940f7b5fb318 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Thu, 1 Aug 2024 22:15:53 +0900 Subject: [PATCH 366/390] (JP) FIX DateTime with min and max operators (#4598) --- doc/manuals.jp/orion-api.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index 4b81e53973..0dad0f89dc 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -1286,7 +1286,7 @@ PUT /v2/entities/E/attrs/A 属性値は変更されません。 -数値とは別に、他の値タイプ (文字列など) がサポートされています。 +数値とは別に、他の値タイプ (文字列や `DateTime` など) がサポートされています。 @@ -1316,7 +1316,7 @@ PUT /v2/entities/E/attrs/A 属性値は変更されません。 -数値とは別に、他の値タイプ (文字列など) がサポートされています。 +数値とは別に、他の値タイプ (文字列や `DateTime` など) がサポートされています。 @@ -1589,7 +1589,8 @@ PUT /v2/entities/E/attrs/A 更新演算子は、エンティティの作成または置換操作で使用できます。特に: -* 数値演算子は 0 を基準とします。たとえば、`{"$inc": 4}` の結果は 4、`{$mul: 1000}` の結果は 0 になります +* 数値演算子は 0 を基準とします。たとえば、`{"$inc": 4}` の結果は 4、`{"$mul": 1000}` の結果は 0 になります +* `$min` または `$max` が使用される場合、エポック時間 (`"1970-01-01T00:00:00Z"`) が `DateTime` の参照として使用されます * `$set` は空のオブジェクト (`{}`) を参照として受け取ります。たとえば、`"$set": {"X": 1}` は単に `{"X": 1}` になります * `$push` と `$addToSet` は空の配列 (`[]`) を参照として受け取ります。たとえば、`{"$push": 4}` は `[ 4 ]` になります * `$pull`、`$pullAll`、および `$unset` は無視されます。これは、演算子が使用される属性がエンティティ内に作成されないことを意味します。 From 316ddfb0c2fcc5cc45a0be2e7e87339574c0b659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 2 Aug 2024 11:38:51 +0200 Subject: [PATCH 367/390] ADD metadata in JEXL context --- src/lib/ngsi/Metadata.cpp | 68 +++++ src/lib/ngsi/Metadata.h | 2 + src/lib/ngsiNotify/Notifier.cpp | 16 + .../metadata_in_jexl_context_basic.test | 287 ++++++++++++++++++ .../metadata_in_jexl_context_builtin.test | 253 +++++++++++++++ .../metadata_in_jexl_context_collision.test | 222 ++++++++++++++ 6 files changed, 848 insertions(+) create mode 100644 test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_basic.test create mode 100644 test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_builtin.test create mode 100644 test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_collision.test diff --git a/src/lib/ngsi/Metadata.cpp b/src/lib/ngsi/Metadata.cpp index c26573a0bf..549502879a 100644 --- a/src/lib/ngsi/Metadata.cpp +++ b/src/lib/ngsi/Metadata.cpp @@ -666,3 +666,71 @@ void Metadata::appendToBsoN(orion::BSONObjBuilder* mdBuilder, orion::BSONArrayBu } } } + + +/* **************************************************************************** +* +* addToContext - +* +* FIXME P5: duplicated from ContextAttrMetibute::addToContext(). Maybe they should be unified +*/ +void Metadata::addToContext(ExprContextObject* exprContextObjectP, bool legacy) +{ + if (compoundValueP != NULL) + { + // In legacy expression, objects are vector are strings to be stored in a std::map + if (valueType == orion::ValueTypeObject) + { + if (legacy) + { + exprContextObjectP->add(name, compoundValueP->toJson(), true); + } + else + { + exprContextObjectP->add(name, compoundValueP->toExprContextObject()); + } + } + else // valueType == orion::ValueTypeVector + { + if (legacy) + { + exprContextObjectP->add(name, compoundValueP->toJson(), true); + } + else + { + exprContextObjectP->add(name, compoundValueP->toExprContextList()); + } + } + } + else if (valueType == orion::ValueTypeNumber) + { + if ((type == DATE_TYPE) || (type == DATE_TYPE_ALT)) + { + exprContextObjectP->add(name, toJsonString(isodate2str(numberValue))); + } + else // regular number + { + exprContextObjectP->add(name, numberValue); + } + } + else if (valueType == orion::ValueTypeString) + { + exprContextObjectP->add(name, toJsonString(stringValue)); + } + else if (valueType == orion::ValueTypeBoolean) + { + exprContextObjectP->add(name, boolValue); + } + else if (valueType == orion::ValueTypeNull) + { + exprContextObjectP->add(name); + } + else if (valueType == orion::ValueTypeNotGiven) + { + LM_E(("Runtime Error (value not given for metadata %s)", name.c_str())); + } + else + { + LM_E(("Runtime Error (invalid value type %s for metadata %s)", valueTypeName(valueType), name.c_str())); + } +} \ No newline at end of file diff --git a/src/lib/ngsi/Metadata.h b/src/lib/ngsi/Metadata.h index e1fcdab163..6faabb18a8 100644 --- a/src/lib/ngsi/Metadata.h +++ b/src/lib/ngsi/Metadata.h @@ -98,6 +98,8 @@ typedef struct Metadata void appendToBsoN(orion::BSONObjBuilder* md, orion::BSONArrayBuilder* mdNames, bool useDefaultType); + void addToContext(ExprContextObject* exprContextObjectP, bool legacy); + std::string check(ApiVersion apiVersion); } Metadata; diff --git a/src/lib/ngsiNotify/Notifier.cpp b/src/lib/ngsiNotify/Notifier.cpp index 0ca4da8582..490eb88e20 100644 --- a/src/lib/ngsiNotify/Notifier.cpp +++ b/src/lib/ngsiNotify/Notifier.cpp @@ -367,6 +367,7 @@ static SenderThreadParams* buildSenderParamsCustom // Used by several macroSubstitute() calls along this function ExprContextObject exprContext(basic); + ExprContextObject exprMetadataContext(basic); // It seems that add() semantics are different in basic and jexl mode. In jexl mode, if the key already exists, it is // updated (in other words, the last added keys is the one that takes precedence). In basic model, if the key already @@ -376,16 +377,31 @@ static SenderThreadParams* buildSenderParamsCustom TIME_EXPR_CTXBLD_START(); exprContext.add("id", en.id); exprContext.add("type", en.type); + if (!basic) { exprContext.add("service", tenant); exprContext.add("servicePath", en.servicePath); exprContext.add("authToken", xauthToken); } + for (unsigned int ix = 0; ix < en.attributeVector.size(); ix++) { en.attributeVector[ix]->addToContext(&exprContext, basic); + + // Add attribute metadata to context + ExprContextObject exprAttrMetadataContext(basic); + for (unsigned int jx = 0; jx < en.attributeVector[ix]->metadataVector.size(); jx++) + { + en.attributeVector[ix]->metadataVector[jx]->addToContext(&exprAttrMetadataContext, basic); + } + exprMetadataContext.add(en.attributeVector[ix]->name, exprAttrMetadataContext); } + + // Add all metadata under the "metadata" context key + // (note that in JEXL if the key already exists, it is updated, so attribute with name "metadata" will never be appear in context) + exprContext.add("metadata", exprMetadataContext); + if (basic) { exprContext.add("service", tenant); diff --git a/test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_basic.test b/test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_basic.test new file mode 100644 index 0000000000..de86c13540 --- /dev/null +++ b/test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_basic.test @@ -0,0 +1,287 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +Metadata in JEXL context (basic case) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression sum: metadata.A.MD1+metadata.B.MD1 +# 02. Create entity E1 with A.MD1=1 and B.MD1=2 +# 03. Update entity E1 with A.MD1=foo and B.MD1=bar +# 04. Update entity E1 with A.MD1=2.1 and B.MD1=-3.8 +# 05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7) +# + + +echo "01. Create custom sub with custom expression sum: metadata.A.MD1+metadata.B.MD1" +echo "===============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "sum": { + "value": "${metadata.A.MD1+metadata.B.MD2}", + "type": "Calculated" + } + } + }, + "attrs": [ "sum" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A.MD1=1 and B.MD1=2" +echo "=============================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 10, + "type": "Number", + "metadata": { + "MD1": { + "value": 1, + "type": "Number" + } + } + }, + "B": { + "value": 20, + "type": "Number", + "metadata": { + "MD2": { + "value": 2, + "type": "Number" + } + } + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A.MD1=foo and B.MD1=bar" +echo "=================================================" +payload='{ + "A": { + "value": 10, + "type": "Number", + "metadata": { + "MD1": { + "value": "foo", + "type": "Text" + } + } + }, + "B": { + "value": 20, + "type": "Number", + "metadata": { + "MD2": { + "value": "bar", + "type": "Text" + } + } + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Update entity E1 with A.MD1=2.1 and B.MD1=-3.8" +echo "==================================================" +payload='{ + "A": { + "value": 10, + "type": "Number", + "metadata": { + "MD1": { + "value": 2.1, + "type": "Number" + } + } + }, + "B": { + "value": 20, + "type": "Number", + "metadata": { + "MD2": { + "value": -3.8, + "type": "Number" + } + } + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7)" +echo "=================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: metadata.A.MD1+metadata.B.MD1 +=============================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A.MD1=1 and B.MD1=2 +============================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A.MD1=foo and B.MD1=bar +================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Update entity E1 with A.MD1=2.1 and B.MD1=-3.8 +================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7) +================================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 129 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 136 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": "foobar" + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 132 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": -1.7 + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_builtin.test b/test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_builtin.test new file mode 100644 index 0000000000..8b013b19e9 --- /dev/null +++ b/test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_builtin.test @@ -0,0 +1,253 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +Metadata in JEXL context (builtin metadata) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create custom sub with custom expression diff: A-metadata.A.previousValue +# 02. Create entity E1 with A=1 +# 03. Update entity E1 with A=4 +# 04. Update entity E1 with A=2 +# 05. Dump accumulator and see three notifications (diff: null, diff: 3, diff: -2) +# + + +echo "01. Create custom sub with custom expression diff: A-metadata.A.previousValue" +echo "=============================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "diff": { + "value": "${A-metadata.A.previousValue}", + "type": "Calculated" + } + } + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with A=1" +echo "=============================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with A=4" +echo "=============================" +payload='{ + "A": { + "value": 4, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "04. Update entity E1 with A=2" +echo "=============================" +payload='{ + "A": { + "value": 2, + "type": "Number" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three notifications (diff: 1, diff: 3, diff: -2)" +echo "=============================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression diff: A-metadata.A.previousValue +============================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with A=1 +============================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with A=4 +============================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Update entity E1 with A=2 +============================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see three notifications (diff: 1, diff: 3, diff: -2) +============================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 179 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 1 + }, + "diff": { + "metadata": {}, + "type": "Calculated", + "value": null + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 176 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 4 + }, + "diff": { + "metadata": {}, + "type": "Calculated", + "value": 3 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 177 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "A": { + "metadata": {}, + "type": "Number", + "value": 2 + }, + "diff": { + "metadata": {}, + "type": "Calculated", + "value": -2 + }, + "id": "E1", + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop diff --git a/test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_collision.test b/test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_collision.test new file mode 100644 index 0000000000..fe3f5946ff --- /dev/null +++ b/test/functionalTest/cases/4594_metadata_in_jexl_context/metadata_in_jexl_context_collision.test @@ -0,0 +1,222 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh +# JEXL_EXPR_FLAVOUR - to mark the test has to execute only when contextBroker includes jexl-expr flavour + +--NAME-- +Metadata in JEXL context (collision in metadata name) + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# This test is intentionally modeling an anti-pattern that should not be used, but we include it +# to check and cover the Context Broker behabiviour in this case +# +# 01. Create custom sub with custom expression sum: metadata.A.MD1 +# 02. Create entity E1 with colliding attribute named metadata +# 03. Update entity E1 with colliding attribute named metadata +# 04. Dump accumulator and see three notifications () +# + + +echo "01. Create custom sub with custom expression sum: metadata.A.MD1" +echo "================================================================" +payload='{ + "subject": { + "entities": [ + { + "id" : "E1", + "type": "T" + } + ] + }, + "notification": { + "httpCustom": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "ngsi": { + "sum": { + "value": "${metadata.A.MD1}", + "type": "Calculated" + } + } + }, + "attrs": [ "sum" ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity E1 with colliding attribute named metadata" +echo "============================================================" +payload='{ + "id": "E1", + "type": "T", + "A": { + "value": 10, + "type": "Number", + "metadata": { + "MD1": { + "value": "from-md", + "type": "Text" + } + } + }, + "metadata": { + "value": { + "A": { + "MD1": "from-attr" + } + }, + "type": "Object" + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity E1 with colliding attribute named metadata" +echo "============================================================" +payload='{ + "A": { + "value": 10, + "type": "Number", + "metadata": { + "MD1": { + "value": "from-md2", + "type": "Text" + } + } + }, + "metadata": { + "value": { + "A": { + "MD1": "from-attr2" + } + }, + "type": "Object" + } +}' +orionCurl --url /v2/entities/E1/attrs -X PATCH --payload "$payload" +echo +echo + + +echo "05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7)" +echo "=================================================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create custom sub with custom expression sum: metadata.A.MD1 +================================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity E1 with colliding attribute named metadata +============================================================ +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/E1?type=T +Content-Length: 0 + + + +03. Update entity E1 with colliding attribute named metadata +============================================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +05. Dump accumulator and see three notifications (sum: 3, sum: foobar, sum: -1.7) +================================================================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 137 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": "from-md" + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 138 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "id": "E1", + "sum": { + "metadata": {}, + "type": "Calculated", + "value": "from-md2" + }, + "type": "T" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 2256824000269ada4b55771a9295b3b7e7cce59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 2 Aug 2024 11:58:29 +0200 Subject: [PATCH 368/390] ADD docu and CNR for #4594 --- CHANGES_NEXT_RELEASE | 1 + doc/manuals/orion-api.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 8663908931..eaffe48fcd 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,5 @@ - Add: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) +- Add: attribute metadata to JEXL context (#4594) - Fix: $max and $min operators were not supported with DateTime attributes (#4585) - Fix: wrong date values should not allowed in subscription's expires field (#4541) - Fix: do not raise DB alarm in case of wrong GeoJSON in client request diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md index 0b12ed9df8..42af692801 100644 --- a/doc/manuals/orion-api.md +++ b/doc/manuals/orion-api.md @@ -75,6 +75,7 @@ - [Additional considerations](#additional-considerations) - [JEXL Support](#jexl-support) - [JEXL usage example](#jexl-usage-example) + - [Metadata support](#metadata-support) - [Evaluation priority](#evaluation-priority) - [Available Transformations](#available-transformations) - [`uppercase`](#uppercase) @@ -2514,6 +2515,19 @@ will trigger a notification like this: ] ``` +### Metadata support + +Attribute metadata is also included in the JEXL evaluation context, along with the actual attributes. This is done using a special context key named `metadata` which value is an object with the following structure: + +* keys are the name of the attributes +* values are object with a key-map of the metadata values of the given attribute + +For instance, the expression `${metadata.A.MD1}` will result in the value of the metadata `MD1` belonging to the attribute `A`. + +Note that [builtin metadata](#builtin-metadata) are automatically added to the `metadata` context key. For instance, the `${A-metadata.A.previousValue}` expression will provide the difference between the current value of `A` (in the update request) and the previous one (before the update request). + +Finally, note that in the rare case an attribute named `metadata` is used (which is an anti-pattern strongly discouraged) it will not be added to the context, as the `metadata` with metadata values will take precedence. + ### Evaluation priority Each time an expression is evaluated, it is added to the expression context, so it can be reused by other expressions. However, by default Orion does not guarantee a given evaluation other. From 19b806f272834a7b6198b53fdf6917e3ec70c410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 2 Aug 2024 14:54:29 +0200 Subject: [PATCH 369/390] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index eaffe48fcd..5b7b94c871 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,7 +1,7 @@ - Add: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556) -- Add: attribute metadata to JEXL context (#4594) +- Add: attribute metadata to JEXL context (#4594, #4560) - Fix: $max and $min operators were not supported with DateTime attributes (#4585) - Fix: wrong date values should not allowed in subscription's expires field (#4541) - Fix: do not raise DB alarm in case of wrong GeoJSON in client request - Upgrade cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString) -- Upgrade Debian version from 12.4 to 12.6 in Dockerfile \ No newline at end of file +- Upgrade Debian version from 12.4 to 12.6 in Dockerfile From a852b1dc2c315bdddeb51741108e71fa5885d95f Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Fri, 2 Aug 2024 22:10:58 +0900 Subject: [PATCH 370/390] (JP) ADD doc about metadata in JEXL context (#4600) --- doc/manuals.jp/orion-api.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md index 0dad0f89dc..b282a4a3f7 100644 --- a/doc/manuals.jp/orion-api.md +++ b/doc/manuals.jp/orion-api.md @@ -75,6 +75,7 @@ - [その他の考慮事項](#additional-considerations) - [JEXL サポート (JEXL Support)](#jexl-support) - [JEXL 使用例 (JEXL usage example)](#jexl-usage-example) + - [メタデータのサポート (Metadata support)](#metadata-support) - [評価の優先順位](#evaluation-priority) - [利用可能な変換 (Available Transformations)](#available-transformations) - [`uppercase`](#uppercase) @@ -2518,6 +2519,21 @@ Orion は、この機能を提供するために cjexl ライブラリに依存 ] ``` + + +### メタデータのサポート (Metadata support) + +属性メタデータも、実際の属性とともに JEXL 評価コンテキストに含まれています。これは、`metadata` という特別なコンテキスト・キーを使用して行われます。このキーの値は、次の構造を持つオブジェクトです: + +* キーは属性の名前 +* 値は、指定された属性のメタデータ値のキー マップを持つオブジェクト + +たとえば、式 `${metadata.A.MD1}` は、属性 `A` に属するメタデータ `MD1` の値になります。 + +[組み込みメタデータ](#builtin-metadata) は、`metadata` コンテキスト キーに自動的に追加されることに注意してください。たとえば、`${A-metadata.A.previousValue}` 式は、`A` の現在の値 (更新要求内) と以前の値 (更新要求前) の差を提供します。 + +最後に、まれに `metadata` という名前の属性が使用される場合 (これはアンチパターンであり、強く推奨されません)、メタデータ値を持つ `metadata` が優先されるため、コンテキストに追加されないことに注意してください。 + ### 評価の優先順位 From 9c76a38b83e8e1b8ad1ce4933e46646a4c35e0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 9 Aug 2024 08:47:08 +0200 Subject: [PATCH 371/390] FIX remove legacyForwarding true in test that does not need it --- .../cases/0501_cors/registration_cors.test | 3 +- .../1492_limit_zero/limit_zero_regs.test | 6 +-- .../3004_v2_reg_create/v2_reg_create.test | 12 ++---- .../v2_reg_create_errors.test | 12 ++---- .../get_registrations.test | 36 ++++++++---------- .../get_registrations_with_pagination.test | 38 +++++++++---------- .../delete_reg_using_v2_created_using_v2.test | 8 +--- .../get_reg_using_v2_created_using_v2.test | 12 ++---- .../entity_without_attributes_not_listed.test | 3 +- ...t_render_in_not_matching_registration.test | 3 +- .../fwd_query_on_fail.test | 3 +- 11 files changed, 53 insertions(+), 83 deletions(-) diff --git a/test/functionalTest/cases/0501_cors/registration_cors.test b/test/functionalTest/cases/0501_cors/registration_cors.test index f36c008487..9bb76a901b 100644 --- a/test/functionalTest/cases/0501_cors/registration_cors.test +++ b/test/functionalTest/cases/0501_cors/registration_cors.test @@ -44,8 +44,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' diff --git a/test/functionalTest/cases/1492_limit_zero/limit_zero_regs.test b/test/functionalTest/cases/1492_limit_zero/limit_zero_regs.test index 1a0da2705c..6189945cbe 100644 --- a/test/functionalTest/cases/1492_limit_zero/limit_zero_regs.test +++ b/test/functionalTest/cases/1492_limit_zero/limit_zero_regs.test @@ -62,8 +62,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -88,8 +87,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' diff --git a/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create.test b/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create.test index 0770b2d40d..dc7c95eebb 100644 --- a/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create.test +++ b/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create.test @@ -58,8 +58,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "inactive" }' @@ -100,7 +99,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 311 +Content-Length: 312 { "dataProvided": { @@ -125,7 +124,7 @@ Content-Length: 311 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "inactive" @@ -138,13 +137,10 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 250 +Content-Length: 206 { "counters": { - "deprecatedFeatures": { - "ngsiv1Forwarding": 2 - }, "jsonRequests": 1, "noPayloadRequests": 2, "requests": { diff --git a/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create_errors.test b/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create_errors.test index 3522afcd28..37b9e29aba 100644 --- a/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create_errors.test +++ b/test/functionalTest/cases/3004_v2_reg_create/v2_reg_create_errors.test @@ -776,8 +776,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": 4 }' @@ -802,8 +801,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "green" }' @@ -854,8 +852,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } } }' orionCurl --url /v2/registrations --payload "$payload" @@ -879,8 +876,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } } }' orionCurl --url /v2/registrations --payload "$payload" diff --git a/test/functionalTest/cases/3005_GET_registrations/get_registrations.test b/test/functionalTest/cases/3005_GET_registrations/get_registrations.test index 1d6df2e9cb..42248ac3b6 100644 --- a/test/functionalTest/cases/3005_GET_registrations/get_registrations.test +++ b/test/functionalTest/cases/3005_GET_registrations/get_registrations.test @@ -60,8 +60,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -86,8 +85,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -112,8 +110,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -138,8 +135,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -223,7 +219,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1053 +Content-Length: 1057 [ { @@ -244,7 +240,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -267,7 +263,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -290,7 +286,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -313,7 +309,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -338,7 +334,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 264 +Content-Length: 265 [ { @@ -359,7 +355,7 @@ Content-Length: 264 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -373,7 +369,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 1053 +Content-Length: 1057 [ { @@ -394,7 +390,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -417,7 +413,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -440,7 +436,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -463,7 +459,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" diff --git a/test/functionalTest/cases/3005_GET_registrations/get_registrations_with_pagination.test b/test/functionalTest/cases/3005_GET_registrations/get_registrations_with_pagination.test index c96163d4c7..552d561044 100644 --- a/test/functionalTest/cases/3005_GET_registrations/get_registrations_with_pagination.test +++ b/test/functionalTest/cases/3005_GET_registrations/get_registrations_with_pagination.test @@ -57,8 +57,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -83,8 +82,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -109,8 +107,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -135,8 +132,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -228,7 +224,7 @@ Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Fiware-Total-Count: 4 Content-Type: application/json -Content-Length: 1053 +Content-Length: 1057 [ { @@ -249,7 +245,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -272,7 +268,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -295,7 +291,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -318,7 +314,7 @@ Content-Length: 1053 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -332,7 +328,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 527 +Content-Length: 529 [ { @@ -353,7 +349,7 @@ Content-Length: 527 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -376,7 +372,7 @@ Content-Length: 527 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -390,7 +386,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 527 +Content-Length: 529 [ { @@ -411,7 +407,7 @@ Content-Length: 527 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -434,7 +430,7 @@ Content-Length: 527 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -448,7 +444,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 264 +Content-Length: 265 [ { @@ -469,7 +465,7 @@ Content-Length: 264 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" diff --git a/test/functionalTest/cases/3006_delete_registrations/delete_reg_using_v2_created_using_v2.test b/test/functionalTest/cases/3006_delete_registrations/delete_reg_using_v2_created_using_v2.test index ff06bae97e..629acf2b23 100644 --- a/test/functionalTest/cases/3006_delete_registrations/delete_reg_using_v2_created_using_v2.test +++ b/test/functionalTest/cases/3006_delete_registrations/delete_reg_using_v2_created_using_v2.test @@ -52,8 +52,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -123,13 +122,10 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 253 +Content-Length: 209 { "counters": { - "deprecatedFeatures": { - "ngsiv1Forwarding": 1 - }, "jsonRequests": 1, "noPayloadRequests": 3, "requests": { diff --git a/test/functionalTest/cases/3008_get_registration/get_reg_using_v2_created_using_v2.test b/test/functionalTest/cases/3008_get_registration/get_reg_using_v2_created_using_v2.test index 6a15472f9a..f0e4d35b43 100644 --- a/test/functionalTest/cases/3008_get_registration/get_reg_using_v2_created_using_v2.test +++ b/test/functionalTest/cases/3008_get_registration/get_reg_using_v2_created_using_v2.test @@ -52,8 +52,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${CP1_PORT}'/v2" - }, - "legacyForwarding": true + } }, "status": "active" }' @@ -101,7 +100,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 262 +Content-Length: 263 { "dataProvided": { @@ -121,7 +120,7 @@ Content-Length: 262 "http": { "url": "http://localhost:REGEX(\d+)/v2" }, - "legacyForwarding": true, + "legacyForwarding": false, "supportedForwardingMode": "all" }, "status": "active" @@ -148,13 +147,10 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 250 +Content-Length: 206 { "counters": { - "deprecatedFeatures": { - "ngsiv1Forwarding": 2 - }, "jsonRequests": 1, "noPayloadRequests": 3, "requests": { diff --git a/test/functionalTest/cases/3176_entity_without_attributes_not_listed/entity_without_attributes_not_listed.test b/test/functionalTest/cases/3176_entity_without_attributes_not_listed/entity_without_attributes_not_listed.test index 456cfcff1a..c3ea085346 100644 --- a/test/functionalTest/cases/3176_entity_without_attributes_not_listed/entity_without_attributes_not_listed.test +++ b/test/functionalTest/cases/3176_entity_without_attributes_not_listed/entity_without_attributes_not_listed.test @@ -55,8 +55,7 @@ payload='{ "http": { "url": "http://contextsource.example.org" }, - "supportedForwardingMode": "all", - "legacyForwarding": true + "supportedForwardingMode": "all" }, "status": "active" }' diff --git a/test/functionalTest/cases/3282_compound_not_render_in_not_matching_registration/compound_not_render_in_not_matching_registration.test b/test/functionalTest/cases/3282_compound_not_render_in_not_matching_registration/compound_not_render_in_not_matching_registration.test index 7a1cdef953..496683ce9c 100644 --- a/test/functionalTest/cases/3282_compound_not_render_in_not_matching_registration/compound_not_render_in_not_matching_registration.test +++ b/test/functionalTest/cases/3282_compound_not_render_in_not_matching_registration/compound_not_render_in_not_matching_registration.test @@ -112,8 +112,7 @@ payload='{ "provider": { "http": { "url": "http://localhost':$CP1_PORT'/v1" - }, - "legacyForwarding": true + } } }' orionCurl --url /v2/registrations --payload "$payload" diff --git a/test/functionalTest/cases/3363_fwd_query_on_fail/fwd_query_on_fail.test b/test/functionalTest/cases/3363_fwd_query_on_fail/fwd_query_on_fail.test index 2eb2ea9e40..e5b12fe77f 100644 --- a/test/functionalTest/cases/3363_fwd_query_on_fail/fwd_query_on_fail.test +++ b/test/functionalTest/cases/3363_fwd_query_on_fail/fwd_query_on_fail.test @@ -86,8 +86,7 @@ payload='{ "provider": { "http": { "url": "http://localhost:'${LISTENER_PORT}'/badresponse" - }, - "legacyForwarding": true + } } }' orionCurl --url /v2/registrations --payload "$payload" From 63dd8acab2ab030bfd9ac5ea3ace102aa615cb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 22 Aug 2024 12:01:21 +0200 Subject: [PATCH 372/390] ADD missing dir in style_check_in_makefile.sh --- scripts/style_check_in_makefile.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/style_check_in_makefile.sh b/scripts/style_check_in_makefile.sh index 8398fb7d30..40e6072e40 100755 --- a/scripts/style_check_in_makefile.sh +++ b/scripts/style_check_in_makefile.sh @@ -57,6 +57,7 @@ style_check src/lib/parseArgs style_check src/lib/cache style_check src/lib/alarmMgr style_check src/lib/metricsMgr +style_check src/lib/expressions style_check test/unittests style_check test/unittests/orionTypes style_check test/unittests/jsonParse @@ -152,7 +153,7 @@ style_check test/unittests/serviceRoutines # o ngsi/AttributeDomainName.cpp (haderding/remove_ngsiv1_indent: 9 lines) # o ngsi/AttributeDomainName.h (haderding/remove_ngsiv1_indent: 5 lines) # o ngsi/AttributeExpression.cpp (haderding/remove_ngsiv1_indent: 9 lines) -# o ngsi/AttributeExpression.h (haderding/remove_ngsiv1_indent: 6 lines) +# o ngsi/Attribute.h (haderding/remove_ngsiv1_indent: 6 lines) # o ngsi/StringList.cpp (haderding/remove_ngsiv1_indent: 13 lines) # o ngsi/StringList.h (haderding/remove_ngsiv1_indent: 5 lines) # o ngsi/ConditionValueList.cpp (haderding/remove_ngsiv1_indent: 13 lines) From 2624cdf280f410e8c72e478ba35203ef99191722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 22 Aug 2024 12:04:38 +0200 Subject: [PATCH 373/390] Update scripts/style_check_in_makefile.sh --- scripts/style_check_in_makefile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/style_check_in_makefile.sh b/scripts/style_check_in_makefile.sh index 40e6072e40..109b0c67f0 100755 --- a/scripts/style_check_in_makefile.sh +++ b/scripts/style_check_in_makefile.sh @@ -153,7 +153,7 @@ style_check test/unittests/serviceRoutines # o ngsi/AttributeDomainName.cpp (haderding/remove_ngsiv1_indent: 9 lines) # o ngsi/AttributeDomainName.h (haderding/remove_ngsiv1_indent: 5 lines) # o ngsi/AttributeExpression.cpp (haderding/remove_ngsiv1_indent: 9 lines) -# o ngsi/Attribute.h (haderding/remove_ngsiv1_indent: 6 lines) +# o ngsi/AttributeExpression.h (haderding/remove_ngsiv1_indent: 6 lines) # o ngsi/StringList.cpp (haderding/remove_ngsiv1_indent: 13 lines) # o ngsi/StringList.h (haderding/remove_ngsiv1_indent: 5 lines) # o ngsi/ConditionValueList.cpp (haderding/remove_ngsiv1_indent: 13 lines) From 3718279e3b86ecb1dcd8fedcdc0388e16522c760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 23 Aug 2024 08:17:05 +0200 Subject: [PATCH 374/390] FIX coverage_functional_test target to work with all tests --- CMakeLists.txt | 4 ++-- makefile | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f7632d895d..47997bc75d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -305,12 +305,12 @@ link_directories("/usr/lib/x86_64-linux-gnu") # # Enabling test harness # -if (UNIT_TEST) +if (UNIT_TEST OR COVERAGE) ENABLE_TESTING() INCLUDE(Dart) SET (CMAKE_TEST_TIMEOUT 60) SET (DART_TIMEOUT 60) - #ADD_SUBDIRECTORY(test/harness) + ADD_SUBDIRECTORY(test/functionalTest/cases) endif (UNIT_TEST) # diff --git a/makefile b/makefile index 8e940023ae..172fcc3a80 100644 --- a/makefile +++ b/makefile @@ -74,7 +74,7 @@ prepare_debug: compile_info prepare_coverage: compile_info mkdir -p BUILD_COVERAGE || true - cd BUILD_COVERAGE && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DUNIT_TEST=True -DCOVERAGE=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) + cd BUILD_COVERAGE && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DCOVERAGE=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) prepare_unit_test: compile_info @echo '------------------------------- prepare_unit_test starts ---------------------------------' @@ -234,7 +234,6 @@ ftd: functional_test_debug test: unit_test functional_test coverage: install_coverage - # FIXME #4418: the functional test part of this target is not working properly. Check issue for details. # Init coverage echo "Initializing coverage files" mkdir -p coverage @@ -292,7 +291,6 @@ coverage_unit_test: build_unit_test genhtml -o coverage coverage/broker.info coverage_functional_test: install_coverage - # FIXME #4418: this target is not working properly. Check issue for details. # Init coverage echo "Initializing coverage files" mkdir -p coverage From 4c3e2f7ea0e3a42a5c1c78bc2d31846d04d9579b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 23 Aug 2024 08:19:34 +0200 Subject: [PATCH 375/390] FIX rename .test file to avoid duplicated namefiles --- ...e_value.test => create_entity_with_empty_attribute_value.test} | 0 .../{restrict_chars_attr.test => restrict_chars_attr_in_ids.test} | 0 ...ct_chars_metadata.test => restrict_chars_metadata_in_ids.test} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename test/functionalTest/cases/0970_create_entity/{empty_attribute_value.test => create_entity_with_empty_attribute_value.test} (100%) rename test/functionalTest/cases/1601_restrict_chars_in_ids/{restrict_chars_attr.test => restrict_chars_attr_in_ids.test} (100%) rename test/functionalTest/cases/1601_restrict_chars_in_ids/{restrict_chars_metadata.test => restrict_chars_metadata_in_ids.test} (100%) diff --git a/test/functionalTest/cases/0970_create_entity/empty_attribute_value.test b/test/functionalTest/cases/0970_create_entity/create_entity_with_empty_attribute_value.test similarity index 100% rename from test/functionalTest/cases/0970_create_entity/empty_attribute_value.test rename to test/functionalTest/cases/0970_create_entity/create_entity_with_empty_attribute_value.test diff --git a/test/functionalTest/cases/1601_restrict_chars_in_ids/restrict_chars_attr.test b/test/functionalTest/cases/1601_restrict_chars_in_ids/restrict_chars_attr_in_ids.test similarity index 100% rename from test/functionalTest/cases/1601_restrict_chars_in_ids/restrict_chars_attr.test rename to test/functionalTest/cases/1601_restrict_chars_in_ids/restrict_chars_attr_in_ids.test diff --git a/test/functionalTest/cases/1601_restrict_chars_in_ids/restrict_chars_metadata.test b/test/functionalTest/cases/1601_restrict_chars_in_ids/restrict_chars_metadata_in_ids.test similarity index 100% rename from test/functionalTest/cases/1601_restrict_chars_in_ids/restrict_chars_metadata.test rename to test/functionalTest/cases/1601_restrict_chars_in_ids/restrict_chars_metadata_in_ids.test From 81a20a650fd1557a3291fa3702b83c800bc7adbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 23 Aug 2024 09:15:38 +0200 Subject: [PATCH 376/390] FIX makefile and CMakeLists.txt --- CMakeLists.txt | 4 ++-- makefile | 15 ++++++++------- .../AppendContextElementRequest_test.cpp | 1 - .../ngsi10/UpdateContextResponse_test.cpp | 1 - .../DiscoverContextAvailabilityResponse_test.cpp | 1 - 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 47997bc75d..9c922231cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -305,13 +305,13 @@ link_directories("/usr/lib/x86_64-linux-gnu") # # Enabling test harness # -if (UNIT_TEST OR COVERAGE) +if (COVERAGE AND NOT UNIT_TEST) ENABLE_TESTING() INCLUDE(Dart) SET (CMAKE_TEST_TIMEOUT 60) SET (DART_TIMEOUT 60) ADD_SUBDIRECTORY(test/functionalTest/cases) -endif (UNIT_TEST) +endif (COVERAGE AND NOT UNIT_TEST) # # Continuing if OK diff --git a/makefile b/makefile index 172fcc3a80..22e2bd9c25 100644 --- a/makefile +++ b/makefile @@ -77,10 +77,12 @@ prepare_coverage: compile_info cd BUILD_COVERAGE && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DCOVERAGE=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) prepare_unit_test: compile_info - @echo '------------------------------- prepare_unit_test starts ---------------------------------' + mkdir -p BUILD_UNITTEST || true + cd BUILD_UNITTEST && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DUNIT_TEST=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) + +prepare_unit_test_coverage: compile_info mkdir -p BUILD_UNITTEST || true cd BUILD_UNITTEST && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DUNIT_TEST=True -DCOVERAGE=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) - @echo '------------------------------- prepare_unit_test ended ---------------------------------' release: prepare_release cd BUILD_RELEASE && make -j$(CPU_COUNT) @@ -209,18 +211,17 @@ uncrustify_changed: git diff --name-only | grep "\.cpp\|\.h" | xargs uncrustify --no-backup -c scripts/uncrustify.cfg build_unit_test: prepare_unit_test - @echo '------------------------------- build_unit_test starts ---------------------------------' cd BUILD_UNITTEST && make -j$(CPU_COUNT) - @echo '------------------------------- build_unit_test ended ---------------------------------' + +build_unit_test_coverage: prepare_unit_test_coverage + cd BUILD_UNITTEST && make -j$(CPU_COUNT) unit_test: build_unit_test - @echo '------------------------------- unit_test starts ---------------------------------' if [ -z "${TEST_FILTER}" ]; then \ BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml; \ else \ BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml --gtest_filter=${TEST_FILTER}; \ fi - @echo '------------------------------- unit_test ended ---------------------------------' functional_test: install ./test/functionalTest/testHarness.sh @@ -266,7 +267,7 @@ coverage: install_coverage lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info genhtml -o coverage coverage/broker.info -coverage_unit_test: build_unit_test +coverage_unit_test: build_unit_test_coverage # Init coverage echo "Initializing coverage files" mkdir -p coverage diff --git a/test/unittests/convenience/AppendContextElementRequest_test.cpp b/test/unittests/convenience/AppendContextElementRequest_test.cpp index 88140f1c9f..fc7cc9fe5f 100644 --- a/test/unittests/convenience/AppendContextElementRequest_test.cpp +++ b/test/unittests/convenience/AppendContextElementRequest_test.cpp @@ -110,7 +110,6 @@ TEST(AppendContextElementRequest, release) AppendContextElementRequest acer; std::string out; ContextAttribute* caP = new ContextAttribute("caName", "caType", "121"); - Metadata* mdP = new Metadata("mdName", "mdType", "122"); acer.contextAttributeVector.push_back(caP); diff --git a/test/unittests/ngsi10/UpdateContextResponse_test.cpp b/test/unittests/ngsi10/UpdateContextResponse_test.cpp index d0613d01c4..1544fbdc9e 100644 --- a/test/unittests/ngsi10/UpdateContextResponse_test.cpp +++ b/test/unittests/ngsi10/UpdateContextResponse_test.cpp @@ -44,7 +44,6 @@ TEST(UpdateContextResponse, jsonRender) UpdateContextResponse* ucrP; ContextElementResponse* cerP; - Metadata* mdP; ContextAttribute* caP; std::string out; diff --git a/test/unittests/ngsi9/DiscoverContextAvailabilityResponse_test.cpp b/test/unittests/ngsi9/DiscoverContextAvailabilityResponse_test.cpp index 35b987f713..8fb0b75c38 100644 --- a/test/unittests/ngsi9/DiscoverContextAvailabilityResponse_test.cpp +++ b/test/unittests/ngsi9/DiscoverContextAvailabilityResponse_test.cpp @@ -95,7 +95,6 @@ TEST(DiscoverContextAvailabilityResponse, jsonRender) ContextRegistrationResponse* crrP; EntityId* eidP; ContextRegistrationAttribute* attrP; - Metadata* mdP; utInit(); From 0fb4368d1c1dd26b7e684d1fb55dfdd4b19acef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 23 Aug 2024 13:46:35 +0200 Subject: [PATCH 377/390] REMOVE coverage (unit_test+fuctional_test) target in makefile --- doc/manuals/admin/build_source.md | 10 +++++++-- makefile | 35 +------------------------------ 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index e86a146047..526698a18e 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -145,6 +145,12 @@ You can generate coverage reports for the Orion Context Broker using the followi * Do first a successful pass for unit_test and functional_test, to check that everything is ok (see above) -* Run coverage +* Run coverage (based in functionalt tests execution, the recommented one): - make coverage INSTALL_DIR=~ + make coverage_functional_test INSTALL_DIR=~ + +* Run coverage (based in unit tests execution, not so good): + + make coverage_unit_test INSTALL_DIR=~ + +*NOTE*: In the past we have a `make coverage` target. But it has maintainability problems (as the one described [in this issue](https://github.com/telefonicaid/fiware-orion/issues/4418)) so we remove it. Note that the unit test are not so important in the current days (it was in the early Context Broker days), in favour of functional test, so it's contributo to coverage is not very signficative. \ No newline at end of file diff --git a/makefile b/makefile index 22e2bd9c25..4faa074143 100644 --- a/makefile +++ b/makefile @@ -234,39 +234,6 @@ ftd: functional_test_debug test: unit_test functional_test -coverage: install_coverage - # Init coverage - echo "Initializing coverage files" - mkdir -p coverage - lcov -i --zerocounters --directory BUILD_COVERAGE/ - lcov --capture --initial --directory BUILD_COVERAGE -b BUILD_COVERAGE --output-file coverage/broker.init.info - # Execute test for coverage - echo "Executing coverage test" - BUILD_COVERAGE/test/unittests/unitTest -t 0-255 --gtest_output=xml:BUILD_COVERAGE/unit_test.xml - if [ -z "${CONTEXTBROKER_TESTENV_SOURCED}" ]; then \ - echo "Execute '. scripts/testEnv.sh' before executing the tests"; \ - exit 1; \ - fi - make test -C BUILD_COVERAGE ARGS="-D ExperimentalTest" TEST_VERBOSE=1 || true - @if [ -e test/functionalTest/cases/*.diff ]; then \ - echo "A .diff file was found in test/functionalTest/cases, which means that ctest failed running the test. This can happen if a \"Ok\""; \ - echo "token is used in the tests specification. Run \"test/functionalTest/testHarness.sh test/functionalTest/cases\" manually to find the problem."; \ - exit 1; \ - fi - @xsltproc scripts/cmake2junit.xsl BUILD_COVERAGE/Testing/`cat BUILD_COVERAGE/Testing/TAG| head -n1`/Test.xml > BUILD_COVERAGE/functional_test.xml - # Generate test report - echo "Generating coverage report" - lcov --directory BUILD_COVERAGE --capture -b BUILD_COVERAGE --output-file coverage/broker.test.info - lcov --add-tracefile coverage/broker.init.info --add-tracefile coverage/broker.test.info --output-file coverage/broker.info - lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info - lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info - lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info - # Remove unit test libraries and libraries developed before contextBroker project init - lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info - lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info - lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info - genhtml -o coverage coverage/broker.info - coverage_unit_test: build_unit_test_coverage # Init coverage echo "Initializing coverage files" @@ -275,7 +242,7 @@ coverage_unit_test: build_unit_test_coverage lcov --capture --initial --directory BUILD_UNITTEST -b BUILD_UNITTEST --output-file coverage/broker.init.info # Execute test for coverage echo "Executing coverage test" - BUILD_UNITTEST/test/unittests/unitTest -t 0-255 --gtest_output=xml:BUILD_UNITTEST/unit_test.xml + BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml # Generate test report echo "Generating coverage report" lcov --directory BUILD_UNITTEST --capture -b BUILD_UNITTEST --output-file coverage/broker.test.info From 8f46ec108694f23613e72b47a1a80e72087f5c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 23 Aug 2024 14:31:03 +0200 Subject: [PATCH 378/390] ADD recovered coverage makefile target --- doc/manuals/admin/build_source.md | 10 ++-------- makefile | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index 526698a18e..e86a146047 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -145,12 +145,6 @@ You can generate coverage reports for the Orion Context Broker using the followi * Do first a successful pass for unit_test and functional_test, to check that everything is ok (see above) -* Run coverage (based in functionalt tests execution, the recommented one): +* Run coverage - make coverage_functional_test INSTALL_DIR=~ - -* Run coverage (based in unit tests execution, not so good): - - make coverage_unit_test INSTALL_DIR=~ - -*NOTE*: In the past we have a `make coverage` target. But it has maintainability problems (as the one described [in this issue](https://github.com/telefonicaid/fiware-orion/issues/4418)) so we remove it. Note that the unit test are not so important in the current days (it was in the early Context Broker days), in favour of functional test, so it's contributo to coverage is not very signficative. \ No newline at end of file + make coverage INSTALL_DIR=~ diff --git a/makefile b/makefile index 4faa074143..2fa3581a2f 100644 --- a/makefile +++ b/makefile @@ -234,6 +234,25 @@ ftd: functional_test_debug test: unit_test functional_test +coverage: coverage_functional_test coverage_unit_test + rm -rf coverage + lcov --capture --directory BUILD_UNITTEST --output-file BUILD_UNITTEST/coverage.info + lcov --capture --directory BUILD_COVERAGE --output-file BUILD_COVERAGE/coverage.info + mkdir coverage + # Generate test report + echo "Generating coverage report" + lcov --add-tracefile BUILD_UNITTEST/coverage.info --add-tracefile BUILD_COVERAGE/coverage.info --output-file coverage/broker.info + lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info + lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info + lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info + # Remove unit test libraries and libraries developed before contextBroker project init + lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info + lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info + lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info + # app/ contains application itself, not libraries which make sense to measure unit_test coverage + lcov -r coverage/broker.info "*/src/app/*" -o coverage/broker.info + genhtml -o coverage coverage/broker.info + coverage_unit_test: build_unit_test_coverage # Init coverage echo "Initializing coverage files" From abe12d27f30d3fddd4d70bad3b3e76776a5a6be0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 23 Aug 2024 15:00:18 +0200 Subject: [PATCH 379/390] ADD CLEAN_COVERAGE_MACRO in makefile --- doc/manuals/admin/build_source.md | 2 ++ makefile | 41 ++++++++++++------------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index e86a146047..e4ac040750 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -148,3 +148,5 @@ You can generate coverage reports for the Orion Context Broker using the followi * Run coverage make coverage INSTALL_DIR=~ + +*NOTE*: note that functional test relying in debug traces are expected to fail under coverage execution (this is due to the LM_T macros used by the debug traces are disabled in the coverage code building, due to they add "noise" in condition coverage, thus making coverage reports more useful). \ No newline at end of file diff --git a/makefile b/makefile index 2fa3581a2f..1ac77aca62 100644 --- a/makefile +++ b/makefile @@ -54,6 +54,19 @@ ifndef MONGO_HOST MONGO_HOST=localhost endif +# Macros +define CLEAN_COVERAGE_REPORT + lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info + lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info + lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info + # Remove unit test libraries and libraries developed before contextBroker project init + lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info + lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info + lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info + # app/ contains application itself, not libraries which make sense to measure unit_test coverage + lcov -r coverage/broker.info "*/src/app/*" -o coverage/broker.info +endef + all: prepare_release release di: install_debug @@ -242,15 +255,7 @@ coverage: coverage_functional_test coverage_unit_test # Generate test report echo "Generating coverage report" lcov --add-tracefile BUILD_UNITTEST/coverage.info --add-tracefile BUILD_COVERAGE/coverage.info --output-file coverage/broker.info - lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info - lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info - lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info - # Remove unit test libraries and libraries developed before contextBroker project init - lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info - lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info - lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info - # app/ contains application itself, not libraries which make sense to measure unit_test coverage - lcov -r coverage/broker.info "*/src/app/*" -o coverage/broker.info + $(CLEAN_COVERAGE_REPORT) genhtml -o coverage coverage/broker.info coverage_unit_test: build_unit_test_coverage @@ -266,15 +271,7 @@ coverage_unit_test: build_unit_test_coverage echo "Generating coverage report" lcov --directory BUILD_UNITTEST --capture -b BUILD_UNITTEST --output-file coverage/broker.test.info lcov --add-tracefile coverage/broker.init.info --add-tracefile coverage/broker.test.info --output-file coverage/broker.info - lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info - lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info - lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info - # Remove unit test libraries and libraries developed before contextBroker project init - lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info - lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info - lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info - # app/ contains application itself, not libraries which make sense to measure unit_test coverage - lcov -r coverage/broker.info "*/src/app/*" -o coverage/broker.info + $(CLEAN_COVERAGE_REPORT) genhtml -o coverage coverage/broker.info coverage_functional_test: install_coverage @@ -300,13 +297,7 @@ coverage_functional_test: install_coverage echo "Generating coverage report" lcov --directory BUILD_COVERAGE --capture -b BUILD_COVERAGE --output-file coverage/broker.test.info lcov --add-tracefile coverage/broker.init.info --add-tracefile coverage/broker.test.info --output-file coverage/broker.info - lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info - lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info - lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info - # Remove unit test libraries and libraries developed before contextBroker project init - lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info - lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info - lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info + $(CLEAN_COVERAGE_REPORT) genhtml -o coverage coverage/broker.info valgrind: install_debug From 38e27409d36efe4d31ec393eedc732de461ae7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 23 Aug 2024 15:05:28 +0200 Subject: [PATCH 380/390] FIX cleanup --- doc/manuals/admin/build_source.md | 2 +- makefile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index e4ac040750..b4bf9499f0 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -149,4 +149,4 @@ You can generate coverage reports for the Orion Context Broker using the followi make coverage INSTALL_DIR=~ -*NOTE*: note that functional test relying in debug traces are expected to fail under coverage execution (this is due to the LM_T macros used by the debug traces are disabled in the coverage code building, due to they add "noise" in condition coverage, thus making coverage reports more useful). \ No newline at end of file +*NOTE*: Functional tests relying in debug traces are expected to fail under coverage execution. This is due to the LM_T macros used by the debug traces are disabled in the coverage code build, as they add "noise" in condition coverage. This way coverage reports are more useful. \ No newline at end of file diff --git a/makefile b/makefile index 1ac77aca62..6116537f90 100644 --- a/makefile +++ b/makefile @@ -86,6 +86,7 @@ prepare_debug: compile_info cd BUILD_DEBUG && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DDEBUG=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) prepare_coverage: compile_info + # FIXME: should we use BUILD_COVERAGE for this or use BUILD_DEBUG? (note that for unit test we don't have BUILT_UNITTEST_COVERAGE....) mkdir -p BUILD_COVERAGE || true cd BUILD_COVERAGE && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DCOVERAGE=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) From 799f73aac93382fc7ca8399bbe4f8eea43f8b3b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 23 Aug 2024 16:25:50 +0200 Subject: [PATCH 381/390] FIX improve doc --- doc/manuals/admin/build_source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manuals/admin/build_source.md b/doc/manuals/admin/build_source.md index b4bf9499f0..8d35154db3 100644 --- a/doc/manuals/admin/build_source.md +++ b/doc/manuals/admin/build_source.md @@ -149,4 +149,4 @@ You can generate coverage reports for the Orion Context Broker using the followi make coverage INSTALL_DIR=~ -*NOTE*: Functional tests relying in debug traces are expected to fail under coverage execution. This is due to the LM_T macros used by the debug traces are disabled in the coverage code build, as they add "noise" in condition coverage. This way coverage reports are more useful. \ No newline at end of file +*NOTE*: Functional tests relying in debug traces are expected to fail under coverage execution (e.g. notification_different_sizes or not_posix_regex_idpattern.test). This is due to the LM_T macros used by the debug traces are disabled in the coverage code build, as they add "noise" in condition coverage. This way coverage reports are more useful. \ No newline at end of file From 6d4e9e5dcc66ebf6389493e97f6baa2963b88147 Mon Sep 17 00:00:00 2001 From: Kazuhito Suda Date: Sat, 24 Aug 2024 07:58:03 +0900 Subject: [PATCH 382/390] (JP) Add documentation about coverage framework (#4607) --- doc/manuals.jp/admin/build_source.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manuals.jp/admin/build_source.md b/doc/manuals.jp/admin/build_source.md index 54e7c19e36..ca4e113d3d 100644 --- a/doc/manuals.jp/admin/build_source.md +++ b/doc/manuals.jp/admin/build_source.md @@ -149,3 +149,5 @@ aarch64 アーキテクチャの場合、apt-get を使用して libxslt をイ * カバレッジを実行します make coverage INSTALL_DIR=~ + +*注意*: デバッグ・トレースに依存する機能テストは、カバレッジ実行で失敗すると予想されます (例: notification_different_sizes または not_posix_regex_idpattern.test)。これは、デバッグ トレースで使用される LM_T マクロが条件カバレッジに "ノイズ" を追加するため、カバレッジ・コード・ビルドで無効になっているためです。この方法では、カバレッジ・レポートがより有用になります。 From 074d0461e72b7d6643f6e51103fe02272ca2d7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 27 Aug 2024 09:38:34 +0200 Subject: [PATCH 383/390] FIX CLEAN_COVDERAGE_REPORT macro in makefile --- makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefile b/makefile index 6116537f90..8ce9dd233b 100644 --- a/makefile +++ b/makefile @@ -63,8 +63,6 @@ define CLEAN_COVERAGE_REPORT lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info - # app/ contains application itself, not libraries which make sense to measure unit_test coverage - lcov -r coverage/broker.info "*/src/app/*" -o coverage/broker.info endef all: prepare_release release @@ -273,6 +271,8 @@ coverage_unit_test: build_unit_test_coverage lcov --directory BUILD_UNITTEST --capture -b BUILD_UNITTEST --output-file coverage/broker.test.info lcov --add-tracefile coverage/broker.init.info --add-tracefile coverage/broker.test.info --output-file coverage/broker.info $(CLEAN_COVERAGE_REPORT) + # app/ contains application itself, not libraries which make sense to measure unit_test coverage + lcov -r coverage/broker.info "*/src/app/*" -o coverage/broker.info genhtml -o coverage coverage/broker.info coverage_functional_test: install_coverage From d0fb1aaea9137af4664ff370607bfb5be538526a Mon Sep 17 00:00:00 2001 From: ArqamFarooqui110719 Date: Thu, 29 Aug 2024 11:16:25 +0530 Subject: [PATCH 384/390] updated code as per comment --- src/lib/common/globals.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/common/globals.cpp b/src/lib/common/globals.cpp index 50e761cf83..ec8be53148 100644 --- a/src/lib/common/globals.cpp +++ b/src/lib/common/globals.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include @@ -612,9 +613,10 @@ double parse8601Time(const std::string& ss) // The following 'for' loop is implemented to handle a specific datetime case where the datetime string // is '2016-04-05T14:10:0x.00Z'. This particular case is being incorrectly PASS through the // sscanf() function i.e. used next to this 'for' loop. - for (char c : ss) + for (int i = 0; ss[i] != '\0'; i++) { - if (std::isalpha(c) && c != 'T' && c != 'Z') + char c = ss[i]; + if (isalpha(c) && c != 'T' && c != 'Z') { return -1; } From 0b4627c08ee2df805e504263c84987ed729db4bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 11 Sep 2024 16:25:24 +0200 Subject: [PATCH 385/390] FIX speciality EntityChange altType into different subtypes --- src/lib/apiTypesV2/Subscription.cpp | 17 ++++++++-- src/lib/apiTypesV2/Subscription.h | 12 ++++++- src/lib/cache/subCache.cpp | 8 ++--- src/lib/mongoBackend/MongoCommonUpdate.cpp | 37 ++++++++++++++++------ 4 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/lib/apiTypesV2/Subscription.cpp b/src/lib/apiTypesV2/Subscription.cpp index c5bdbe9fc8..6716848f2f 100644 --- a/src/lib/apiTypesV2/Subscription.cpp +++ b/src/lib/apiTypesV2/Subscription.cpp @@ -42,7 +42,7 @@ ngsiv2::SubAltType parseAlterationType(const std::string& altType) { if (altType == "entityChange") { - return ngsiv2::SubAltType::EntityChange; + return ngsiv2::SubAltType::EntityChangeBothValueAndMetadata; } else if (altType == "entityUpdate") { @@ -64,13 +64,26 @@ ngsiv2::SubAltType parseAlterationType(const std::string& altType) +/* **************************************************************************** +* +* isChangeAltType - +*/ +bool isChangeAltType(ngsiv2::SubAltType altType) +{ + return (altType == ngsiv2::SubAltType::EntityChangeBothValueAndMetadata) || + (altType == ngsiv2::SubAltType::EntityChangeOnlyMetadata) || + (altType == ngsiv2::SubAltType::EntityChangeOnlyValue); +} + + + /* **************************************************************************** * * subAltType2string - */ std::string subAltType2string(ngsiv2::SubAltType altType) { - if (altType == ngsiv2::SubAltType::EntityChange) + if (isChangeAltType(altType)) { return "entityChange"; } diff --git a/src/lib/apiTypesV2/Subscription.h b/src/lib/apiTypesV2/Subscription.h index 1212226096..3f6669472e 100644 --- a/src/lib/apiTypesV2/Subscription.h +++ b/src/lib/apiTypesV2/Subscription.h @@ -57,7 +57,9 @@ typedef enum NotificationType */ typedef enum SubAltType { - EntityChange, + EntityChangeBothValueAndMetadata, // this is the reference one used by parsing/rendering logic + EntityChangeOnlyValue, + EntityChangeOnlyMetadata, EntityUpdate, EntityCreate, EntityDelete, @@ -192,4 +194,12 @@ extern std::string subAltType2string(ngsiv2::SubAltType altType); +/* **************************************************************************** +* +* isChangeAltType - +*/ +extern bool isChangeAltType(ngsiv2::SubAltType altType); + + + #endif // SRC_LIB_APITYPESV2_SUBSCRIPTION_H_ diff --git a/src/lib/cache/subCache.cpp b/src/lib/cache/subCache.cpp index 5f27c0cd9d..6653496e2c 100644 --- a/src/lib/cache/subCache.cpp +++ b/src/lib/cache/subCache.cpp @@ -403,7 +403,7 @@ static bool matchAltType(CachedSubscription* cSubP, ngsiv2::SubAltType targetAlt // If subAltTypeV size == 0 default alteration types are update with change and create if (cSubP->subAltTypeV.size() == 0) { - if ((targetAltType == ngsiv2::SubAltType::EntityChange) || (targetAltType == ngsiv2::SubAltType::EntityCreate)) + if ((isChangeAltType(targetAltType)) || (targetAltType == ngsiv2::SubAltType::EntityCreate)) { return true; } @@ -417,9 +417,9 @@ static bool matchAltType(CachedSubscription* cSubP, ngsiv2::SubAltType targetAlt ngsiv2::SubAltType altType = cSubP->subAltTypeV[ix]; // EntityUpdate is special, it is a "sub-type" of EntityChange - if (targetAltType == ngsiv2::SubAltType::EntityChange) + if (isChangeAltType(targetAltType)) { - if ((altType == ngsiv2::SubAltType::EntityUpdate) || (altType == ngsiv2::SubAltType::EntityChange)) + if ((altType == ngsiv2::SubAltType::EntityUpdate) || (isChangeAltType(altType))) { return true; } @@ -507,7 +507,7 @@ static bool subMatch return false; } } - else if ((targetAltType == ngsiv2::EntityChange) || (targetAltType == ngsiv2::EntityCreate)) + else if ((isChangeAltType(targetAltType)) || (targetAltType == ngsiv2::EntityCreate)) { if (!attributeMatch(cSubP, attrsWithModifiedValue) && !(cSubP->notifyOnMetadataChange && attributeMatch(cSubP, attrsWithModifiedMd))) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 3e5b3b28ff..c0a29c37f1 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -1493,7 +1493,7 @@ static bool matchAltType(orion::BSONObj sub, ngsiv2::SubAltType targetAltType) // change and create. Maybe this could be check at MongoDB query stage, but seems be more complex if (altTypeStrings.size() == 0) { - if ((targetAltType == ngsiv2::SubAltType::EntityChange) || (targetAltType == ngsiv2::SubAltType::EntityCreate)) + if ((isChangeAltType(targetAltType)) || (targetAltType == ngsiv2::SubAltType::EntityCreate)) { return true; } @@ -1513,9 +1513,9 @@ static bool matchAltType(orion::BSONObj sub, ngsiv2::SubAltType targetAltType) else { // EntityUpdate is special, it is a "sub-type" of EntityChange - if (targetAltType == ngsiv2::SubAltType::EntityChange) + if (isChangeAltType(targetAltType)) { - if ((altType == ngsiv2::SubAltType::EntityUpdate) || (altType == ngsiv2::SubAltType::EntityChange)) + if ((altType == ngsiv2::SubAltType::EntityUpdate) || (isChangeAltType(targetAltType))) { return true; } @@ -1659,11 +1659,13 @@ static bool addTriggeredSubscriptions_noCache continue; } } - else if ((targetAltType == ngsiv2::EntityChange) || (targetAltType == ngsiv2::EntityCreate)) + else if ((isChangeAltType(targetAltType)) || (targetAltType == ngsiv2::EntityCreate)) { // Skip if: 1) there is no change in the *value* of attributes listed in conditions.attrs and 2) there is no change // in the *metadata* of the attributes listed in conditions.attrs (the 2) only if notifyOnMetadtaChange is true) - if (!condValueAttrMatch(sub, attrsWithModifiedValue) && !(notifyOnMetadataChange && condValueAttrMatch(sub, attrsWithModifiedMd))) + bool b1 = condValueAttrMatch(sub, attrsWithModifiedValue); + bool b2 = condValueAttrMatch(sub, attrsWithModifiedMd); + if (!b1 && !(notifyOnMetadataChange && b2)) { continue; } @@ -2766,24 +2768,41 @@ static bool processContextAttributeVector { attrsWithModifiedValue.push_back(ca->name); attrsWithModifiedMd.push_back(ca->name); + targetAltType = ngsiv2::SubAltType::EntityChangeBothValueAndMetadata; } else if (changeType == CHANGE_ONLY_VALUE) { attrsWithModifiedValue.push_back(ca->name); + if ((targetAltType == ngsiv2::SubAltType::EntityChangeBothValueAndMetadata) || (targetAltType == ngsiv2::SubAltType::EntityChangeOnlyMetadata)) + { + targetAltType = ngsiv2::SubAltType::EntityChangeBothValueAndMetadata; + } + else + { + targetAltType = ngsiv2::SubAltType::EntityChangeOnlyValue; + } } else if (changeType == CHANGE_ONLY_MD) { attrsWithModifiedMd.push_back(ca->name); + if ((targetAltType == ngsiv2::SubAltType::EntityChangeBothValueAndMetadata) || (targetAltType == ngsiv2::SubAltType::EntityChangeOnlyValue)) + { + targetAltType = ngsiv2::SubAltType::EntityChangeBothValueAndMetadata; + } + else + { + targetAltType = ngsiv2::SubAltType::EntityChangeOnlyMetadata; + } } - attributes.push_back(ca->name); + attributes.push_back(ca->name); /* If actual update then targetAltType changes from EntityUpdate (the value used to initialize * the variable) to EntityChange */ - if (changeType != NO_CHANGE) + /*if (changeType != NO_CHANGE) { targetAltType = ngsiv2::SubAltType::EntityChange; - } + }*/ } /* Add triggered subscriptions */ @@ -3948,7 +3967,7 @@ static unsigned int updateEntity * previous addTriggeredSubscriptions() invocations. Before that, we add * builtin attributes and metadata (both NGSIv1 and NGSIv2 as this is * for notifications and NGSIv2 builtins can be used in NGSIv1 notifications) */ - addBuiltins(notifyCerP, subAltType2string(ngsiv2::SubAltType::EntityChange)); + addBuiltins(notifyCerP, subAltType2string(ngsiv2::SubAltType::EntityChangeBothValueAndMetadata)); unsigned int notifSent = processSubscriptions(subsToNotify, notifyCerP, tenant, From 248e2d7e270c9310d55ec3385e566e71cc6adde7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 11 Sep 2024 16:58:00 +0200 Subject: [PATCH 386/390] FIX little fix in test regression --- src/lib/mongoBackend/MongoCommonUpdate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index c0a29c37f1..758c1cbb0a 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -1515,7 +1515,7 @@ static bool matchAltType(orion::BSONObj sub, ngsiv2::SubAltType targetAltType) // EntityUpdate is special, it is a "sub-type" of EntityChange if (isChangeAltType(targetAltType)) { - if ((altType == ngsiv2::SubAltType::EntityUpdate) || (isChangeAltType(targetAltType))) + if ((altType == ngsiv2::SubAltType::EntityUpdate) || (isChangeAltType(altType))) { return true; } From a152cb9c0649592fc28727b7c7151724d746698e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 11 Sep 2024 17:36:44 +0200 Subject: [PATCH 387/390] FIX metadata modifications are not considered as change (with regards to subscription alterationTypes) if notifyOnMetadataChange is false --- CHANGES_NEXT_RELEASE | 1 + src/lib/apiTypesV2/Subscription.h | 4 +- src/lib/cache/subCache.cpp | 12 +- src/lib/mongoBackend/MongoCommonUpdate.cpp | 24 +- ...ityupdate_with_notifyonmetadatachange.test | 332 ++++++++++++++++++ 5 files changed, 358 insertions(+), 15 deletions(-) create mode 100644 test/functionalTest/cases/4605_alterationtype_entityupdate_with_notifyonmetadatachange/alterationtype_entityupdate_with_notifyonmetadatachange.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index f9628185bc..f705fc336d 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -3,6 +3,7 @@ - Fix: $max and $min operators were not supported with DateTime attributes (#4585) - Fix: wrong date values should not allowed in subscription's expires field (#4541) - Fix: do not raise DB alarm in case of wrong GeoJSON in client request +- Fix: metadata modifications are not considered as change (with regards to subscription alterationTypes) if notifyOnMetadataChange is false (#4605) - Upgrade cjexl version from 0.3.0 to 0.4.0 (new transformations: now, getTime and toIsoString) - Upgrade Debian version from 12.4 to 12.6 in Dockerfile - Fix: invalid date in expires field of subscription (#2303) diff --git a/src/lib/apiTypesV2/Subscription.h b/src/lib/apiTypesV2/Subscription.h index 3f6669472e..88d6660038 100644 --- a/src/lib/apiTypesV2/Subscription.h +++ b/src/lib/apiTypesV2/Subscription.h @@ -57,7 +57,9 @@ typedef enum NotificationType */ typedef enum SubAltType { - EntityChangeBothValueAndMetadata, // this is the reference one used by parsing/rendering logic + // EntityChange has been specialized into three sub-types in order to solve #4605 + // (EntityChangeBothValueAndMetadata is thre reference one used in parsing/rendering logic) + EntityChangeBothValueAndMetadata, EntityChangeOnlyValue, EntityChangeOnlyMetadata, EntityUpdate, diff --git a/src/lib/cache/subCache.cpp b/src/lib/cache/subCache.cpp index 6653496e2c..d19bd7b201 100644 --- a/src/lib/cache/subCache.cpp +++ b/src/lib/cache/subCache.cpp @@ -490,6 +490,11 @@ static bool subMatch return false; } + // If notifyOnMetadataChange is false and only metadata has been changed, we "downgrade" to ngsiv2::EntityUpdate + if (!cSubP->notifyOnMetadataChange && (targetAltType == ngsiv2::EntityChangeOnlyMetadata)) + { + targetAltType = ngsiv2::EntityUpdate; + } // // If one of the attribute names in the scope vector @@ -509,8 +514,11 @@ static bool subMatch } else if ((isChangeAltType(targetAltType)) || (targetAltType == ngsiv2::EntityCreate)) { - if (!attributeMatch(cSubP, attrsWithModifiedValue) && - !(cSubP->notifyOnMetadataChange && attributeMatch(cSubP, attrsWithModifiedMd))) + // No match if: 1) there is no change in the *value* of attributes listed in conditions.attrs and 2) there is no change + // in the *metadata* of the attributes listed in conditions.attrs (the 2) only if notifyOnMetadataChange is true) + bool b1 = attributeMatch(cSubP, attrsWithModifiedValue); + bool b2 = attributeMatch(cSubP, attrsWithModifiedMd); + if (!b1 && !(cSubP->notifyOnMetadataChange && b2)) { LM_T(LmtSubCacheMatch, ("No match due to attributes")); return false; diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 758c1cbb0a..a866f45446 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -1641,15 +1641,21 @@ static bool addTriggeredSubscriptions_noCache if (subs.count(subIdStr) == 0) { + // Early extraction of fiedl from DB document. The rest of fields are got later + bool notifyOnMetadataChange = sub.hasField(CSUB_NOTIFYONMETADATACHANGE)? getBoolFieldF(sub, CSUB_NOTIFYONMETADATACHANGE) : true; + + // If notifyOnMetadataChange is false and only metadata has been changed, we "downgrade" to ngsiv2::EntityUpdate + if (!notifyOnMetadataChange && (targetAltType == ngsiv2::EntityChangeOnlyMetadata)) + { + targetAltType = ngsiv2::EntityUpdate; + } + // Check alteration type if (!matchAltType(sub, targetAltType)) { continue; } - // Early extraction of fiedl from DB document. The rest of fields are got later - bool notifyOnMetadataChange = sub.hasField(CSUB_NOTIFYONMETADATACHANGE)? getBoolFieldF(sub, CSUB_NOTIFYONMETADATACHANGE) : true; - // Depending of the alteration type, we use the list of attributes in the request or the list // with effective modifications. Note that EntityDelete doesn't check the list if (targetAltType == ngsiv2::EntityUpdate) @@ -1662,9 +1668,10 @@ static bool addTriggeredSubscriptions_noCache else if ((isChangeAltType(targetAltType)) || (targetAltType == ngsiv2::EntityCreate)) { // Skip if: 1) there is no change in the *value* of attributes listed in conditions.attrs and 2) there is no change - // in the *metadata* of the attributes listed in conditions.attrs (the 2) only if notifyOnMetadtaChange is true) + // in the *metadata* of the attributes listed in conditions.attrs (the 2) only if notifyOnMetadataChange is true) bool b1 = condValueAttrMatch(sub, attrsWithModifiedValue); bool b2 = condValueAttrMatch(sub, attrsWithModifiedMd); + if (!b1 && !(notifyOnMetadataChange && b2)) { continue; @@ -2795,14 +2802,7 @@ static bool processContextAttributeVector } } - attributes.push_back(ca->name); - - /* If actual update then targetAltType changes from EntityUpdate (the value used to initialize - * the variable) to EntityChange */ - /*if (changeType != NO_CHANGE) - { - targetAltType = ngsiv2::SubAltType::EntityChange; - }*/ + attributes.push_back(ca->name); } /* Add triggered subscriptions */ diff --git a/test/functionalTest/cases/4605_alterationtype_entityupdate_with_notifyonmetadatachange/alterationtype_entityupdate_with_notifyonmetadatachange.test b/test/functionalTest/cases/4605_alterationtype_entityupdate_with_notifyonmetadatachange/alterationtype_entityupdate_with_notifyonmetadatachange.test new file mode 100644 index 0000000000..1e2fc196a5 --- /dev/null +++ b/test/functionalTest/cases/4605_alterationtype_entityupdate_with_notifyonmetadatachange/alterationtype_entityupdate_with_notifyonmetadatachange.test @@ -0,0 +1,332 @@ +# Copyright 2024 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +alterationType entityUpdate in combination with notifyOnMetadataChange + +--SHELL-INIT-- +dbInit CB +brokerStart CB +accumulatorStart --pretty-print + +--SHELL-- + +# +# 01. Create sub with alterationType entityUpdate+entityCreate and notifyOnMetadataChange false +# 02. Create entity dummy-device-01 with two attributes +# 03. Update entity dummy-device-01 metadata in two attributes +# 04. Dump accumulator and see three notifications +# + + +echo "01. Create sub with alterationType entityUpdate+entityCreate and notifyOnMetadataChange false" +echo "=============================================================================================" +payload='{ + "subject": { + "entities": [ + { + "idPattern": ".*", + "type": "DummyDevice" + } + ], + "condition": { + "attrs": [ + "temperature", + "humidity" + ], + "notifyOnMetadataChange": false, + "alterationTypes": [ + "entityUpdate", + "entityCreate" + ] + } + }, + "notification": { + "attrs": [ + "temperature", + "humidity" + ], + "onlyChangedAttrs": true, + "attrsFormat": "normalized", + "http": { + "url": "http://127.0.0.1:'${LISTENER_PORT}'/notify" + }, + "metadata": [ + "TimeInstant" + ] + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" +echo +echo + + +echo "02. Create entity dummy-device-01 with two attributes" +echo "=====================================================" +payload=' +{ + "id": "dummy-device-01", + "type": "DummyDevice", + "temperature": { + "type": "Number", + "value": 20, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:00:00.000Z" + } + } + }, + "humidity": { + "type": "Number", + "value": 50, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:00:00.000Z" + } + } + } +}' +orionCurl --url /v2/entities --payload "$payload" +echo +echo + + +echo "03. Update entity dummy-device-01 metadata in two attributes" +echo "============================================================" +payload='{ + "temperature": { + "type": "Number", + "value": 20, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:10:00.000Z" + } + } + }, + "humidity": { + "type": "Number", + "value": 50, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:10:00.000Z" + } + } + } +}' +orionCurl --url /v2/entities/dummy-device-01/attrs -X POST --payload "$payload" +echo +echo + + +echo "03. Update entity dummy-device-01 without actual update" +echo "=======================================================" +payload='{ + "temperature": { + "type": "Number", + "value": 20 + }, + "humidity": { + "type": "Number", + "value": 50 + } +}' +orionCurl --url /v2/entities/dummy-device-01/attrs -X POST --payload "$payload" +echo +echo + + +echo "04. Dump accumulator and see three notifications" +echo "================================================" +accumulatorDump +echo +echo + + +--REGEXPECT-- +01. Create sub with alterationType entityUpdate+entityCreate and notifyOnMetadataChange false +============================================================================================= +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Create entity dummy-device-01 with two attributes +===================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/entities/dummy-device-01?type=DummyDevice +Content-Length: 0 + + + +03. Update entity dummy-device-01 metadata in two attributes +============================================================ +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +03. Update entity dummy-device-01 without actual update +======================================================= +HTTP/1.1 204 No Content +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + + + +04. Dump accumulator and see three notifications +================================================ +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 347 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "humidity": { + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:00:00.000Z" + } + }, + "type": "Number", + "value": 50 + }, + "id": "dummy-device-01", + "temperature": { + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:00:00.000Z" + } + }, + "type": "Number", + "value": 20 + }, + "type": "DummyDevice" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 347 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "humidity": { + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:10:00.000Z" + } + }, + "type": "Number", + "value": 50 + }, + "id": "dummy-device-01", + "temperature": { + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:10:00.000Z" + } + }, + "type": "Number", + "value": 20 + }, + "type": "DummyDevice" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= +POST http://127.0.0.1:REGEX(\d+)/notify +Fiware-Servicepath: / +Content-Length: 347 +User-Agent: orion/REGEX(\d+\.\d+\.\d+.*) +Ngsiv2-Attrsformat: normalized +Host: 127.0.0.1:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}); cbnotif=1 + +{ + "data": [ + { + "humidity": { + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:10:00.000Z" + } + }, + "type": "Number", + "value": 50 + }, + "id": "dummy-device-01", + "temperature": { + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-08-20T00:10:00.000Z" + } + }, + "type": "Number", + "value": 20 + }, + "type": "DummyDevice" + } + ], + "subscriptionId": "REGEX([0-9a-f]{24})" +} +======================================= + + +--TEARDOWN-- +brokerStop CB +dbDrop CB +accumulatorStop From 27b65087646ee058289ffb7436276478066aaae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 11 Sep 2024 17:47:38 +0200 Subject: [PATCH 388/390] FIX typo --- ...type_entityupdate_with_notifyonmetadatachange.test | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/functionalTest/cases/4605_alterationtype_entityupdate_with_notifyonmetadatachange/alterationtype_entityupdate_with_notifyonmetadatachange.test b/test/functionalTest/cases/4605_alterationtype_entityupdate_with_notifyonmetadatachange/alterationtype_entityupdate_with_notifyonmetadatachange.test index 1e2fc196a5..bbfb1f476a 100644 --- a/test/functionalTest/cases/4605_alterationtype_entityupdate_with_notifyonmetadatachange/alterationtype_entityupdate_with_notifyonmetadatachange.test +++ b/test/functionalTest/cases/4605_alterationtype_entityupdate_with_notifyonmetadatachange/alterationtype_entityupdate_with_notifyonmetadatachange.test @@ -34,7 +34,8 @@ accumulatorStart --pretty-print # 01. Create sub with alterationType entityUpdate+entityCreate and notifyOnMetadataChange false # 02. Create entity dummy-device-01 with two attributes # 03. Update entity dummy-device-01 metadata in two attributes -# 04. Dump accumulator and see three notifications +# 04. Update entity dummy-device-01 without actual update +# 05. Dump accumulator and see three notifications # @@ -141,7 +142,7 @@ echo echo -echo "03. Update entity dummy-device-01 without actual update" +echo "04. Update entity dummy-device-01 without actual update" echo "=======================================================" payload='{ "temperature": { @@ -158,7 +159,7 @@ echo echo -echo "04. Dump accumulator and see three notifications" +echo "05. Dump accumulator and see three notifications" echo "================================================" accumulatorDump echo @@ -194,7 +195,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -03. Update entity dummy-device-01 without actual update +04. Update entity dummy-device-01 without actual update ======================================================= HTTP/1.1 204 No Content Date: REGEX(.*) @@ -202,7 +203,7 @@ Fiware-Correlator: REGEX([0-9a-f\-]{36}) -04. Dump accumulator and see three notifications +05. Dump accumulator and see three notifications ================================================ POST http://127.0.0.1:REGEX(\d+)/notify Fiware-Servicepath: / From 4ce5d3ff250427e771c7e0b7e7b39877aba4bc8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 11 Sep 2024 18:12:43 +0200 Subject: [PATCH 389/390] FIX minor fix --- src/lib/cache/subCache.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/cache/subCache.cpp b/src/lib/cache/subCache.cpp index d19bd7b201..23b1af6c8c 100644 --- a/src/lib/cache/subCache.cpp +++ b/src/lib/cache/subCache.cpp @@ -452,6 +452,12 @@ static bool subMatch ngsiv2::SubAltType targetAltType ) { + // If notifyOnMetadataChange is false and only metadata has been changed, we "downgrade" to ngsiv2::EntityUpdate + if (!cSubP->notifyOnMetadataChange && (targetAltType == ngsiv2::EntityChangeOnlyMetadata)) + { + targetAltType = ngsiv2::EntityUpdate; + } + // Check alteration type if (!matchAltType(cSubP, targetAltType)) { @@ -490,12 +496,6 @@ static bool subMatch return false; } - // If notifyOnMetadataChange is false and only metadata has been changed, we "downgrade" to ngsiv2::EntityUpdate - if (!cSubP->notifyOnMetadataChange && (targetAltType == ngsiv2::EntityChangeOnlyMetadata)) - { - targetAltType = ngsiv2::EntityUpdate; - } - // // If one of the attribute names in the scope vector // of the subscription has the same name as the incoming attribute. there is a match. From a82d726ed813064835bf755515ba44995ae5ba88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 11 Sep 2024 18:16:54 +0200 Subject: [PATCH 390/390] FIX to reference presentation and version numbers --- README.md | 2 +- doc/manuals.jp/devel/README.md | 2 +- doc/manuals/devel/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c3a2834151..61f25363ff 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ recommended to have a look to the brief ### Introductory presentations - Orion Context Broker - [(en)](https://www.slideshare.net/slideshow/orion-context-broker-introduction-20240604/269503234) + [(en)](https://www.slideshare.net/slideshow/orion-context-broker-introduction-20240911/271735060) [(jp)](https://www.slideshare.net/slideshow/orion-context-broker-introduction-20240605/269515246) - NGSIv2 Overview for Developers That Already Know NGSIv1 [(en)](https://www.slideshare.net/fermingalan/orion-context-broker-ngsiv2-overview-for-developers-that-already-know-ngsiv1-20220523) diff --git a/doc/manuals.jp/devel/README.md b/doc/manuals.jp/devel/README.md index bfbf13e1d1..b1e9228816 100644 --- a/doc/manuals.jp/devel/README.md +++ b/doc/manuals.jp/devel/README.md @@ -1,6 +1,6 @@ # 開発マニュアル -*注 : このドキュメントでは、リリース 4.0.x の Orion Context Broker について説明しています。* +*注 : このドキュメントでは、リリース 4.1.x の Orion Context Broker について説明しています。* ## 対象読者 diff --git a/doc/manuals/devel/README.md b/doc/manuals/devel/README.md index dd6d690c79..3f7f9c0af0 100644 --- a/doc/manuals/devel/README.md +++ b/doc/manuals/devel/README.md @@ -1,6 +1,6 @@ # Development Manual -*Note: This document describes Orion Context Broker as of release 4.0.x.* +*Note: This document describes Orion Context Broker as of release 4.1.x.* ## Intended audience The intended audience of this manual is developers that need to understand the internals of the Orion Context Broker