Skip to content

Commit

Permalink
Merge pull request #1473 from FIWARE/issue/1418-2
Browse files Browse the repository at this point in the history
Fixed issue #1418
  • Loading branch information
kzangeli authored Nov 17, 2023
2 parents 2565788 + 43d1d92 commit 10081c0
Show file tree
Hide file tree
Showing 5 changed files with 223 additions and 61 deletions.
2 changes: 2 additions & 0 deletions CHANGES_NEXT_RELEASE
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,5 @@ Fixed issues:
* #1456 - Bug fix - entity id+type duplicated in forwarded request of "Create Entity"
* #1458 - Supporting system timestamps (createdAt/modifiedAt) in q
* #1451 - Bug fix - removed a trailing ampersand from the URI for the connection to mongodb
* #1418 - Performance - Faster startup when there are GeoProperties in DB

137 changes: 85 additions & 52 deletions src/lib/orionld/mongoc/mongocGeoIndexInit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -49,17 +49,8 @@ extern "C"
//
bool mongocGeoIndexInit(void)
{
bson_t mongoFilter;
mongoc_cursor_t* mongoCursorP;
const bson_t* mongoDocP;
bson_t* options = BCON_NEW("projection", "{",
"attrs", BCON_BOOL(true),
"_id", BCON_BOOL(false),
"}");
//
// Create the filter for the query - no restriction, we want all entities!
//
bson_init(&mongoFilter);

//
// DB Connection
Expand All @@ -82,14 +73,83 @@ bool mongocGeoIndexInit(void)
//
mCollectionP = mongoc_client_get_collection(orionldState.mongoc.client, tenantP->mongoDbName, "entities");

// Aggregation pipeline

// {
// "pipeline":
// [
// {
// $project: {
// attributeKeys: {
// $filter: {
// input: { $objectToArray: "$attrs" },
// as: "item",
// cond: { $eq: ["$$item.v.type", "GeoProperty"] }
// }
// }
// }
// },
// {
// $unwind: "$attributeKeys"
// },
// {
// $group: {
// _id: "$attributeKeys.k"
// }
// }
// ]
// }

bson_t* pipeline = bson_new();
bson_t* project;
bson_t* unwind;
bson_t* group;
bson_t* objectArray = bson_new();

project = BCON_NEW("$project", "{",
"attributeKeys", "{",
"$filter", "{",
"input", "{", "$objectToArray", BCON_UTF8("$attrs"), "}",
"as", BCON_UTF8("item"),
"cond", "{", "$eq", "[", BCON_UTF8("$$item.v.type"), BCON_UTF8("GeoProperty"), "]", "}",
"}",
"}",
"}");

unwind = BCON_NEW("$unwind", BCON_UTF8("$attributeKeys"));

group = BCON_NEW("$group", "{",
"_id", BCON_UTF8("$attributeKeys.k"),
"}");

bson_append_document(objectArray, "0", 1, project);
bson_append_document(objectArray, "1", 1, unwind);
bson_append_document(objectArray, "2", 1, group);

// Append the array to the document
bson_append_array(pipeline, "pipeline", 8, objectArray);

// Print the BSON document as a JSON string
if (lmTraceIsSet(LmtMongoc))
{
char* str = bson_as_relaxed_extended_json(pipeline, NULL);
LM_T(LmtMongoc, ("%s", str));
}

//
// Run the query
//
if ((mongoCursorP = mongoc_collection_find_with_opts(mCollectionP, &mongoFilter, options, NULL)) == NULL)
mongoCursorP = mongoc_collection_aggregate(
mCollectionP,
MONGOC_QUERY_NONE,
pipeline,
NULL, // Additional options, can be NULL for default options
NULL); // Result (pass NULL if you don't need it)


if (mongoCursorP == NULL)
{
LM_E(("Internal Error (mongoc_collection_find_with_opts ERROR)"));
bson_destroy(options);
bson_destroy(&mongoFilter);
LM_E(("Internal Error (mongoc_collection_aggregate ERROR)"));
mongoc_collection_destroy(mCollectionP);
mongoc_cursor_destroy(mongoCursorP);
return NULL;
Expand All @@ -99,57 +159,30 @@ bool mongocGeoIndexInit(void)
{
char* title;
char* detail;
KjNode* entityNodeP;
KjNode* attrsP;
KjNode* _idObjectP;
KjNode* _idNodeP;

entityNodeP = mongocKjTreeFromBson(mongoDocP, &title, &detail);
if (entityNodeP == NULL)
{
LM_W(("mongocKjTreeFromBson failed"));
continue;
}
_idObjectP = mongocKjTreeFromBson(mongoDocP, &title, &detail);
_idNodeP = (_idObjectP == NULL)? NULL : kjLookup(_idObjectP, "_id");

attrsP = entityNodeP->value.firstChildP;
if (attrsP == NULL) // Entity without attributes ?
if (_idNodeP == NULL)
{
LM_W(("Entity without attributes?"));
LM_W(("_idNodeP is NULL"));
continue;
}

//
// Foreach Attribute, check if GeoProperty and if so create its geo index
//
for (KjNode* attrP = attrsP->value.firstChildP; attrP != NULL; attrP = attrP->next)
{
KjNode* typeP = kjLookup(attrP, "type");

if (typeP == NULL)
{
LM_E(("Database Error (attribute '%s' has no 'type' field)", attrP->name));
continue;
}

if (typeP->type != KjString)
{
LM_E(("Database Error (attribute with a 'type' field that is not a string)"));
continue;
}

if (strcmp(typeP->value.s, "GeoProperty") == 0)
{
if (dbGeoIndexLookup(tenantP->tenant, attrP->name) == NULL)
mongocGeoIndexCreate(tenantP, attrP->name);
}
}
char* geoPropertyName = _idNodeP->value.s;

LM_T(LmtMongoc, ("Found geoProperty: '%s'", geoPropertyName));

if (dbGeoIndexLookup(tenantP->tenant, geoPropertyName) == NULL)
mongocGeoIndexCreate(tenantP, geoPropertyName);
}

mongoc_cursor_destroy(mongoCursorP);
mongoc_collection_destroy(mCollectionP);
tenantP = tenantP->next;
}

bson_destroy(options);
bson_destroy(&mongoFilter);

return true;
}
20 changes: 12 additions & 8 deletions test/functionalTest/cases/0000_ngsild/ngsild_geoindexes.test
Original file line number Diff line number Diff line change
Expand Up @@ -280,26 +280,28 @@ Date: REGEX(.*)
Link: <https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-contextREGEX(.*)

[
#SORT_START
{
"tenant": "tn1",
"attribute": "g2"
"attribute": "g1"
},
{
"tenant": "tn1",
"attribute": "g1"
"attribute": "g2"
},
{
"tenant": "",
"attribute": "g3"
"attribute": "g1"
},
{
"tenant": "",
"attribute": "g2"
},
{
"tenant": "",
"attribute": "g1"
"attribute": "g3"
}
#SORT_END
]


Expand Down Expand Up @@ -333,30 +335,32 @@ Date: REGEX(.*)
Link: <https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-contextREGEX(.*)

[
#SORT_START
{
"tenant": "tn3",
"attribute": "g1"
},
{
"tenant": "tn1",
"attribute": "g2"
"attribute": "g1"
},
{
"tenant": "tn1",
"attribute": "g1"
"attribute": "g2"
},
{
"tenant": "",
"attribute": "g3"
"attribute": "g1"
},
{
"tenant": "",
"attribute": "g2"
},
{
"tenant": "",
"attribute": "g1"
"attribute": "g3"
}
#SORT_END
]


Expand Down
121 changes: 121 additions & 0 deletions test/functionalTest/cases/0000_ngsild/ngsild_issue_1418.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# Copyright 2023 FIWARE Foundation e.V.
#
# This file is part of Orion-LD Context Broker.
#
# Orion-LD Context Broker is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Orion-LD Context Broker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/.
#
# For those usages not covered by this license please contact with
# orionld at fiware dot org

# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh

--NAME--
Batch Create with an array of three entities, see issue #776

--SHELL-INIT--
dbInit CB
orionldStart CB -mongocOnly

--SHELL--

#
# 01. Batch create 1000 entities with 2 geoproperties each
# 02. Kill and restart the broker (and measure the time it takes to start)
#

echo "01. Batch create 1000 entities with 2 geoproperties each"
echo "========================================================"
typeset -i eId
eId=1
while [ $eId -le 1000 ]
do
entity='{
"id": "urn:ngsi-ld:TemperatureSensor:'$eId'",
"type": "TemperatureSensor",
"location": {
"type": "GeoProperty",
"value": {
"type": "Point",
"coordinates": [1,2]
}
},
"locationArea": {
"type": "GeoProperty",
"value": {
"type": "Point",
"coordinates": [1,2]
}
},
"p1": 1,
"p2": 1,
"p3": 1,
"p4": 1,
"p5": 1,
"p6": 1,
"p7": 1,
"p8": 1,
"p9": 1,
"p10": 1,
"p11": 1,
"p12": 1,
"p13": 1,
"p14": 1,
"p15": 1,
"p16": 1,
"p17": 1,
"p18": 1,
"p19": 1
}'
orionCurl --url /ngsi-ld/v1/entities --payload "$entity"
eId=$eId+1
done | grep Location | wc -l


echo
echo

echo "02. Kill and restart the broker (and measure the time it takes to start)"
echo "========================================================================"
brokerStop CB
typeset -i before
typeset -i after
typeset -i diff
before=$(date +%s)
orionldStart CB -mongocOnly -t 222
after=$(date +%s)
diff=$after-$before
if [ $diff -gt 2 ]
then
echo Too slow: $diff seconds!!
else
echo OK!
fi
echo
echo


--REGEXPECT--
01. Batch create 1000 entities with 2 geoproperties each
========================================================
1000


02. Kill and restart the broker (and measure the time it takes to start)
========================================================================
OK!


--TEARDOWN--
brokerStop CB
dbDrop CB
4 changes: 3 additions & 1 deletion test/functionalTest/testHarness.sh
Original file line number Diff line number Diff line change
Expand Up @@ -985,9 +985,11 @@ function partExecute()
#
grep -v "already exists" $dirname/$filename.$what.stderr > $dirname/$filename.$what.stderr2
grep -v "mongoc: falling back to malloc for counters." $dirname/$filename.$what.stderr2 > $dirname/$filename.$what.stderr3
grep -v "mongoc: Falling back to malloc for counters." $dirname/$filename.$what.stderr3 > $dirname/$filename.$what.stderr
grep -v "mongoc: Falling back to malloc for counters." $dirname/$filename.$what.stderr3 > $dirname/$filename.$what.stderr4
grep -v "screen size is bogus" $dirname/$filename.$what.stderr4 > $dirname/$filename.$what.stderr
rm -f $dirname/$filename.$what.stderr2
rm -f $dirname/$filename.$what.stderr3
rm -f $dirname/$filename.$what.stderr4

#
# Check that stderr is empty
Expand Down

0 comments on commit 10081c0

Please sign in to comment.