From 89b51be8f59b02fab07ca5030562aa12bf396cb4 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 19 Oct 2023 15:49:40 +0200 Subject: [PATCH 1/7] update doc --- .../ngsi_mongo_sink.md | 20 ++++++ .../ngsi_sth_sink.md | 63 ++++++++++++------- 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index d7b1cd6dd..fa47f5c47 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -13,6 +13,7 @@ Content: * [Database and table names](#section1.3.2) * [Row-like storing](#section1.3.3) * [Column-like storing](#section1.3.4) + * [MongoDB Indexes](#section1.4) * [Administration guide](#section2) * [Configuration](#section2.1) * [Use cases](#section2.2) @@ -286,6 +287,25 @@ If `data_model=dm-by-entity` and `attr_persistence=column` then `NGSIMongoSink` [Top](#top) +### MongoDB Indexes + +Since version 3.0.0 of cygnus index named `cyg_agg_opt` is created in this way but depending on datamodel: + +| datamodel | keys | +| :--------------- | :------------------------------------------------- | +| DMBYSERVICEPATH | entityId, entityType, attrName, resolution, origin | +| DMBYENTITY | attrName, resolution, origin | +| DMBYATTRIBUTE | resolution, origin | + +Since version 3.0.0 of cygnus an index named `cyg_raw_opt` is created in this way but depending on datamodel: + +| datamodel | keys | +| :-------------- | :--------------------------------------------------------------- | +| DMBYSERVICEPATH | recvTime, entityId, entityType, attrName, attrType, attrValue | +| DMBYENTITY | recvTime, attrName, attrType, attrValue | +| DMBYATTRIBUTE | recvTime, attrType, attrValue | + + ## Administration guide ### Configuration `NGSIMongoSink` is configured through the following parameters: diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md index ae6ca7437..ae7e2656f 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md @@ -11,6 +11,7 @@ Content: * [`NGSIEvent`](#section1.3.1) * [Database and table names](#section1.3.2) * [Storing](#section1.3.3) + * [MongoDB Indexes](#section1.4) * [Administration guide](#section2) * [Configuration](#section2.1) * [Use cases](#section2.2) @@ -120,30 +121,30 @@ Assuming the following `NGSIEvent` is created from a notified NGSI context data ngsi-event={ headers={ - content-type=application/json, - timestamp=1429535775, - transactionId=1429535775-308-0000000000, - correlationId=1429535775-308-0000000000, - fiware-service=vehicles, - fiware-servicepath=/4wheels, - + content-type=application/json, + timestamp=1429535775, + transactionId=1429535775-308-0000000000, + correlationId=1429535775-308-0000000000, + fiware-service=vehicles, + fiware-servicepath=/4wheels, + }, body={ - entityId=car1, - entityType=car, - attributes=[ - { - attrName=speed, - attrType=float, - attrValue=112.9 - }, - { - attrName=oil_level, - attrType=float, - attrValue=74.6 - } - ] - } + entityId=car1, + entityType=car, + attributes=[ + { + attrName=speed, + attrType=float, + attrValue=112.9 + }, + { + attrName=oil_level, + attrType=float, + attrValue=74.6 + } + ] + } } @@ -288,6 +289,24 @@ Assuming `data_model=dm-by-entity` and all the possible resolutions as configura [Top](#top) +### MongoDB Indexes + +Since version 3.0.0 of cygnus index named `cyg_agg_opt` is created in this way but depending on datamodel: + +| datamodel | keys | +| :--------------- | :------------------------------------------------- | +| DMBYSERVICEPATH | entityId, entityType, attrName, resolution, origin | +| DMBYENTITY | attrName, resolution, origin | +| DMBYATTRIBUTE | resolution, origin | + +Since version 3.0.0 of cygnus an index named `cyg_raw_opt` is created in this way but depending on datamodel: + +| datamodel | keys | +| :-------------- | :--------------------------------------------------------------- | +| DMBYSERVICEPATH | recvTime, entityId, entityType, attrName, attrType, attrValue | +| DMBYENTITY | recvTime, attrName, attrType, attrValue | +| DMBYATTRIBUTE | recvTime, attrType, attrValue | + ## Administration guide ### Configuration `NGSISTHSink` is configured through the following parameters: From 1c9f22f8d13d20ea7bf63860615ead0ef0ce1f6f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 20 Oct 2023 09:39:36 +0200 Subject: [PATCH 2/7] Update ngsi_mongo_sink.md --- .../flume_extensions_catalogue/ngsi_mongo_sink.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index fa47f5c47..27f5d01b4 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -288,15 +288,6 @@ If `data_model=dm-by-entity` and `attr_persistence=column` then `NGSIMongoSink` [Top](#top) ### MongoDB Indexes - -Since version 3.0.0 of cygnus index named `cyg_agg_opt` is created in this way but depending on datamodel: - -| datamodel | keys | -| :--------------- | :------------------------------------------------- | -| DMBYSERVICEPATH | entityId, entityType, attrName, resolution, origin | -| DMBYENTITY | attrName, resolution, origin | -| DMBYATTRIBUTE | resolution, origin | - Since version 3.0.0 of cygnus an index named `cyg_raw_opt` is created in this way but depending on datamodel: | datamodel | keys | From 698d5be8b28a63d3e072d107188ae23c857a3fb1 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 20 Oct 2023 09:40:28 +0200 Subject: [PATCH 3/7] Update ngsi_sth_sink.md --- .../flume_extensions_catalogue/ngsi_sth_sink.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md index ae7e2656f..74b0530c6 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md @@ -299,14 +299,6 @@ Since version 3.0.0 of cygnus index named `cyg_agg_opt` is created in this way b | DMBYENTITY | attrName, resolution, origin | | DMBYATTRIBUTE | resolution, origin | -Since version 3.0.0 of cygnus an index named `cyg_raw_opt` is created in this way but depending on datamodel: - -| datamodel | keys | -| :-------------- | :--------------------------------------------------------------- | -| DMBYSERVICEPATH | recvTime, entityId, entityType, attrName, attrType, attrValue | -| DMBYENTITY | recvTime, attrName, attrType, attrValue | -| DMBYATTRIBUTE | recvTime, attrType, attrValue | - ## Administration guide ### Configuration `NGSISTHSink` is configured through the following parameters: From 49a579c04992c578286c50f233ee60bd3bc70953 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 20 Oct 2023 12:59:15 +0200 Subject: [PATCH 4/7] update datamodel types --- .../flume_extensions_catalogue/ngsi_mongo_sink.md | 10 +++++----- .../flume_extensions_catalogue/ngsi_sth_sink.md | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index 27f5d01b4..e5ee961a3 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -290,11 +290,11 @@ If `data_model=dm-by-entity` and `attr_persistence=column` then `NGSIMongoSink` ### MongoDB Indexes Since version 3.0.0 of cygnus an index named `cyg_raw_opt` is created in this way but depending on datamodel: -| datamodel | keys | -| :-------------- | :--------------------------------------------------------------- | -| DMBYSERVICEPATH | recvTime, entityId, entityType, attrName, attrType, attrValue | -| DMBYENTITY | recvTime, attrName, attrType, attrValue | -| DMBYATTRIBUTE | recvTime, attrType, attrValue | +| datamodel | keys | +| :---------------- | :--------------------------------------------------------------- | +| dm-by-servicepath | recvTime, entityId, entityType, attrName, attrType, attrValue | +| dm-by-entity | recvTime, attrName, attrType, attrValue | +| dm-by-attribute | recvTime, attrType, attrValue | ## Administration guide diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md index 74b0530c6..d3a51cfce 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md @@ -293,11 +293,11 @@ Assuming `data_model=dm-by-entity` and all the possible resolutions as configura Since version 3.0.0 of cygnus index named `cyg_agg_opt` is created in this way but depending on datamodel: -| datamodel | keys | -| :--------------- | :------------------------------------------------- | -| DMBYSERVICEPATH | entityId, entityType, attrName, resolution, origin | -| DMBYENTITY | attrName, resolution, origin | -| DMBYATTRIBUTE | resolution, origin | +| datamodel | keys | +| :----------------- | :------------------------------------------------- | +| dm-by-service-path | entityId, entityType, attrName, resolution, origin | +| dm-by-entity | attrName, resolution, origin | +| dm-by-attribute | resolution, origin | ## Administration guide ### Configuration From fb337ff3165991d38aa3a17751d81ec5ec10013e Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 23 Oct 2023 13:00:42 +0200 Subject: [PATCH 5/7] Update ngsi_mongo_sink.md --- doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index e5ee961a3..ad57f0081 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -296,6 +296,7 @@ Since version 3.0.0 of cygnus an index named `cyg_raw_opt` is created in this wa | dm-by-entity | recvTime, attrName, attrType, attrValue | | dm-by-attribute | recvTime, attrType, attrValue | +Note that attempting to use any other datamodel different from these ones will cause a "Unknown data model" ERROR in log traces. In addition, note that default datamodel for STH/Mongo sink is dm-by-entity as all others sinks which uses datamodel. ## Administration guide ### Configuration From 99ac83f9441bdb53ed49fe829e91a54caf59b37a Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 23 Oct 2023 13:01:18 +0200 Subject: [PATCH 6/7] Update ngsi_sth_sink.md --- doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md index d3a51cfce..7d6e90902 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md @@ -299,6 +299,9 @@ Since version 3.0.0 of cygnus index named `cyg_agg_opt` is created in this way b | dm-by-entity | attrName, resolution, origin | | dm-by-attribute | resolution, origin | + +Note that attempting to use any other datamodel different from these ones will cause a "Unknown data model" ERROR in log traces. In addition, note that default datamodel for STH/Mongo sink is dm-by-entity as all others sinks which uses datamodel. + ## Administration guide ### Configuration `NGSISTHSink` is configured through the following parameters: From e2f8151d2d762c46a4bd8860f7082869c2580e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 23 Oct 2023 13:08:01 +0200 Subject: [PATCH 7/7] Apply suggestions from code review --- doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md | 2 +- doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index ad57f0081..1bd8299e4 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -296,7 +296,7 @@ Since version 3.0.0 of cygnus an index named `cyg_raw_opt` is created in this wa | dm-by-entity | recvTime, attrName, attrType, attrValue | | dm-by-attribute | recvTime, attrType, attrValue | -Note that attempting to use any other datamodel different from these ones will cause a "Unknown data model" ERROR in log traces. In addition, note that default datamodel for STH/Mongo sink is dm-by-entity as all others sinks which uses datamodel. +Note that attempting to use any other datamodel different from these ones will cause a "Unknown data model" ERROR in log traces. In addition, note that default datamodel for NGSIMongoSink is dm-by-entity as all others sinks which uses datamodel. ## Administration guide ### Configuration diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md index 7d6e90902..346763010 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_sth_sink.md @@ -300,7 +300,7 @@ Since version 3.0.0 of cygnus index named `cyg_agg_opt` is created in this way b | dm-by-attribute | resolution, origin | -Note that attempting to use any other datamodel different from these ones will cause a "Unknown data model" ERROR in log traces. In addition, note that default datamodel for STH/Mongo sink is dm-by-entity as all others sinks which uses datamodel. +Note that attempting to use any other datamodel different from these ones will cause a "Unknown data model" ERROR in log traces. In addition, note that default datamodel for NGSISTHSink is dm-by-entity as all others sinks which uses datamodel. ## Administration guide ### Configuration