From e53b905d75bb533480af741151ec48d7152858de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CMadhuNEC=E2=80=9D?= Date: Wed, 22 Feb 2023 12:18:55 +0000 Subject: [PATCH 1/4] Improved aggrMethod to accept multiple values --- CHANGES_NEXT_RELEASE | 1 + lib/database/sthDatabase.js | 20 +++++++- lib/server/sthServer.js | 7 ++- test/unit/sthTestUtils.js | 97 ++++++++++++++++++++++++++----------- test/unit/sth_test.js | 42 ++++++++++++++++ 5 files changed, 135 insertions(+), 32 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 9bf13c4b..90a3e5d1 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ +- Improved aggrMethod to accept multiple values (#432, partial) - Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) diff --git a/lib/database/sthDatabase.js b/lib/database/sthDatabase.js index b66a446f..9bc18ae6 100644 --- a/lib/database/sthDatabase.js +++ b/lib/database/sthDatabase.js @@ -760,7 +760,21 @@ function getAggregatedData(data, callback) { 'points.offset': 1, 'points.samples': 1 }; - fieldFilter['points.' + aggregatedFunction] = 1; + function findAggregatedFunction(aggregatedFunction) { + let aggregatedFunctionsArray = []; + if (aggregatedFunction.includes('all')) { + aggregatedFunctionsArray = ['min','max','sum','sum2','occur']; + } else { + aggregatedFunctionsArray = aggregatedFunction.split(','); + } + return aggregatedFunctionsArray; + } + + const aggregatedFunctions = findAggregatedFunction(aggregatedFunction); + + for (let i = 0; i < aggregatedFunctions.length; i++) { + fieldFilter['points.' + aggregatedFunctions[i]] = 1; + } let originFilter; if (from && to) { @@ -783,7 +797,9 @@ function getAggregatedData(data, callback) { offset: '$points.offset', samples: '$points.samples' }; - pushAccumulator[aggregatedFunction] = '$points.' + aggregatedFunction; + for (let i = 0; i < aggregatedFunctions.length; i++) { + pushAccumulator[aggregatedFunctions[i]] = '$points.' + aggregatedFunctions[i]; + } let matchCondition; switch (sthConfig.DATA_MODEL) { diff --git a/lib/server/sthServer.js b/lib/server/sthServer.js index 27152a0d..d4366514 100644 --- a/lib/server/sthServer.js +++ b/lib/server/sthServer.js @@ -63,6 +63,10 @@ function doStartServer(host, port, callback) { reply({ error: 'BadRequest', description: error.output.payload.message }).code(400); } + const list = ['min', 'max', 'sum', 'sum2', 'occur', 'all']; + const joinedList = '(' + list.join('|') + ')'; + const regex = new RegExp('^' + joinedList + '(,' + joinedList + ')*$'); + const config = { validate: { headers: sthHeaderValidator, @@ -74,8 +78,7 @@ function doStartServer(host, port, callback) { // prettier-ignore hOffset: joi.number().integer().greater(-1).optional(), // prettier-ignore - aggrMethod: joi.string().valid( - 'max', 'min', 'sum', 'sum2', 'occur').optional(), + aggrMethod: joi.string().regex(regex).optional(), // prettier-ignore aggrPeriod: joi.string().required().valid( 'month', 'day', 'hour', 'minute', 'second').optional(), diff --git a/test/unit/sthTestUtils.js b/test/unit/sthTestUtils.js index 50d312ae..960f0a51 100644 --- a/test/unit/sthTestUtils.js +++ b/test/unit/sthTestUtils.js @@ -912,21 +912,22 @@ function aggregatedDataAvailableSinceDateTest(ngsiVersion, params, done) { break; } - let value; - switch (aggrMethod) { - case 'min': - case 'max': + let value,valueSum,valueSum2,valueOccur; + const aggrMethods=aggrMethod.split(','); + + for (let i=0;i Date: Tue, 2 May 2023 06:28:34 +0000 Subject: [PATCH 2/4] Updated docs for aggrMethod --- CHANGES_NEXT_RELEASE | 5 +++-- doc/manuals/aggregated-data-retrieval.md | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 90a3e5d1..5dd9d5de 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ -- Improved aggrMethod to accept multiple values (#432, partial) -- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) +- Add: aggrMethod accept multiple values separated by comma (#432, partial) +- Add: aggrMethod 'all' to get all the possible aggregations (#432, partial) +- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) diff --git a/doc/manuals/aggregated-data-retrieval.md b/doc/manuals/aggregated-data-retrieval.md index 6a98c746..f13ff15c 100644 --- a/doc/manuals/aggregated-data-retrieval.md +++ b/doc/manuals/aggregated-data-retrieval.md @@ -19,9 +19,11 @@ The requests for aggregated time series context information can use the followin - **aggrMethod**: The aggregation method. The STH component supports the following aggregation methods: `max` (maximum value), `min` (minimum value), `sum` (sum of all the samples) and `sum2` (sum of the square value of all the - samples) for numeric attribute values and `occur` for attributes values of type string. Combining the information - provided by these aggregated methods with the number of samples, it is possible to calculate probabilistic values - such as the average value, the variance as well as the standard deviation. It is a mandatory parameter. + samples) for numeric attribute values and `occur` for attributes values of type string. It accepts multiple values + separated by comma (min,max) to get multiple aggregation method values. Additionally, `aggrMethod=all` can be used + to get all the aggregation method values. Combining the information provided by these aggregated methods with the + number of samples, it is possible to calculate probabilistic values such as the average value, the variance as well + as the standard deviation. It is a mandatory parameter. - **aggrPeriod**: Aggregation period or resolution. A fixed resolution determines the origin time format and the possible offsets. It is a mandatory parameter. Possible valid resolution values supported by the STH are: `month`, `day`, `hour`, `minute` and `second`. From 3b51b4c42e14b77f331525db1fcd0926a79afffd Mon Sep 17 00:00:00 2001 From: MadhuNEC Date: Fri, 22 Sep 2023 06:35:02 +0000 Subject: [PATCH 3/4] Updated 400 Bad Request test cases --- doc/manuals/aggregated-data-retrieval.md | 2 +- lib/server/sthServer.js | 8 ++--- test/unit/sthTestUtils.js | 42 ++++++++++++++++++++++++ test/unit/sth_test.js | 32 ++++++++++++++++++ 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/doc/manuals/aggregated-data-retrieval.md b/doc/manuals/aggregated-data-retrieval.md index f13ff15c..e7ba416c 100644 --- a/doc/manuals/aggregated-data-retrieval.md +++ b/doc/manuals/aggregated-data-retrieval.md @@ -20,7 +20,7 @@ The requests for aggregated time series context information can use the followin - **aggrMethod**: The aggregation method. The STH component supports the following aggregation methods: `max` (maximum value), `min` (minimum value), `sum` (sum of all the samples) and `sum2` (sum of the square value of all the samples) for numeric attribute values and `occur` for attributes values of type string. It accepts multiple values - separated by comma (min,max) to get multiple aggregation method values. Additionally, `aggrMethod=all` can be used + separated by comma (eg. `aggrMethod=min,max`) to get multiple aggregation method values. Additionally, `aggrMethod=all` can be used to get all the aggregation method values. Combining the information provided by these aggregated methods with the number of samples, it is possible to calculate probabilistic values such as the average value, the variance as well as the standard deviation. It is a mandatory parameter. diff --git a/lib/server/sthServer.js b/lib/server/sthServer.js index d4366514..21b66cd6 100644 --- a/lib/server/sthServer.js +++ b/lib/server/sthServer.js @@ -63,9 +63,9 @@ function doStartServer(host, port, callback) { reply({ error: 'BadRequest', description: error.output.payload.message }).code(400); } - const list = ['min', 'max', 'sum', 'sum2', 'occur', 'all']; - const joinedList = '(' + list.join('|') + ')'; - const regex = new RegExp('^' + joinedList + '(,' + joinedList + ')*$'); + const aggList = ['min', 'max', 'sum', 'sum2', 'occur', 'all']; + const joinedAggList = '(' + aggList.join('|') + ')'; + const aggRegex = new RegExp('^' + joinedAggList + '(,' + joinedAggList + ')*$'); const config = { validate: { @@ -78,7 +78,7 @@ function doStartServer(host, port, callback) { // prettier-ignore hOffset: joi.number().integer().greater(-1).optional(), // prettier-ignore - aggrMethod: joi.string().regex(regex).optional(), + aggrMethod: joi.string().regex(aggRegex).optional(), // prettier-ignore aggrPeriod: joi.string().required().valid( 'month', 'day', 'hour', 'minute', 'second').optional(), diff --git a/test/unit/sthTestUtils.js b/test/unit/sthTestUtils.js index 960f0a51..99b3f98f 100644 --- a/test/unit/sthTestUtils.js +++ b/test/unit/sthTestUtils.js @@ -1794,6 +1794,47 @@ function status200Test(ngsiVersion, options, done) { } } +/** + * Bad Request 400 status test case + * @param ngsiVersion NGSI version to use. Anything different from 2 (included undefined) means v1 + * @param {Object} options Options to generate the URL + * @param {Function} done Callback + */ +function status400Test(ngsiVersion, options, done) { + if (ngsiVersion === 2) { + request( + { + uri: getURL(sthTestConfig.API_OPERATION.READ_V2, options), + method: 'GET', + headers: { + 'Fiware-Service': sthConfig.DEFAULT_SERVICE, + 'Fiware-ServicePath': sthConfig.DEFAULT_SERVICE_PATH + } + }, + function(err, response, body) { + expect(response.statusCode).to.equal(400); + done(); + } + ); + } else { + // FIXME: remove the else branch when NGSIv1 becomes obsolete + request( + { + uri: getURL(sthTestConfig.API_OPERATION.READ, options), + method: 'GET', + headers: { + 'Fiware-Service': sthConfig.DEFAULT_SERVICE, + 'Fiware-ServicePath': sthConfig.DEFAULT_SERVICE_PATH + } + }, + function(err, response, body) { + expect(response.statusCode).to.equal(400); + done(); + } + ); + } +} + /** * Test to check that in case of updating a numeric attribute value aggregated data: * - If the value of the attribute is the same, it is only aggregated once @@ -2579,6 +2620,7 @@ module.exports = { cleanDatabaseSuite, eventNotificationSuite, status200Test, + status400Test, numericAggregatedDataUpdatedTest, textualAggregatedDataUpdatedTest, aggregatedDataNonExistentTest, diff --git a/test/unit/sth_test.js b/test/unit/sth_test.js index 13c39d75..feadc0e2 100644 --- a/test/unit/sth_test.js +++ b/test/unit/sth_test.js @@ -340,6 +340,22 @@ describe('sth tests', function() { }) ); + it( + 'should respond with 400 - Bad Request if aggrMethod is not from [min,max,sum,sum2,occur,all]', + sthTestUtils.status400Test.bind(null, 2, { + aggrMethod: 'foo', + aggrPeriod: 'second' + }) + ); + + it( + 'should respond with 400 - Bad Request if aggrMethod are multiple and not from [min,max,sum,sum2,occur,all]', + sthTestUtils.status400Test.bind(null, 2, { + aggrMethod: 'foo,all', + aggrPeriod: 'second' + }) + ); + it( 'should respond with 200 - OK if aggrMethod and aggrPeriod query params - NGSIv1', sthTestUtils.status200Test.bind(null, 1, { @@ -363,6 +379,22 @@ describe('sth tests', function() { aggrPeriod: 'second' }) ); + + it( + 'should respond with 400 - Bad Request if aggrMethod is not from [min,max,sum,sum2,occur,all] - NGSIv1', + sthTestUtils.status400Test.bind(null, 1, { + aggrMethod: 'foo', + aggrPeriod: 'second' + }) + ); + + it( + 'should respond with 400 - Bad Request if aggrMethod are multiple and not from [min,max,sum,sum2,occur,all] - NGSIv1', + sthTestUtils.status400Test.bind(null, 1, { + aggrMethod: 'foo,all', + aggrPeriod: 'second' + }) + ); }); function eachEventTestSuiteContainer(attrName, attrType, includeTimeInstantMetadata) { From 885c78c1509c6147dfc965354d3b62a1cd6d436d Mon Sep 17 00:00:00 2001 From: MadhuNEC Date: Fri, 22 Sep 2023 08:15:08 +0000 Subject: [PATCH 4/4] Fixed lint error --- test/unit/sthTestUtils.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/unit/sthTestUtils.js b/test/unit/sthTestUtils.js index 99b3f98f..e4f134c7 100644 --- a/test/unit/sthTestUtils.js +++ b/test/unit/sthTestUtils.js @@ -912,10 +912,10 @@ function aggregatedDataAvailableSinceDateTest(ngsiVersion, params, done) { break; } - let value,valueSum,valueSum2,valueOccur; - const aggrMethods=aggrMethod.split(','); + let value, valueSum, valueSum2, valueOccur; + const aggrMethods = aggrMethod.split(','); - for (let i=0;i