diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 25e0be54d..05feb0873 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ -- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (telefonicaid/iotagent-json#827) +- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825, #1612) +- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) diff --git a/doc/api.md b/doc/api.md index 19970cfc7..b614a2799 100644 --- a/doc/api.md +++ b/doc/api.md @@ -32,6 +32,7 @@ - [Measurement transformation order](#measurement-transformation-order) - [Multientity measurement transformation support (`object_id`)](#multientity-measurement-transformation-support-object_id) - [Timestamp Processing](#timestamp-processing) + - [Multimeasure support](#multimeasure-support) - [Overriding global Context Broker host](#overriding-global-context-broker-host) - [Multitenancy, FIWARE Service and FIWARE ServicePath](#multitenancy-fiware-service-and-fiware-servicepath) - [Secured access to the Context Broker](#secured-access-to-the-context-broker) @@ -1051,6 +1052,127 @@ Some additional considerations to take into account: measure of after a mapping, as described in the previous bullet) then it is refused (so a failover to server timestamp will take place). +## Multimeasure support + +A device could receive several measures at the same time. + +For example: + +```json +[ + { + "vol": 0 + }, + { + "vol": 1 + }, + { + "vol": 2 + } +] +``` + +In this case a batch update (`POST /v2/op/update`) to CB will be generated with the following NGSI v2 payload: + +```json +{ + "actionType": "append", + "entities": [ + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 0 + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 1 + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 1 + } + } + ] +} +``` + +Moreover if a multimeasure contains TimeInstant attribute, then CB update is sorted by attribute TimeInstant: + +For example: + +```json +[ + { + "vol": 0, + "TimeInstant": "2024-04-10T10:15:00Z" + }, + { + "vol": 1, + "TimeInstant": "2024-04-10T10:10:00Z" + }, + { + "vol": 2, + "TimeInstant": "2024-04-10T10:05:00Z" + } +] +``` + +In this case a batch update (`POST /v2/op/update`) to CB will be generated with the following NGSI v2 payload: + +```json +{ + "actionType": "append", + "entities": [ + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 2 + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-04-10T10:05:00Z" + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 1 + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-04-10T10:10:00Z" + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 0 + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-04-10T10:15:00Z" + } + } + ] +} +``` + ## Overriding global Context Broker host **cbHost**: Context Broker host URL. This option can be used to override the global CB configuration for specific types diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index dc5e3b733..56c0a9ede 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -260,7 +260,7 @@ function sendQueryValueNgsi2(entityName, attributes, typeInformation, token, cal * @param {Object} typeInformation Configuration information for the device. * @param {String} token User token to identify against the PEP Proxies (optional). */ -function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, callback) { +function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, token, callback) { //aux function used to builf JEXL context. //it returns a flat object from an Attr array function reduceAttrToPlainObject(attrs, initObj = {}) { @@ -273,339 +273,354 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call return initObj; } } - - let entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entoties data striucture - let jexlctxt = {}; //will store the whole context (not just for JEXL) - let payload = {}; //will store the final payload - let plainMeasures = null; //will contain measures POJO - let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); - //Make a clone and overwrite - typeInformation = JSON.parse(JSON.stringify(typeInformation)); + let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(originTypeInformation); //Check mandatory information: type - if (!typeInformation || !typeInformation.type) { - callback(new errors.TypeNotFound(null, entityName, typeInformation)); + if (!originTypeInformation || !originTypeInformation.type) { + callback(new errors.TypeNotFound(null, entityName, originTypeInformation)); return; } - //Rename all measures with matches with id and type to measure_id and measure_type - for (let measure of measures) { - if (measure.name === 'id' || measure.name === 'type') { - measure.name = constants.MEASURE + measure.name; - } - } - //Make a copy of measures in an plain object: plainMeasures - plainMeasures = reduceAttrToPlainObject(measures); - - //Build the initital JEXL Context - //All the measures (avoid references make another copy instead) - jexlctxt = reduceAttrToPlainObject(measures); - //All the static - jexlctxt = reduceAttrToPlainObject(typeInformation.staticAttributes, jexlctxt); - //id type Service and Subservice - jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt); + let payload = {}; //will store the final payload + let entities = {}; + payload.actionType = 'append'; + payload.entities = []; + const currentIsoDate = new Date().toISOString(); + const currentMoment = moment(currentIsoDate); //Managing timestamp (mustInsertTimeInstant flag to decide if we should insert Timestamp later on) - const mustInsertTimeInstant = typeInformation.timestamp !== undefined ? typeInformation.timestamp : false; - - logger.debug( - context, - 'sendUpdateValueNgsi2 called with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j', - entityName, - plainMeasures, - typeInformation, - jexlctxt, - mustInsertTimeInstant - ); + const mustInsertTimeInstant = + originTypeInformation.timestamp !== undefined ? originTypeInformation.timestamp : false; - //Now we can calculate the EntityName of primary entity - let entityNameCalc = null; - if (typeInformation.entityNameExp !== undefined && typeInformation.entityNameExp !== '') { - try { - logger.debug(context, 'sendUpdateValueNgsi2 entityNameExp %j', typeInformation.entityNameExp); - entityNameCalc = expressionPlugin.applyExpression(typeInformation.entityNameExp, jexlctxt, typeInformation); - } catch (e) { - logger.debug( - context, - 'Error evaluating expression for entityName: %j with context: %j', - typeInformation.entityNameExp, - jexlctxt - ); - } + // Check if measures is a single measure or a array of measures (a multimeasure) + if (originMeasures[0] && !originMeasures[0][0]) { + originMeasures = [originMeasures]; } + for (let measures of originMeasures) { + entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entities data structure + let jexlctxt = {}; //will store the whole context (not just for JEXL) + + let plainMeasures = null; //will contain measures POJO + //Make a clone and overwrite + let typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); + + //Rename all measures with matches with id and type to measure_id and measure_type + for (let measure of measures) { + if (measure.name === 'id' || measure.name === 'type') { + measure.name = constants.MEASURE + measure.name; + } + } - entityName = entityNameCalc ? entityNameCalc : entityName; - //enrich JEXL context - jexlctxt['entity_name'] = entityName; + //Make a copy of measures in an plain object: plainMeasures + plainMeasures = reduceAttrToPlainObject(measures); + //Build the initital JEXL Context + //All the measures (avoid references make another copy instead) + jexlctxt = reduceAttrToPlainObject(measures); + //All the static + jexlctxt = reduceAttrToPlainObject(typeInformation.staticAttributes, jexlctxt); + //id type Service and Subservice + jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt); - let preprocessedAttr = []; - //Add Raw Static, Lazy, Command and Actives attr attributes - if (typeInformation && typeInformation.staticAttributes) { - preprocessedAttr = preprocessedAttr.concat(typeInformation.staticAttributes); - } - if (typeInformation && typeInformation.lazy) { - preprocessedAttr = preprocessedAttr.concat(typeInformation.lazy); - } - if (typeInformation && typeInformation.active) { - preprocessedAttr = preprocessedAttr.concat(typeInformation.active); - } + logger.debug( + context, + 'sendUpdateValueNgsi2 loop with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j', + entityName, + plainMeasures, + typeInformation, + jexlctxt, + mustInsertTimeInstant + ); - //Proccess every proto Attribute to populate entities data steuture - entities[entityName] = {}; - entities[entityName][typeInformation.type] = []; - - for (let currentAttr of preprocessedAttr) { - let hitted = false; //any measure, expressiom or value hit the attr (avoid propagate "silent attr" with null values ) - let attrEntityName = entityName; - let attrEntityType = typeInformation.type; - let valueExpression = null; - //manage active attr without object__id (name by default) - currentAttr.object_id = currentAttr.object_id ? currentAttr.object_id : currentAttr.name; - //Enrich the attr (skip, hit, value, meta-timeInstant) - currentAttr.skipValue = currentAttr.skipValue ? currentAttr.skipValue : null; - - //determine AttrEntityName for multientity - if ( - currentAttr.entity_name !== null && - currentAttr.entity_name !== undefined && - currentAttr.entity_name !== '' && - typeof currentAttr.entity_name == 'string' - ) { + //Now we can calculate the EntityName of primary entity + let entityNameCalc = null; + if (typeInformation.entityNameExp !== undefined && typeInformation.entityNameExp !== '') { try { - logger.debug( - context, - 'Evaluating attribute: %j, for entity_name(exp):%j, with ctxt: %j', - currentAttr.name, - currentAttr.entity_name, - jexlctxt + logger.debug(context, 'sendUpdateValueNgsi2 entityNameExp %j', typeInformation.entityNameExp); + entityNameCalc = expressionPlugin.applyExpression( + typeInformation.entityNameExp, + jexlctxt, + typeInformation ); - attrEntityName = jexlParser.applyExpression(currentAttr.entity_name, jexlctxt, typeInformation); - if (!attrEntityName) { - attrEntityName = currentAttr.entity_name; - } } catch (e) { logger.debug( context, - 'Exception evaluating entityNameExp:%j, with jexlctxt: %j', - currentAttr.entity_name, + 'Error evaluating expression for entityName: %j with context: %j', + typeInformation.entityNameExp, jexlctxt ); - attrEntityName = currentAttr.entity_name; } } - //determine AttrEntityType for multientity - if ( - currentAttr.entity_type !== null && - currentAttr.entity_type !== undefined && - currentAttr.entity_type !== '' && - typeof currentAttr.entity_type === 'string' - ) { - attrEntityType = currentAttr.entity_type; - } + entityName = entityNameCalc ? entityNameCalc : entityName; + //enrich JEXL context + jexlctxt['entity_name'] = entityName; - //PRE POPULATE CONTEXT - jexlctxt[currentAttr.name] = plainMeasures[currentAttr.object_id]; - - //determine Value - if (currentAttr.value !== undefined) { - //static attributes already have a value - hitted = true; - valueExpression = currentAttr.value; - } else if (plainMeasures[currentAttr.object_id] !== undefined) { - //we have got a meaure for that Attr - //actives ¿lazis? - hitted = true; - valueExpression = plainMeasures[currentAttr.object_id]; + let preprocessedAttr = []; + //Add Raw Static, Lazy, Command and Actives attr attributes + if (typeInformation && typeInformation.staticAttributes) { + preprocessedAttr = preprocessedAttr.concat(typeInformation.staticAttributes); } - //remove measures that has been shadowed by an alias (some may be left and managed later) - //Maybe we must filter object_id if there is name == object_id - measures = measures.filter((item) => item.name !== currentAttr.object_id && item.name !== currentAttr.name); - - if ( - currentAttr.expression !== undefined && - currentAttr.expression !== '' && - typeof currentAttr.expression == 'string' - ) { - try { + if (typeInformation && typeInformation.lazy) { + preprocessedAttr = preprocessedAttr.concat(typeInformation.lazy); + } + if (typeInformation && typeInformation.active) { + preprocessedAttr = preprocessedAttr.concat(typeInformation.active); + } + + //Proccess every proto Attribute to populate entities data steuture + entities[entityName] = {}; + entities[entityName][typeInformation.type] = []; + + for (let currentAttr of preprocessedAttr) { + let hitted = false; //any measure, expressiom or value hit the attr (avoid propagate "silent attr" with null values ) + let attrEntityName = entityName; + let attrEntityType = typeInformation.type; + let valueExpression = null; + //manage active attr without object__id (name by default) + currentAttr.object_id = currentAttr.object_id ? currentAttr.object_id : currentAttr.name; + //Enrich the attr (skip, hit, value, meta-timeInstant) + currentAttr.skipValue = currentAttr.skipValue ? currentAttr.skipValue : null; + + //determine AttrEntityName for multientity + if ( + currentAttr.entity_name !== null && + currentAttr.entity_name !== undefined && + currentAttr.entity_name !== '' && + typeof currentAttr.entity_name == 'string' + ) { + try { + logger.debug( + context, + 'Evaluating attribute: %j, for entity_name(exp):%j, with ctxt: %j', + currentAttr.name, + currentAttr.entity_name, + jexlctxt + ); + attrEntityName = jexlParser.applyExpression(currentAttr.entity_name, jexlctxt, typeInformation); + if (!attrEntityName) { + attrEntityName = currentAttr.entity_name; + } + } catch (e) { + logger.debug( + context, + 'Exception evaluating entityNameExp:%j, with jexlctxt: %j', + currentAttr.entity_name, + jexlctxt + ); + attrEntityName = currentAttr.entity_name; + } + } + + //determine AttrEntityType for multientity + if ( + currentAttr.entity_type !== null && + currentAttr.entity_type !== undefined && + currentAttr.entity_type !== '' && + typeof currentAttr.entity_type === 'string' + ) { + attrEntityType = currentAttr.entity_type; + } + + //PRE POPULATE CONTEXT + jexlctxt[currentAttr.name] = plainMeasures[currentAttr.object_id]; + + //determine Value + if (currentAttr.value !== undefined) { + //static attributes already have a value hitted = true; - valueExpression = jexlParser.applyExpression(currentAttr.expression, jexlctxt, typeInformation); - //we fallback to null if anything unexpecte happend - if (valueExpression === null || valueExpression === undefined || Number.isNaN(valueExpression)) { + valueExpression = currentAttr.value; + } else if (plainMeasures[currentAttr.object_id] !== undefined) { + //we have got a meaure for that Attr + //actives ¿lazis? + hitted = true; + valueExpression = plainMeasures[currentAttr.object_id]; + } + //remove measures that has been shadowed by an alias (some may be left and managed later) + //Maybe we must filter object_id if there is name == object_id + measures = measures.filter((item) => item.name !== currentAttr.object_id && item.name !== currentAttr.name); + + if ( + currentAttr.expression !== undefined && + currentAttr.expression !== '' && + typeof currentAttr.expression == 'string' + ) { + try { + hitted = true; + valueExpression = jexlParser.applyExpression(currentAttr.expression, jexlctxt, typeInformation); + //we fallback to null if anything unexpecte happend + if (valueExpression === null || valueExpression === undefined || Number.isNaN(valueExpression)) { + valueExpression = null; + } + } catch (e) { valueExpression = null; } - } catch (e) { - valueExpression = null; + logger.debug( + context, + 'Evaluated attr: %j, with expression: %j, and ctxt: %j resulting: %j', + currentAttr.name, + currentAttr.expression, + jexlctxt, + valueExpression + ); } - logger.debug( - context, - 'Evaluated attr: %j, with expression: %j, and ctxt: %j resulting: %j', - currentAttr.name, - currentAttr.expression, - jexlctxt, - valueExpression - ); - } - currentAttr.hitted = hitted; - currentAttr.value = valueExpression; + currentAttr.hitted = hitted; + currentAttr.value = valueExpression; - //store de New Attributte in entity data structure - if (hitted === true) { - if (entities[attrEntityName] === undefined) { - entities[attrEntityName] = {}; - } - if (entities[attrEntityName][attrEntityType] === undefined) { - entities[attrEntityName][attrEntityType] = []; + //store de New Attributte in entity data structure + if (hitted === true) { + if (entities[attrEntityName] === undefined) { + entities[attrEntityName] = {}; + } + if (entities[attrEntityName][attrEntityType] === undefined) { + entities[attrEntityName][attrEntityType] = []; + } + //store de New Attributte + entities[attrEntityName][attrEntityType].push(currentAttr); } - //store de New Attributte - entities[attrEntityName][attrEntityType].push(currentAttr); - } - //RE-Populate de JEXLcontext (except for null or NaN we preffer undefined) - jexlctxt[currentAttr.name] = valueExpression; + //RE-Populate de JEXLcontext (except for null or NaN we preffer undefined) + jexlctxt[currentAttr.name] = valueExpression; - // Expand metadata value expression - if (currentAttr.metadata) { - for (var metaKey in currentAttr.metadata) { - if (currentAttr.metadata[metaKey].expression && metaKey !== constants.TIMESTAMP_ATTRIBUTE) { - let newAttrMeta = {}; - if (currentAttr.metadata[metaKey].type) { - newAttrMeta['type'] = currentAttr.metadata[metaKey].type; - } - let metaValueExpression; - try { - metaValueExpression = jexlParser.applyExpression( - currentAttr.metadata[metaKey].expression, - jexlctxt, - typeInformation - ); - //we fallback to null if anything unexpecte happend - if ( - metaValueExpression === null || - metaValueExpression === undefined || - Number.isNaN(metaValueExpression) - ) { + // Expand metadata value expression + if (currentAttr.metadata) { + for (var metaKey in currentAttr.metadata) { + if (currentAttr.metadata[metaKey].expression && metaKey !== constants.TIMESTAMP_ATTRIBUTE) { + let newAttrMeta = {}; + if (currentAttr.metadata[metaKey].type) { + newAttrMeta['type'] = currentAttr.metadata[metaKey].type; + } + let metaValueExpression; + try { + metaValueExpression = jexlParser.applyExpression( + currentAttr.metadata[metaKey].expression, + jexlctxt, + typeInformation + ); + //we fallback to null if anything unexpecte happend + if ( + metaValueExpression === null || + metaValueExpression === undefined || + Number.isNaN(metaValueExpression) + ) { + metaValueExpression = null; + } + } catch (e) { metaValueExpression = null; } - } catch (e) { - metaValueExpression = null; + newAttrMeta['value'] = metaValueExpression; + currentAttr.metadata[metaKey] = newAttrMeta; } - newAttrMeta['value'] = metaValueExpression; - currentAttr.metadata[metaKey] = newAttrMeta; } } } - } - //now we can compute explicit (Bool or Array) with the complete JexlContext - let explicit = false; - if (typeof typeInformation.explicitAttrs === 'string') { - try { - explicit = jexlParser.applyExpression(typeInformation.explicitAttrs, jexlctxt, typeInformation); - if (explicit instanceof Array && mustInsertTimeInstant) { - explicit.push(constants.TIMESTAMP_ATTRIBUTE); + //now we can compute explicit (Bool or Array) with the complete JexlContext + let explicit = false; + if (typeof typeInformation.explicitAttrs === 'string') { + try { + explicit = jexlParser.applyExpression(typeInformation.explicitAttrs, jexlctxt, typeInformation); + if (explicit instanceof Array && mustInsertTimeInstant) { + explicit.push(constants.TIMESTAMP_ATTRIBUTE); + } + logger.debug( + context, + 'Calculated explicitAttrs with expression: %j and ctxt: %j resulting: %j', + typeInformation.explicitAttrs, + jexlctxt, + explicit + ); + } catch (e) { + // nothing to do: exception is already logged at info level } - logger.debug( - context, - 'Calculated explicitAttrs with expression: %j and ctxt: %j resulting: %j', - typeInformation.explicitAttrs, - jexlctxt, - explicit - ); - } catch (e) { - // nothing to do: exception is already logged at info level + } else if (typeof typeInformation.explicitAttrs == 'boolean') { + explicit = typeInformation.explicitAttrs; } - } else if (typeof typeInformation.explicitAttrs == 'boolean') { - explicit = typeInformation.explicitAttrs; - } - - //more mesures may be added to the attribute list (unnhandled/left mesaures) l - if (explicit === false && Object.keys(measures).length > 0) { - entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures); - } - - //PRE-PROCESSING FINISHED - //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload - //Get ready to build and send NGSI payload (entities-->payload) - payload.actionType = 'append'; + //more mesures may be added to the attribute list (unnhandled/left mesaures) l + if (explicit === false && Object.keys(measures).length > 0) { + entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures); + } - payload.entities = []; - const currentIsoDate = new Date().toISOString(); - const currentMoment = moment(currentIsoDate); - for (let ename in entities) { - for (let etype in entities[ename]) { - let e = {}; - e.id = String(ename); - e.type = String(etype); - let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. - let timestampAttrs = null; - if (mustInsertTimeInstant) { - // get timestamp for current entity - - timestampAttrs = entities[ename][etype].filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE); - if (timestampAttrs && timestampAttrs.length > 0) { - timestamp.value = timestampAttrs[0]['value']; - } + //PRE-PROCESSING FINISHED + //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload + //Get ready to build and send NGSI payload (entities-->payload) + + for (let ename in entities) { + for (let etype in entities[ename]) { + let e = {}; + e.id = String(ename); + e.type = String(etype); + let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. + let timestampAttrs = null; + if (mustInsertTimeInstant) { + // get timestamp for current entity - if (timestamp.value) { - if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) { - callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation)); - return; + timestampAttrs = entities[ename][etype].filter( + (item) => item.name === constants.TIMESTAMP_ATTRIBUTE + ); + if (timestampAttrs && timestampAttrs.length > 0) { + timestamp.value = timestampAttrs[0]['value']; } - } else { - if (!typeInformation.timezone) { - timestamp.value = currentIsoDate; - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + + if (timestamp.value) { + if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) { + callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation)); + return; + } } else { - timestamp.value = currentMoment - .tz(typeInformation.timezone) - .format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + if (!typeInformation.timezone) { + timestamp.value = currentIsoDate; + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + } else { + timestamp.value = currentMoment + .tz(typeInformation.timezone) + .format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + } } } - } - //extract attributes - let isEmpty = true; - for (let attr of entities[ename][etype]) { - if ( - attr.name !== 'id' && - attr.name !== 'type' && - (attr.value !== attr.skipValue || attr.skipValue === undefined) && - (attr.hitted || attr.hitted === undefined) && //undefined is for pure measures - (typeof explicit === 'boolean' || //true and false already handled - (explicit instanceof Array && //check the array version - (explicit.includes(attr.name) || - explicit.some( - (item) => attr.object_id !== undefined && item.object_id === attr.object_id - )))) - ) { - isEmpty = false; - if (mustInsertTimeInstant) { - // Add TimeInstant to all attribute metadata of all entities - if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { - if (!attr.metadata) { - attr.metadata = {}; + //extract attributes + let isEmpty = true; + for (let attr of entities[ename][etype]) { + if ( + attr.name !== 'id' && + attr.name !== 'type' && + (attr.value !== attr.skipValue || attr.skipValue === undefined) && + (attr.hitted || attr.hitted === undefined) && //undefined is for pure measures + (typeof explicit === 'boolean' || //true and false already handled + (explicit instanceof Array && //check the array version + (explicit.includes(attr.name) || + explicit.some( + (item) => attr.object_id !== undefined && item.object_id === attr.object_id + )))) + ) { + isEmpty = false; + if (mustInsertTimeInstant) { + // Add TimeInstant to all attribute metadata of all entities + if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { + if (!attr.metadata) { + attr.metadata = {}; + } + attr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; } - attr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; } + e[attr.name] = { type: attr.type, value: attr.value, metadata: attr.metadata }; } - e[attr.name] = { type: attr.type, value: attr.value, metadata: attr.metadata }; } - } - if (!isEmpty) { - if (mustInsertTimeInstant) { - e[constants.TIMESTAMP_ATTRIBUTE] = timestamp; + if (!isEmpty) { + if (mustInsertTimeInstant) { + e[constants.TIMESTAMP_ATTRIBUTE] = timestamp; + } + payload.entities.push(e); } - payload.entities.push(e); } } - } + } // end for (let measures of originMeasures) let url = '/v2/op/update'; - let options = NGSIUtils.createRequestObject(url, typeInformation, token); + let options = NGSIUtils.createRequestObject(url, originTypeInformation, token); options.json = payload; // Prevent to update an entity with an empty payload: more than id and type @@ -620,13 +635,17 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call // Note that the options object is prepared for the second case (multi entity), so we "patch" it // only in the first case - //Multientity more than one name o more than one type at primary entity - let multientity = Object.keys(entities).length > 1 || Object.keys(entities[entityName]).length > 1; + //Multi: multientity (more than one name o more than one type at primary entity) + // of multimeasure (originMeasures is an array of more than one element) + let multi = + Object.keys(entities).length > 1 || + Object.keys(entities[entityName]).length > 1 || + originMeasures.length > 1; - if (!multientity) { + if (!multi) { // recreate options object to use single entity update url = '/v2/entities?options=upsert'; - options = NGSIUtils.createRequestObject(url, typeInformation, token); + options = NGSIUtils.createRequestObject(url, originTypeInformation, token); delete payload.actionType; let entityAttrs = payload.entities[0]; @@ -643,7 +662,20 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call transformedObject.type = entityAttrs.type; options.json = transformedObject; options.method = 'POST'; - } // else: keep current options object created for a batch update + } else if (payload.entities.every((entity) => 'TimeInstant' in entity)) { + // Try sort entities by TimeInstant + payload.entities.sort( + (a, b) => new Date(a.TimeInstant.value).getTime() - new Date(b.TimeInstant.value).getTime() + ); + options.json = payload; + } else { + // keep current options object created for a batch update + logger.debug( + context, + "some entities lack the 'TimeInstant' key. Sorting is not feasible: %j ", + payload.entities + ); + } //Send the NGSI request logger.debug(context, 'Updating device value in the Context Broker at: %j', options.url); @@ -651,7 +683,7 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call request( options, - generateNGSI2OperationHandler('update', entityName, typeInformation, token, options, callback) + generateNGSI2OperationHandler('update', entityName, originTypeInformation, token, options, callback) ); } else { logger.debug( diff --git a/test/functional/README.md b/test/functional/README.md index 47d477be0..6020a6952 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -39,7 +39,7 @@ test cases are automatically generated. Each test case is defined as an object w or if the `transport` element is not defined. See the "Advanced features" section for more information. - `shouldName`: The name of the `IT` test case. This will be used to generate the test case name in the mocha test suite. - - `type`: The type of the test case. This can be `single` or `multientity`. See the "Advanced features" section + - `type`: The type of the test case. This can be `single`, `multimeasure` or `multientity`. See the "Advanced features" section for more information. - `measure`: The JSON object that will be sent to the IoTA JSON measure API. This will be used to send the measure. It contains the following elements: @@ -199,41 +199,11 @@ as a batch operation (see the following example). #### Multimeasures -It is also supported to test cases in which is sent more than one measure. To do so, you need to define the test case -`expectation` as an array, with one object for each measurement. Then, the suite will recognize the array length and will -expect the same number of NGSI requests. I.E: +It is also supported to test cases in which is sent more than one measure. To do so, you need to set add to the test case +the parameter `should.type` to the value `'multimeasure'`. -```js -[ - { - id: 'TheLightType2:MQTT_2', - type: 'TheLightType2', - temperature: { - value: 10, - type: 'Number' - }, - status: { - value: false, - type: 'Boolean' - } - }, - { - id: 'TheLightType2:MQTT_2', - type: 'TheLightType2', - temperature: { - value: 20, - type: 'Number' - }, - status: { - value: true, - type: 'Boolean' - } - } -]; -``` - -You also should define the measure as multimeasure. This is done by defining the `measure` JSON element as an array of -objects. Each object will be a measure that will be sent to the Context Broker in a different request. I.E: +You must define the measure as multimeasure. This is done by defining the `measure` JSON element as an array of +objects. I.E: ```javascript measure: { @@ -246,16 +216,69 @@ measure: { json: [ { s: false, - t: 10 + t: 21 }, { s: true, - t: 20 + t: 22 + }, + { + s: false, + t: 23 + } + ] +} +``` + +And you should define the test case `expectation` as an object, following a Context Broker batch operation. I.E: + +```js +expectation: { + actionType: 'append', + entities: [ + { + id: 'TheLightType2:MQTT_2', + type: 'TheLightType2', + temperature: { + type: 'Number', + value: 21 + }, + status: { + type: 'Boolean', + value: false + } + }, + { + id: 'TheLightType2:MQTT_2', + type: 'TheLightType2', + temperature: { + type: 'Number', + value: 22 + }, + status: { + type: 'Boolean', + value: true + } + }, + { + id: 'TheLightType2:MQTT_2', + type: 'TheLightType2', + temperature: { + type: 'Number', + value: 23 + }, + status: { + type: 'Boolean', + value: false + } } ] } ``` +Then, a batch request would be sent to the Context Broker containing the different measures. More information about +how the IoT Agent send multimeasures to the Context Broker [here](/doc/api.md#multimeasure-support). + #### Transport The test suite supports using the internal node lib function `iotAgentLib.update`, `HTTP` or `MQTT` for measure sending. diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 76993086d..a4a80946c 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -446,7 +446,7 @@ const testCases = [ ] }, { - describeName: '0021 Simple group with active attributes with metadata', + describeName: '0021 - Simple group with active attributes with metadata', provision: { url: 'http://localhost:' + config.iota.server.port + '/iot/services', method: 'POST', @@ -513,6 +513,399 @@ const testCases = [ } ] }, + { + describeName: '0022 - Simple group with active attributes and multimeasures', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [ + { + object_id: 'a', + name: 'attr_a', + type: 'Number' + } + ], + static_attributes: [] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures to Context Broker preserving value types and name mappings', + type: 'multimeasure', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: [ + { + a: 0 + }, + { + a: 1 + }, + { + a: 2 + }, + { + a: 3 + }, + { + a: 4 + }, + { + a: 5 + }, + { + a: 6 + } + ] + }, + expectation: { + actionType: 'append', + entities: [ + { + attr_a: { + type: 'Number', + value: 0 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 1 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 2 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 3 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 4 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 5 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 6 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + } + ] + } + }, + { + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings and order', + type: 'multimeasure', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: [ + { + a: 0, + TimeInstant: '2024-04-10T10:00:00Z' + }, + { + a: 1, + TimeInstant: '2024-04-10T10:05:00Z' + }, + { + a: 2, + TimeInstant: '2024-04-10T10:10:00Z' + }, + { + a: 3, + TimeInstant: '2024-04-10T10:15:00Z' + }, + { + a: 4, + TimeInstant: '2024-04-10T10:20:00Z' + }, + { + a: 5, + TimeInstant: '2024-04-10T10:25:00Z' + }, + { + a: 6, + TimeInstant: '2024-04-10T10:30:00Z' + } + ] + }, + expectation: { + actionType: 'append', + entities: [ + { + attr_a: { + type: 'Number', + value: 0 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:00:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 1 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:05:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 2 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:10:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 3 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:15:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 4 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:20:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 5 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:25:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 6 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:30:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + } + ] + } + }, + { + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings and sorted by TimeInstant', + type: 'multimeasure', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: [ + { + a: 0, + TimeInstant: '2024-04-10T10:15:00Z' + }, + { + a: 1, + TimeInstant: '2024-04-10T10:05:00Z' + }, + { + a: 2, + TimeInstant: '2024-04-10T10:20:00Z' + }, + { + a: 3, + TimeInstant: '2024-04-10T10:00:00Z' + }, + { + a: 4, + TimeInstant: '2024-04-10T10:10:00Z' + }, + { + a: 5, + TimeInstant: '2024-04-10T10:30:00Z' + }, + { + a: 6, + TimeInstant: '2024-04-10T10:25:00Z' + } + ] + }, + expectation: { + actionType: 'append', + entities: [ + { + attr_a: { + type: 'Number', + value: 3 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:00:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 1 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:05:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 4 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:10:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 0 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:15:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 2 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:20:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 6 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:25:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 5 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:30:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + } + ] + } + } + ] + }, // 0100 - JEXL TESTS { describeName: '0100 - Simple group with active attribute + JEXL expression boolean (!)', @@ -2618,56 +3011,56 @@ const testCases = [ actionType: 'append', entities: [ { - id: globalEnv.entity_name, + id: 'TestType:TestDevice1', type: globalEnv.entity_type, - a: { + a1: { value: 23, type: 'Text', metadata: { TimeInstant: { - value: _.isDateString, + value: '2011-01-01T01:11:11.111Z', type: 'DateTime' } } }, TimeInstant: { - value: _.isDateString, + value: '2011-01-01T01:11:11.111Z', type: 'DateTime' } }, { - id: 'TestType:TestDevice1', + id: 'TestType:TestDevice2', type: globalEnv.entity_type, - a1: { + a2: { value: 23, type: 'Text', metadata: { TimeInstant: { - value: '2011-01-01T01:11:11.111Z', + value: '2022-02-02T02:22:22.222Z', type: 'DateTime' } } }, TimeInstant: { - value: '2011-01-01T01:11:11.111Z', + value: '2022-02-02T02:22:22.222Z', type: 'DateTime' } }, { - id: 'TestType:TestDevice2', + id: globalEnv.entity_name, type: globalEnv.entity_type, - a2: { + a: { value: 23, type: 'Text', metadata: { TimeInstant: { - value: '2022-02-02T02:22:22.222Z', + value: _.isDateString, type: 'DateTime' } } }, TimeInstant: { - value: '2022-02-02T02:22:22.222Z', + value: _.isDateString, type: 'DateTime' } } diff --git a/test/functional/testUtils.js b/test/functional/testUtils.js index f41e8433a..495e8d470 100644 --- a/test/functional/testUtils.js +++ b/test/functional/testUtils.js @@ -98,28 +98,61 @@ function sendMeasureIotaLib(measure, provision) { * @param {Object} json * @returns {Array} measures */ -function jsonToIotaMeasures(json) { - let measures = []; - for (let key in json) { - /* eslint-disable-next-line no-prototype-builtins */ - if (json.hasOwnProperty(key)) { - let measure = { - name: key, - value: json[key] - }; - // A bit of Magic. If the key is TimeInstant, we set the type to DateTime. - // When sending the data through iot - if (key === 'TimeInstant') { - measure.type = 'DateTime'; - } else { - // Although the type is not meaningfull and we could have picked any string for this, - // we have aligned with DEFAULT_ATTRIBUTE_TYPE constant in IOTA-JSON and IOTA-UL repositories - measure.type = 'Text'; +function jsonToIotaMeasures(originJson) { + // FIXME: maybe this could be refactored to use less code + if (originJson && originJson[0]) { + // multimeasure case + let finalMeasures = []; + + for (let json of originJson) { + let measures = []; + for (let key in json) { + /* eslint-disable-next-line no-prototype-builtins */ + if (json.hasOwnProperty(key)) { + let measure = { + name: key, + value: json[key] + }; + // A bit of Magic. If the key is TimeInstant, we set the type to DateTime. + // When sending the data through iot + if (key === 'TimeInstant') { + measure.type = 'DateTime'; + } else { + // Although the type is not meaningfull and we could have picked any string for this, + // we have aligned with DEFAULT_ATTRIBUTE_TYPE constant in IOTA-JSON and IOTA-UL repositories + measure.type = 'Text'; + } + measures.push(measure); + } + } + finalMeasures.push(measures); + } + return finalMeasures; + } else { + let json = originJson; + + let measures = []; + for (let key in json) { + /* eslint-disable-next-line no-prototype-builtins */ + if (json.hasOwnProperty(key)) { + let measure = { + name: key, + value: json[key] + }; + // A bit of Magic. If the key is TimeInstant, we set the type to DateTime. + // When sending the data through iot + if (key === 'TimeInstant') { + measure.type = 'DateTime'; + } else { + // Although the type is not meaningfull and we could have picked any string for this, + // we have aligned with DEFAULT_ATTRIBUTE_TYPE constant in IOTA-JSON and IOTA-UL repositories + measure.type = 'Text'; + } + measures.push(measure); } - measures.push(measure); } + return measures; } - return measures; } /** @@ -170,7 +203,7 @@ async function testCase(measure, expectation, provision, env, config, type, tran let receivedContext = []; let cbMockRoute = ''; // Set the correct route depending if the test is multientity or not - if (type === 'multientity') { + if (type === 'multientity' || type === 'multimeasure') { cbMockRoute = '/v2/op/update'; } else { cbMockRoute = '/v2/entities?options=upsert';