ADD evalPriority feature

CHANGES_NEXT_RELEASE

@@ -1,2 +1,3 @@
+- Fix: custom notification ngsi patching evaluation priority based in evalPriority builtin metadata (#4556)
 - Fix: wrong date values should not allowed in subscription's expires field (#4541)
 - Fix: do not raise DB alarm in case of wrong GeoJSON in client request

doc/manuals/orion-api.md

@@ -75,6 +75,7 @@
       - [Additional considerations](#additional-considerations)
     - [JEXL Support](#jexl-support)
       - [JEXL usage example](#jexl-usage-example)
+      - [Evaluation priority](#evaluation-priority)
       - [Available Transformations](#available-transformations)
         - [`uppercase`](#uppercase)
         - [`lowercase`](#lowercase)
@@ -746,6 +747,8 @@ type has an special semantic for Orion:
    * `DateTime`
    * `geo:json`
 
+* `evalPriority`: used by expression evaluation. Have a look to [this specific section](#evaluation-priority) for details.
+
 At the present moment `ignoreType` is supported only for geo-location types, this way allowing a
 mechanism to overcome the limit of only one geo-location per entity (more details
 in [Geospatial properties of entities](#geospatial-properties-of-entities) section). Support
@@ -2507,6 +2510,64 @@ will trigger a notification like this:
 ]
 ```
 
+### Evaluation priority
+
+Each time an expression is evaluated, it is added to the expression context, so it can be reused by other expressions. However, by default Orion does not guarantee a given evaluation other.
+
+Thus, if we have this:
+
+```
+"httpCustom": {
+  ...
+  "ngsi": {
+    "A": {
+      "value": "${16|sqrt}",
+      "type": "Calculated"
+    },
+    "B": {
+      "value": "${A/100}",
+      "type": "Calculated"
+    }
+  },
+  "attrs": [ "B" ]
+}
+```
+
+in the resulting notification `B` could be set to the desired `0.04` (if `A` is evaluated before `B`) or to the underised `null` (if `B` is evaluated before `A`), randomly.
+
+In order to overcome this problem, the `evalPriority` metadata can be used to define the evaluation order. It works this way:
+
+* `evalPriority` metadata is a number from 1 (first evaluation) to 100000 (last evaluation)
+* Expressions are evaluated in incresing order of priority
+* In case of ties, Orion does not guarantee a particular evaluation order. Thus, expressions in the same priority level must be considered independent or bad things would happen
+* If no `evalPriority` is set, default 100000 is used
+* `evalPriority` only has a meaning in `notification.httpCustom.ngsi` in subscriptions. As metadata in regular entities (e.g. an entity created with `POST /v2/entities`) Orion doesn't implements any semantic for it
+
+Using `evalPriority` the above example could be reformuled this way:
+
+```
+"httpCustom": {
+  ...
+  "ngsi": {
+    "A": {
+      "value": "${16|sqrt}",
+      "type": "Calculated",
+      "metadata": {
+        "evalPriority": {
+          "value": 1,
+          "type": "Number"
+        }
+      }
+    },
+    "B": {
+      "value": "${A/100}",
+      "type": "Calculated"
+    }
+  },
+  "attrs": [ "B" ]
+}
+```
+
 ### Available Transformations
 
 #### `uppercase`

src/lib/common/JsonHelper.cpp

@@ -138,7 +138,7 @@ std::string objectToJson(std::map<std::string, std::string>& list)
 *
 * JsonObjectHelper -
 */
-JsonObjectHelper::JsonObjectHelper(): empty(true), closed(false)
+JsonObjectHelper::JsonObjectHelper(): empty(true)
 {
   ss += '{';
 }
@@ -280,15 +280,17 @@ void JsonObjectHelper::addNull(const std::string& key)
 *
 * JsonObjectHelper::str -
 */
-std::string JsonObjectHelper::str()
+std::string JsonObjectHelper::str(bool closed)
 {
-  // This check allows to call str() several times (needed when this is used in ExprContext)
-  if (!closed)
+  // closed == false used in ExprContext logic
+  if (closed)
   {
-    ss += '}';
-    closed = true;
+    return ss + '}';
+  }
+  else
+  {
+    return ss;
   }
-  return ss;
 }
 
 
@@ -297,7 +299,7 @@ std::string JsonObjectHelper::str()
 *
 * JsonVectorHelper -
 */
-JsonVectorHelper::JsonVectorHelper(): empty(true), closed(false)
+JsonVectorHelper::JsonVectorHelper(): empty(true)
 {
   ss += '[';
 }
@@ -422,17 +424,13 @@ void JsonVectorHelper::addNull(void)
 
 
 
+
 /* ****************************************************************************
 *
 * JsonVectorHelper::str -
 */
-std::string JsonVectorHelper::str()
+std::string JsonVectorHelper::str(void)
 {
-  // This check allows to call str() several times (needed when this is used in ExprContext)
-  if (!closed)
-  {
-    ss += ']';
-    closed = true;
-  }
+  ss += ']';
   return ss;
 }

src/lib/common/JsonHelper.h

@@ -45,12 +45,11 @@ class JsonObjectHelper
   void        addBool(const std::string& key, bool b);
   void        addNull(const std::string& key);
 
-  std::string str();
+  std::string str(bool closed = true);
 
 private:
  std::string  ss;
  bool         empty;
- bool         closed;
 };
 
 
@@ -68,12 +67,11 @@ class JsonVectorHelper
   void        addNull(void);
 
 
-  std::string str();
+  std::string str(void);
 
 private:
  std::string  ss;
  bool         empty;
- bool         closed;
 };
 
 

src/lib/common/errorMessages.h

@@ -87,7 +87,11 @@
 #define ERROR_DESC_BAD_REQUEST_FORMAT_INVALID                 "invalid render format for notifications"
 #define ERROR_DESC_BAD_REQUEST_SERVICE_NOT_FOUND              "Service not found. Check your URL as probably it is wrong."
 #define ERROR_DESC_BAD_REQUEST_WRONG_GEOJSON                  "Wrong GeoJson"
-#define ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF "metadata are not allowed in ngsi field in custom notifications"
+#define ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF "only evalPriority metadata is allowed in ngsi field in custom notifications"
+
+#define ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MUST_BE_A_NUMBER  "evalPriority metadata must be a number"
+#define ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MIN_ERROR         "evalPriority metadata minimum priority is " STR(MIN_PRIORITY)
+#define ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MAX_ERROR         "evalPriority metadata maximum priority is "  STR(MAX_PRIORITY)
 
 #define ERROR_NOT_FOUND                               "NotFound"
 #define ERROR_DESC_NOT_FOUND_ENTITY                   "The requested entity has not been found. Check type and id"

src/lib/common/limits.h

@@ -229,4 +229,14 @@
 
 
 
+/* ****************************************************************************
+*
+* MIX_PRIORITY and MAX_PRIORITY
+*
+*/
+#define MIN_PRIORITY  1
+#define MAX_PRIORITY  100000
+
+
+
 #endif  // SRC_LIB_COMMON_LIMITS_H_

src/lib/expressions/ExprContext.cpp

@@ -47,7 +47,7 @@ ExprContextObject::ExprContextObject(bool _basic)
 */
 std::string ExprContextObject::getJexlContext(void)
 {
-  return jh.str();
+  return jh.str(false) + '}';
 }
 
 
@@ -83,7 +83,14 @@ void ExprContextObject::add(const std::string &key, const std::string &_value, b
   else
   {
     LM_T(LmtExpr, ("adding to JEXL expression context object (string): %s=%s", key.c_str(), _value.c_str()));
-    jh.addString(key, _value);
+    if (raw)
+    {
+      jh.addRaw(key, _value);
+    }
+    else
+    {
+      jh.addString(key, _value);
+    }
   }
 }
 

src/lib/jsonParseV2/parseSubscription.cpp

@@ -554,10 +554,28 @@ static std::string parseCustomPayload
           return badInput(ciP, r);
         }
 
-        // metadadata are now allowed in this case
-        if (caP->metadataVector.size() > 0)
+        // only evalPriority metadadata is allowed in this case
+        for (unsigned ix = 0; ix < caP->metadataVector.size(); ix++)
         {
-          return badInput(ciP, ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF);
+          if (caP->metadataVector[ix]->name != NGSI_MD_EVAL_PRIORITY)
+          {
+            return badInput(ciP, ERROR_DESC_BAD_REQUEST_METADATA_NOT_ALLOWED_CUSTOM_NOTIF);
+          }
+          else
+          {
+            if (caP->metadataVector[ix]->valueType != orion::ValueTypeNumber)
+            {
+              return badInput(ciP, ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MUST_BE_A_NUMBER);
+            }
+            if (caP->metadataVector[ix]->numberValue < MIN_PRIORITY)
+            {
+              return badInput(ciP, ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MIN_ERROR);
+            }
+            if (caP->metadataVector[ix]->numberValue > MAX_PRIORITY)
+            {
+              return badInput(ciP, ERROR_DESC_BAD_REQUEST_EVALPRIORITY_MAX_ERROR);
+            }
+          }
         }
       }
     }

src/lib/ngsi/ContextAttribute.cpp

@@ -763,6 +763,26 @@ bool ContextAttribute::getLocation(orion::BSONObj* attrsP) const
 
 
 
+/* ****************************************************************************
+*
+* getEvalPriority -
+*/
+double ContextAttribute::getEvalPriority(void)
+{
+  for (unsigned int ix = 0; ix < metadataVector.size(); ix++)
+  {
+    if (metadataVector[ix]->name == NGSI_MD_EVAL_PRIORITY)
+    {
+      return metadataVector[ix]->numberValue;
+    }
+  }
+
+  // if the attribute doesn't have evalPriority metadata, then max priority is assumed
+  return MAX_PRIORITY;
+}
+
+
+
 /* ****************************************************************************
 *
 * toJsonV1AsObject -
@@ -1124,9 +1144,9 @@ std::string ContextAttribute::toJson(const std::vector<std::string>&  metadataFi
   filterAndOrderMetadata(metadataFilter, &orderedMetadata);
 
   //
-  // metadata (note that ngsi field in custom notifications doesn't include metadata)
+  // metadata (note that ngsi field in custom notifications avoids empty metadata array, i.e. "metadata": {})
   //
-  if (!renderNgsiField)
+  if ((!renderNgsiField) || (metadataVector.size() > 0))
   {
     jh.addRaw("metadata", metadataVector.toJson(orderedMetadata));
   }
@@ -1140,9 +1160,10 @@ std::string ContextAttribute::toJson(const std::vector<std::string>&  metadataFi
 * toJsonValue -
 *
 * To be used by options=values and options=unique renderings
+* Also used by the ngsi expression logic
 *
 */
-std::string ContextAttribute::toJsonValue(void)
+std::string ContextAttribute::toJsonValue(ExprContextObject* exprContextObjectP)
 {
   if (compoundValueP != NULL)
   {
@@ -1164,10 +1185,7 @@ std::string ContextAttribute::toJsonValue(void)
   }
   else if (valueType == orion::ValueTypeString)
   {
-    std::string out = "\"";
-    out += toJsonString(stringValue);
-    out += '"';
-    return out;
+    return smartStringValue(stringValue, exprContextObjectP, "null");
   }
   else if (valueType == orion::ValueTypeBoolean)
   {

src/lib/ngsi/ContextAttribute.h

@@ -95,6 +95,8 @@ typedef struct ContextAttribute
   /* Check if attribute means a location  */
   bool  getLocation(orion::BSONObj* attrsP) const;
 
+  double getEvalPriority(void);
+
   std::string  toJsonV1(bool                             asJsonObject,
                         RequestType                      request,
                         const std::vector<std::string>&  metadataFilter,
@@ -110,7 +112,7 @@ typedef struct ContextAttribute
 
   std::string  toJson(const std::vector<std::string>&  metadataFilter, bool renderNgsiField = false, ExprContextObject* exprContextObjectP = NULL);
 
-  std::string  toJsonValue(void);
+  std::string  toJsonValue(ExprContextObject* exprContextObjectP = NULL);
 
   std::string  toJsonAsValue(ApiVersion       apiVersion,
                              bool             acceptedTextPlain,

src/lib/ngsi/Metadata.h

@@ -46,6 +46,7 @@
 *
 * Metadata interpreted by Orion Context Broker, i.e. not custom metadata
 */
+#define NGSI_MD_EVAL_PRIORITY      "evalPriority"
 #define NGSI_MD_IGNORE_TYPE        "ignoreType"
 #define NGSI_MD_PREVIOUSVALUE      "previousValue"   // Special metadata
 #define NGSI_MD_ACTIONTYPE         "actionType"      // Special metadata