FIWARE · kzangeli · Jan 19, 2024 · Jan 19, 2024
diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE
@@ -2,5 +2,5 @@ Fixed Issue:
   * #1535  No matching with 'q' when matching subscriptions for deletion of an entity
 
 New Features:
-  * Support for VocabularyProperty (highlu untested and 'q' expansions for vocab-properties is not yet implemented)
-
+  * Support for VocabularyProperty (highly untested and 'q' expansions for vocab-properties is not yet implemented)
+  * Support for the new URL parameter "format" for output formats (normalized, concise, simplified)
diff --git a/src/lib/apiTypesV2/Entity.cpp b/src/lib/apiTypesV2/Entity.cpp
@@ -83,7 +83,7 @@ std::string Entity::render(bool comma)
 
   OrionldRenderFormat  renderFormat = RF_NORMALIZED;
 
-  if      (orionldState.uriParamOptions.keyValues    == true)  { renderFormat = RF_KEYVALUES;     }
+  if      (orionldState.out.format                   == RF_KEYVALUES)  { renderFormat = RF_KEYVALUES;     }
   else if (orionldState.uriParamOptions.values       == true)  { renderFormat = RF_VALUES;        }
   else if (orionldState.uriParamOptions.uniqueValues == true)  { renderFormat = RF_UNIQUE_VALUES; }
 

diff --git a/src/lib/jsonParseV2/parseContextAttribute.cpp b/src/lib/jsonParseV2/parseContextAttribute.cpp
@@ -198,7 +198,7 @@ std::string parseContextAttribute
 
   caP->name = name;
 
-  if (orionldState.uriParamOptions.keyValues)
+  if (orionldState.out.format == RF_SIMPLIFIED)
   {
     if (type == "String")
     {

diff --git a/src/lib/orionld/common/langStringExtract.cpp b/src/lib/orionld/common/langStringExtract.cpp
@@ -109,7 +109,7 @@ void langValueFix(KjNode* attrP, KjNode* valueP, KjNode* typeP, const char* lang
 
   if (langValueNodeP == NULL)
   {
-    if (orionldState.uriParamOptions.keyValues == false)
+    if (orionldState.out.format != RF_SIMPLIFIED)
     {
       valueP->type      = KjString;
       valueP->value.s   = (char*) "empty languageMap ...";
@@ -122,7 +122,7 @@ void langValueFix(KjNode* attrP, KjNode* valueP, KjNode* typeP, const char* lang
   }
   else if (langValueNodeP->type == KjString)
   {
-    if (orionldState.uriParamOptions.keyValues == false)
+    if (orionldState.out.format != RF_SIMPLIFIED)
     {
       valueP->type      = KjString;
       valueP->value.s   = langValueNodeP->value.s;
@@ -135,7 +135,7 @@ void langValueFix(KjNode* attrP, KjNode* valueP, KjNode* typeP, const char* lang
   }
   else  // It's an array
   {
-    if (orionldState.uriParamOptions.keyValues == false)
+    if (orionldState.out.format != RF_SIMPLIFIED)
     {
       valueP->type                       = KjArray;
       valueP->value.firstChildP          = langValueNodeP->value.firstChildP;
@@ -152,7 +152,7 @@ void langValueFix(KjNode* attrP, KjNode* valueP, KjNode* typeP, const char* lang
     langValueNodeP->lastChild         = NULL;
   }
 
-  if (orionldState.uriParamOptions.keyValues == false)
+  if (orionldState.out.format != RF_SIMPLIFIED)
   {
     if (langValueNodeP != NULL)
     {

diff --git a/src/lib/orionld/common/orionldState.h b/src/lib/orionld/common/orionldState.h
@@ -166,6 +166,7 @@ typedef struct OrionldUriParams
   bool      local;
   bool      onlyIds;
   bool      entityMap;
+  char*     format;
 
   double    observedAtAsDouble;
   uint64_t  mask;

diff --git a/src/lib/orionld/dbModel/dbModelFromApiAttribute.cpp b/src/lib/orionld/dbModel/dbModelFromApiAttribute.cpp
@@ -248,7 +248,7 @@ bool dbModelFromApiAttribute(KjNode* attrP, KjNode* dbAttrsP, KjNode* attrAddedV
     // Very Special case: If key-values is set, and an uri param 'observedAt' is present, and we're doing a patchEntity2, then:
     // modify the observedAt sub-attr accordingly
     //
-    if ((orionldState.uriParamOptions.keyValues == true) && (orionldState.uriParams.observedAt != NULL) && (dbMdP != NULL))
+    if ((orionldState.out.format == RF_SIMPLIFIED) && (orionldState.uriParams.observedAt != NULL) && (dbMdP != NULL))
     {
       KjNode* observedAtP = kjLookup(dbMdP, "observedAt");
       if (observedAtP != NULL)

diff --git a/src/lib/orionld/legacyDriver/legacyGetEntities.cpp b/src/lib/orionld/legacyDriver/legacyGetEntities.cpp
@@ -145,8 +145,8 @@ bool legacyGetEntities(void)
   char*                 coordinates    = orionldState.uriParams.coordinates;
   bool                  local          = orionldState.uriParams.local;
   char*                 lang           = orionldState.uriParams.lang;
-  bool                  keyValues      = orionldState.uriParamOptions.keyValues;
-  bool                  concise        = orionldState.uriParamOptions.concise;
+  bool                  keyValues      = orionldState.out.format == RF_SIMPLIFIED;
+  bool                  concise        = orionldState.out.format == RF_CONCISE;
   char*                 idString       = (id   != NULL)? id      : idPattern;
   const char*           isIdPattern    = (id   != NULL)? "false" : "true";
   bool                  isTypePattern  = (type != NULL)? false   : true;
@@ -483,7 +483,7 @@ bool legacyGetEntities(void)
   if (countP != NULL)
     orionldHeaderAdd(&orionldState.out.headers, HttpResultsCount, NULL, *countP);
 
-  if (orionldState.uriParamOptions.concise == true)
+  if (orionldState.out.format == RF_CONCISE)
     kjEntityNormalizedToConcise(orionldState.responseTree, NULL);  // lang already taken care of by apiEntityLanguageProps
 
   mongoRequest.release();

diff --git a/src/lib/orionld/legacyDriver/legacyGetEntity.cpp b/src/lib/orionld/legacyDriver/legacyGetEntity.cpp
@@ -223,14 +223,14 @@ static KjNode* orionldForwardGetEntityPart(KjNode* registrationP, char* entityId
 
   if (*newUriParamAttrsString != 0)
   {
-    if (orionldState.uriParamOptions.keyValues)
+    if (orionldState.out.format == RF_SIMPLIFIED)
       snprintf(urlPath, size, "%s/ngsi-ld/v1/entities/%s?options=keyValues&attrs=%s", uriDirP, entityId, newUriParamAttrsString);
     else
       snprintf(urlPath, size, "%s/ngsi-ld/v1/entities/%s?attrs=%s", uriDirP, entityId, newUriParamAttrsString);
   }
   else
   {
-    if (orionldState.uriParamOptions.keyValues)
+    if (orionldState.out.format == RF_SIMPLIFIED)
       snprintf(urlPath, size, "%s/ngsi-ld/v1/entities/%s?options=keyValues", uriDirP, entityId);
     else
       snprintf(urlPath, size, "%s/ngsi-ld/v1/entities/%s", uriDirP, entityId);
@@ -429,8 +429,8 @@ bool legacyGetEntity(void)
                                                    eqAttrV,
                                                    attrsMandatory,
                                                    orionldState.uriParamOptions.sysAttrs,
-                                                   orionldState.uriParamOptions.keyValues,
-                                                   orionldState.uriParamOptions.concise,
+                                                   orionldState.out.format == RF_SIMPLIFIED,
+                                                   orionldState.out.format == RF_CONCISE,
                                                    orionldState.uriParams.datasetId,
                                                    geometryProperty,
                                                    &orionldState.geoPropertyNode,
@@ -483,7 +483,7 @@ bool legacyGetEntity(void)
                                                          orionldState.uriParams.geometryProperty,
                                                          orionldState.geoPropertyMissing,
                                                          orionldState.linkHeaderAdded,
-                                                         orionldState.uriParamOptions.concise,
+                                                         orionldState.out.format == RF_CONCISE,
                                                          orionldState.contextP->url);
 
   return true;

diff --git a/src/lib/orionld/mhd/mhdConnectionInit.cpp b/src/lib/orionld/mhd/mhdConnectionInit.cpp
@@ -163,7 +163,7 @@ static void ipAddressAndPort(void)
 //
 // optionsParse -
 //
-static void optionsParse(const char* options)
+void optionsParse(const char* options)
 {
   char* optionStart = (char*) options;
   char* cP          = (char*) options;
@@ -182,10 +182,10 @@ static void optionsParse(const char* options)
       if      (strcmp(optionStart, "update")        == 0)  orionldState.uriParamOptions.update        = true;
       else if (strcmp(optionStart, "replace")       == 0)  orionldState.uriParamOptions.replace       = true;
       else if (strcmp(optionStart, "noOverwrite")   == 0)  orionldState.uriParamOptions.noOverwrite   = true;
-      else if (strcmp(optionStart, "keyValues")     == 0)  orionldState.uriParamOptions.keyValues     = true;
-      else if (strcmp(optionStart, "simplified")    == 0)  orionldState.uriParamOptions.keyValues     = true;
-      else if (strcmp(optionStart, "concise")       == 0)  orionldState.uriParamOptions.concise       = true;
-      else if (strcmp(optionStart, "normalized")    == 0)  orionldState.uriParamOptions.normalized    = true;
+      else if (strcmp(optionStart, "keyValues")     == 0)  { if (orionldState.uriParams.format == NULL) orionldState.uriParamOptions.keyValues     = true; }
+      else if (strcmp(optionStart, "simplified")    == 0)  { if (orionldState.uriParams.format == NULL) orionldState.uriParamOptions.keyValues     = true; }
+      else if (strcmp(optionStart, "concise")       == 0)  { if (orionldState.uriParams.format == NULL) orionldState.uriParamOptions.concise       = true; }
+      else if (strcmp(optionStart, "normalized")    == 0)  { if (orionldState.uriParams.format == NULL) orionldState.uriParamOptions.normalized    = true; }
       else if (strcmp(optionStart, "sysAttrs")      == 0)  orionldState.uriParamOptions.sysAttrs      = true;
       else if (strcmp(optionStart, "fromDb")        == 0)  orionldState.uriParamOptions.fromDb        = true;
       else if (strcmp(optionStart, "append")        == 0)  orionldState.uriParamOptions.append        = true;  // NGSIv2 compatibility
@@ -676,9 +676,24 @@ MHD_Result orionldUriArgumentGet(void* cbDataP, MHD_ValueKind kind, const char*
   else if (strcmp(key, "options") == 0)
   {
     orionldState.uriParams.options = (char*) value;
-    optionsParse(value);
     orionldState.uriParams.mask |= ORIONLD_URIPARAM_OPTIONS;
   }
+  else if (strcmp(key, "format") == 0)
+  {
+    orionldState.uriParams.format = (char*) value;
+
+    if      (strcmp(value, "normalized") == 0)  orionldState.out.format = RF_NORMALIZED;
+    else if (strcmp(value, "concise")    == 0)  orionldState.out.format = RF_CONCISE;
+    else if (strcmp(value, "simplified") == 0)  orionldState.out.format = RF_SIMPLIFIED;
+    else if (strcmp(value, "keyValues")  == 0)  orionldState.out.format = RF_SIMPLIFIED;
+    else
+    {
+      orionldError(OrionldBadRequestData, "Bad value for URI parameter /format/", value, 400);
+      return MHD_YES;
+    }
+
+    orionldState.uriParams.mask |= ORIONLD_URIPARAM_FORMAT;
+  }
   else if (strcmp(key, "geometry") == 0)
   {
     orionldState.uriParams.geometry = (char*) value;
@@ -1152,7 +1167,14 @@ MHD_Result mhdConnectionInit
   // 5. GET URI params
   //
   MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, orionldUriArgumentGet, NULL);
-  LM_T(LmtCount, ("orionldState.uriParams.count: %s", K_FT(orionldState.uriParams.count)));
+
+  //
+  // As "format" has precedence over options=concise/simplified/etc, the call to optionsParse must wait until after
+  // all calls to orionldUriArgumentGet are done.
+  // Because, optionsParse n eeds to know whether "format" has been used.
+  //
+  if (orionldState.uriParams.options != NULL)
+    optionsParse(orionldState.uriParams.options);
 
   //
   // Format of response payload

diff --git a/src/lib/orionld/payloadCheck/pCheckAttribute.cpp b/src/lib/orionld/payloadCheck/pCheckAttribute.cpp
@@ -221,7 +221,7 @@ inline bool pCheckAttributeString
   //
   // If all those are fulfilled, then the value (object) of the Relationship will be modified
   //
-  if ((orionldState.uriParamOptions.keyValues == true)                &&
+  if ((orionldState.out.format == RF_SIMPLIFIED)                      &&
       (orionldState.serviceP->serviceRoutine  == orionldPatchEntity2) &&
       (attrTypeFromDb                         == Relationship))
   {
@@ -985,7 +985,7 @@ static bool pCheckAttributeObject
     if ((attrTypeFromDb != NoAttributeType) && (attributeType != attrTypeFromDb))
     {
       // This might be OK, if keyValues is ON => attributeType is a guess
-      if (orionldState.uriParamOptions.keyValues == false)
+      if (orionldState.out.format != RF_SIMPLIFIED)
       {
         const char* title = attrTypeChangeTitle(attrTypeFromDb, attributeType);
         orionldError(OrionldBadRequestData, title, attrP->name, 400);
@@ -1046,7 +1046,7 @@ static bool pCheckAttributeObject
     if ((attrTypeFromDb != NoAttributeType) && (attributeType != NoAttributeType) && (attributeType != attrTypeFromDb))
     {
       // This might be OK, if keyValues is ON => attributeType is a guess
-      if (orionldState.uriParamOptions.keyValues == false)
+      if (orionldState.out.format != RF_SIMPLIFIED)
       {
         const char* title = attrTypeChangeTitle(attrTypeFromDb, attributeType);
         orionldError(OrionldBadRequestData, title, attrP->name, 400);

diff --git a/src/lib/orionld/service/orionldServiceInit.cpp b/src/lib/orionld/service/orionldServiceInit.cpp
@@ -216,6 +216,7 @@ static void restServicePrepare(OrionLdRestService* serviceP, OrionLdRestServiceS
   else if (serviceP->serviceRoutine == orionldGetEntities)
   {
     serviceP->uriParams |= ORIONLD_URIPARAM_OPTIONS;
+    serviceP->uriParams |= ORIONLD_URIPARAM_FORMAT;
     serviceP->uriParams |= ORIONLD_URIPARAM_LIMIT;
     serviceP->uriParams |= ORIONLD_URIPARAM_OFFSET;
     serviceP->uriParams |= ORIONLD_URIPARAM_COUNT;
@@ -238,6 +239,7 @@ static void restServicePrepare(OrionLdRestService* serviceP, OrionLdRestServiceS
   else if (serviceP->serviceRoutine == orionldGetEntity)
   {
     serviceP->uriParams |= ORIONLD_URIPARAM_OPTIONS;
+    serviceP->uriParams |= ORIONLD_URIPARAM_FORMAT;
     serviceP->uriParams |= ORIONLD_URIPARAM_ATTRS;
     serviceP->uriParams |= ORIONLD_URIPARAM_GEOMETRYPROPERTY;
     serviceP->uriParams |= ORIONLD_URIPARAM_LANG;

diff --git a/src/lib/orionld/serviceRoutines/orionldGetEntitiesLocal.cpp b/src/lib/orionld/serviceRoutines/orionldGetEntitiesLocal.cpp
@@ -203,10 +203,7 @@ bool orionldGetEntitiesLocal
   else
   {
     KjNode*              apiEntityArray  = kjArray(orionldState.kjsonP, NULL);
-    OrionldRenderFormat  rf              = RF_NORMALIZED;
-
-    if      (orionldState.uriParamOptions.concise   == true) rf = RF_CONCISE;
-    else if (orionldState.uriParamOptions.keyValues == true) rf = RF_SIMPLIFIED;
+    OrionldRenderFormat  rf              = orionldState.out.format;
 
     if (orionldState.out.contentType == MT_GEOJSON)
     {

diff --git a/src/lib/orionld/serviceRoutines/orionldGetEntity.cpp b/src/lib/orionld/serviceRoutines/orionldGetEntity.cpp
@@ -370,7 +370,7 @@ bool orionldGetEntity(void)
                                           orionldState.uriParams.geometryProperty,
                                           orionldState.geoPropertyMissing,
                                           orionldState.linkHeaderAdded,
-                                          orionldState.uriParamOptions.concise,
+                                          orionldState.out.format == RF_CONCISE,
                                           orionldState.contextP->url);
 
     //

diff --git a/src/lib/orionld/serviceRoutines/orionldPatchEntity2.cpp b/src/lib/orionld/serviceRoutines/orionldPatchEntity2.cpp
@@ -575,7 +575,7 @@ bool orionldPatchEntity2(void)
   // If options=simplified/keyValues is present, the behaviour is very different
   // BUT, if I add the "value" to each attribute in the payload body, I have "Normalized" it
   //
-  if (orionldState.uriParamOptions.keyValues == true)
+  if (orionldState.out.format == RF_SIMPLIFIED)
   {
     if (apiEntitySimplifiedToNormalized(orionldState.requestTree, dbAttrsP) == false)
       return false;

diff --git a/src/lib/orionld/serviceRoutines/orionldPostQuery.cpp b/src/lib/orionld/serviceRoutines/orionldPostQuery.cpp
@@ -95,10 +95,7 @@ bool orionldPostQuery(void)
     return false;
 
   KjNode*              apiEntityArray  = kjArray(orionldState.kjsonP, NULL);
-  OrionldRenderFormat  rf              = RF_NORMALIZED;
-
-  if      (orionldState.uriParamOptions.concise   == true) rf = RF_CONCISE;
-  else if (orionldState.uriParamOptions.keyValues == true) rf = RF_SIMPLIFIED;
+  OrionldRenderFormat  rf              = orionldState.out.format;
 
   if (orionldState.out.contentType == MT_GEOJSON)
   {

diff --git a/src/lib/orionld/subCache/subCacheCreate.cpp b/src/lib/orionld/subCache/subCacheCreate.cpp
@@ -0,0 +1,109 @@
+/*
+*
+* Copyright 2024 FIWARE Foundation e.V.
+*
+* This file is part of Orion-LD Context Broker.
+*
+* Orion-LD Context Broker is free software: you can redistribute it and/or
+* modify it under the terms of the GNU Affero General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* Orion-LD Context Broker is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+* General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/.
+*
+* For those usages not covered by this license please contact with
+* orionld at fiware dot org
+*
+* Author: Ken Zangelin
+*/
+extern "C"
+{
+#include "kjson/KjNode.h"                                        // KjNode
+#include "kjson/kjLookup.h"                                      // kjLookup
+}
+
+#include "logMsg/logMsg.h"                                       // LM_*
+
+#include "orionld/types/OrionldTenant.h"                         // OrionldTenant
+#include "orionld/types/SubCache.h"                              // SubCache
+#include "orionld/common/orionldState.h"                         // orionldState
+#include "orionld/context/orionldContextFromUrl.h"               // orionldContextFromUrl
+#include "orionld/mongoc/mongocSubscriptionsIter.h"              // mongocSubscriptionsIter
+#include "orionld/dbModel/dbModelToApiSubscription.h"            // dbModelToApiSubscription
+#include "orionld/subCache/subCacheItemAdd.h"                    // subCacheItemAdd
+#include "orionld/subCache/subCacheCreate.h"                     // Own interface
+
+
+
+extern void apiModelToCacheSubscription(KjNode* apiSubscriptionP);
+// -----------------------------------------------------------------------------
+//
+// subIterFunc -
+//
+int subIterFunc(SubCache* scP, KjNode* dbSubP)
+{
+  // Convert DB Sub to API Sub
+  if (dbModelToApiSubscription(dbSubP, true, true) == false)
+    LM_RE(-1, ("dbModelToApiSubscription failed"));
+
+  // The DB Subscription 'dbSubP' is now in API Subscription format (after calling dbModelToApiSubscription)
+  KjNode* apiSubP = dbSubP;
+
+  // If a jsonldContext is given for the subscription, make sure it's valid
+  OrionldContext* jsonldContextP = NULL;
+  KjNode*         jsonldContextNodeP = kjLookup(apiSubscriptionP, "jsonldContext");
+  if (jsonldContextNodeP != NULL)
+  {
+    jsonldContextP = orionldContextFromUrl(jsonldContextNodeP->value.s, NULL);
+
+    if (jsonldContextP == NULL)
+    {
+      LM_W(("Unable to resolve a Subscription @context for a sub-cache item"));
+      return 0;
+    }
+  }
+
+  // Subscription Id
+  KjNode* subIdNodeP = kjLookup(apiSubP, "id");
+  char*   subId      = (subIdNodeP != NULL)? subIdNodeP->value.s : (char*) "no:sub:id";
+
+  // Convert API Sub to Cache Sub
+  apiModelToCacheSubscription(apiSubP);
+
+  // Insert cacheSubP in tenantP->subCache
+  subCacheItemAdd(scP, subId, apiSubP, true, jsonldContextP);
+
+  return 0;
+}
+
+
+
+// -----------------------------------------------------------------------------
+//
+// subCacheCreate -
+//
+SubCache* subCacheCreate(OrionldTenant* tenantP, bool scanSubs)
+{
+  SubCache* scP = (SubCache*) malloc(sizeof(SubCache));
+
+  if (scP == NULL)
+    LM_RE(NULL, ("Out of memory (attempt to create a subscription cache)"));
+
+  scP->tenantP  = tenantP;
+  scP->subList  = NULL;
+  scP->last     = NULL;
+
+  if (scanSubs)
+  {
+    if (mongocSubscriptionsIter(scP, subIterFunc) != 0)
+      LM_E(("mongocSubscriptionsIter failed"));
+  }
+
+  return scP;
+}