Merge pull request #1684 from FIWARE/merge/withdevelop

Merge/withdevelop

CHANGES_NEXT_RELEASE

@@ -1,6 +1,18 @@
 ## Fixed Issues:
+  #1621: URL-Encoding single quote in attribute values in TRoE DB
+  #1649: Crash when using subscriptions in a mixed environment (LD and V2)
+  #1682: Default User Context 
 
 ## New Features:
-  * Distributed subscriptions: subordinate subscriptions are DELETED when their "father" is deleted
   * Native support for DDS
+  * Distributed subscriptions: subordinate subscriptions are DELETED when their "father" is deleted.
+  * Support for the URL parameter 'csf' in GET /ngsi-ld/v1/csourceRegistrations
+  * Support for the URL parameter 'orderBy' (must be an attribute) in GET /ngsi-ld/v1/entities, but only if 'local' is set. (This is not NGSI-LD standard. Yet ...)
+    - Also supporting sorting on entity id, type, and modifiedAt
+    - One more URL parameter '?reverse=true' to reverse the sorting order
+  * Support for GET /ngsi-ld/v1/info/sourceIdentity
+  * Support for management::localOnly in registrations
+  * Support for new CLI parameter '-pageSize' to set the default pagination limit (default is 20 if -pageSize is not used)
+
 ## Notes
+  * Lots of improvements for subordinate subscriptions - still not 100% ready, but a lot better.

src/app/orionld/orionld.cpp

@@ -56,7 +56,7 @@
 */
 #include <stdio.h>
 #include <unistd.h>                                         // getppid, fork, setuid, sleep, gethostname, etc.
-#include <string.h>
+#include <string.h>                                         // strchr
 #include <fcntl.h>                                          // open
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -249,6 +249,8 @@ char            coreContextVersion[64];
 bool            triggerOperation = false;
 bool            noprom           = false;
 bool            noArrayReduction = false;
+char            subordinateEndpoint[256];
+char            defaultUserContextUrl[256];
 bool            ddsSupport       = false;
 char            ddsSubsTopics[512];
 char            ddsTopicType[512];
@@ -353,6 +355,9 @@ char            ddsEnablerConfigFile[512];
 #define DDS_TOPIC_TYPE_DESC    "DDS topic type"
 #define DDS_CONFIG_FILE_DESC   "DDS configuration file"
 #define DDS_ENABLER_CONFIG_FILE_DESC   "DDS Enabler configuration file"
+#define SUBORDINATE_ENDPOINT_DESC  "endpoint URL for reception of notificatiopns from subordinate subscriptions (distributed subscriptions)"
+#define PAGE_SIZE_DESC         "default page size (no of entities, subscriptions, registrations)"
+#define DUC_URL_DESC           "URL to default user context"
 
 
 
@@ -412,7 +417,6 @@ PaArgument paArgs[] =
   { "-connectionMemory",      &connectionMemory,        "CONN_MEMORY",               PaUInt,    PaOpt,  64,               0,      1024,             CONN_MEMORY_DESC         },
   { "-maxConnections",        &maxConnections,          "MAX_CONN",                  PaUInt,    PaOpt,  1020,             1,      PaNL,             MAX_CONN_DESC            },
   { "-reqPoolSize",           &reqPoolSize,             "TRQ_POOL_SIZE",             PaUInt,    PaOpt,  0,                0,      1024,             REQ_POOL_SIZE            },
-
   { "-inReqPayloadMaxSize",   &inReqPayloadMaxSize,     "IN_REQ_PAYLOAD_MAX_SIZE",   PaULong,   PaOpt,  MB(1),            0,      PaNL,             IN_REQ_PAYLOAD_MAX_SIZE_DESC },
   { "-outReqMsgMaxSize",      &outReqMsgMaxSize,        "OUT_REQ_MSG_MAX_SIZE",      PaULong,   PaOpt,  MB(8),            0,      PaNL,             OUT_REQ_MSG_MAX_SIZE_DESC    },
   { "-notificationMode",      &notificationMode,        "NOTIF_MODE",                PaString,  PaOpt,  _i "transient",   PaNL,   PaNL,             NOTIFICATION_MODE_DESC   },
@@ -458,11 +462,14 @@ PaArgument paArgs[] =
   { "-lmtmp",                 &lmtmp,                   "TMP_TRACES",                PaBool,    PaHid,  true,             false,  true,             TMPTRACES_DESC           },
   { "-noprom",                &noprom,                  "NO_PROM",                   PaBool,    PaHid,  false,            false,  true,             NO_PROM_DESC             },
   { "-noArrayReduction",      &noArrayReduction,        "NO_ARRAY_REDUCTION",        PaBool,    PaHid,  false,            false,  true,             NO_ARR_REDUCT_DESC       },
+  { "-subordinateEndpoint",   &subordinateEndpoint,     "SUBORDINATE_ENDPOINT",      PaStr,     PaOpt,  _i "",           PaNL,   PaNL,             SUBORDINATE_ENDPOINT_DESC },
+  { "-pageSize",              &pageSize,                "PAGE_SIZE",                 PaInt,     PaOpt,  20,              1,      1000,             PAGE_SIZE_DESC            },
   { "-dds",                   &ddsSupport,              "DDS",                       PaBool,    PaOpt,  false,            false,  true,             USE_DDS_DESC             },
   { "-ddsSubsTopics",         ddsSubsTopics,            "DDS_SUBS_TOPICS",           PaString,  PaOpt,  _i "",            PaNL,   PaNL,             DDS_SUBS_TOPICS_DESC     },
   { "-ddsTopicType",          ddsTopicType,             "DDS_TOPIC_TYPE",            PaString,  PaOpt,  _i "NGSI-LD",     PaNL,   PaNL,             DDS_TOPIC_TYPE_DESC      },
   { "-ddsConfigFile",         ddsConfigFile,            "DDS_CONFIG_FILE",           PaString,  PaOpt,  _i "",            PaNL,   PaNL,             DDS_CONFIG_FILE_DESC     },
   { "-ddsEnablerConfigFile",  ddsEnablerConfigFile,     "DDS_CONFIG_FILE_PATH",      PaString,  PaOpt,  _i "",            PaNL,   PaNL,             DDS_ENABLER_CONFIG_FILE_DESC },
+  { "-duc",                   defaultUserContextUrl,    "DUC_URL",                   PaString,  PaOpt,  _i "",           PaNL,   PaNL,             DUC_URL_DESC              },
 
   PA_END_OF_ARGS
 };
@@ -1115,6 +1122,23 @@ int main(int argC, char* argV[])
     }
   }
 
+  if (subordinateEndpoint[0] != 0)
+  {
+    char* slash = strchr(subordinateEndpoint, '/');
+
+    if ((slash != NULL) && (slash[1] == '/') && (slash[2] != 0))  // xxx:// is skipped
+      slash = strchr(&slash[2], '/');
+
+    char* prefix = slash;
+    subordinatePathLen = snprintf(subordinatePath, sizeof(subordinatePath) - 1, "%s/notifications/", prefix);
+
+    LM_T(LmtSubordinate, ("entity subordinate prefix: '%s'", subordinateEndpoint));
+    LM_T(LmtSubordinate, ("path portion:              '%s'", prefix));
+    LM_T(LmtSubordinate, ("subordinatePath:           '%s'", subordinatePath));
+  }
+  else
+    bzero(subordinatePath, sizeof(subordinatePath));
+
 #if 0
   //
   // Uncomment this piece of code and run the functests (-ld) to make sure everything works "more or less" with the Legacy Driver disabled.

src/app/orionld/orionldRestServices.cpp

@@ -69,6 +69,7 @@
 #include "orionld/serviceRoutines/orionldPutAttribute.h"
 #include "orionld/serviceRoutines/orionldGetEntityMap.h"
 #include "orionld/serviceRoutines/orionldDeleteEntityMap.h"
+#include "orionld/serviceRoutines/orionldGetInfo.h"
 
 #include "orionld/serviceRoutines/orionldGetTemporalEntities.h"
 #include "orionld/serviceRoutines/orionldGetTemporalEntity.h"
@@ -105,6 +106,7 @@ static OrionLdRestServiceSimplified getServiceV[] =
   { "/ngsi-ld/v1/csourceRegistrations",    orionldGetRegistrations    },
   { "/ngsi-ld/v1/jsonldContexts/*",        orionldGetContext          },
   { "/ngsi-ld/v1/jsonldContexts",          orionldGetContexts         },
+  { "/ngsi-ld/v1/info/sourceIdentity",     orionldGetInfo             },
   { "/ngsi-ld/v1/temporal/entities/*",     orionldGetTemporalEntity   },
   { "/ngsi-ld/v1/temporal/entities",       orionldGetTemporalEntities },
   { "/ngsi-ld/ex/v1/version",              orionldGetVersion          },

src/lib/logMsg/traceLevels.h

@@ -105,6 +105,7 @@ typedef enum TraceLevels
   LmtContextCacheStats,                // Context Cache Statistics
   LmtContextDownload,                  // Context Download
   LmtCoreContext,                      // Core Context
+  LmtUserContext,                      // User Context
 
   // GeoJSON
   LmtGeoJSON = 110,                    // GeoJSON ... everything (for now)
@@ -125,6 +126,11 @@ typedef enum TraceLevels
   LmtEntityMapRetrieve,                // Retrieval of an entity map
   LmtEntityMapDetail,                  // Details of the entity-registration maps
 
+  //
+  // Subordinate Subscriptions
+  //
+  LmtSubordinate = 140,
+
   //
   // Misc
   //
@@ -135,6 +141,7 @@ typedef enum TraceLevels
   LmtKjlParse,                         // Trace level start for K libs
   LmtMqtt = 205,                       // MQTT notifications
   LmtQ,                                // Query Language
+  LmtCsf,                              // CSF - 'q' for registrations
   LmtPostgres,                         // Postgres (TRoE)
   LmtSql,                              // SQL command for TRoE
   LmtPgPool,                           // Postgres Connection Pool

src/lib/mongoBackend/mongoGetSubscriptions.cpp

@@ -78,7 +78,13 @@ using ngsiv2::EntID;
 */
 static void extractSubscriptionId(Subscription* s, const BSONObj* rP)
 {
-  s->id = getFieldF(rP, "_id").OID().toString();
+  mongo::BSONElement  bsonElement = rP->getField("_id");
+  mongo::BSONType     bsonType    = bsonElement.type();
+
+  if (bsonType == mongo::jstOID)
+    s->id = bsonElement.OID().toString();
+  else
+    s->id = bsonElement.String();
 }
 
 

src/lib/orionld/common/orionldState.cpp

@@ -110,6 +110,8 @@ PernotSubCache    pernotSubCache;
 EntityMap*        entityMaps        = NULL;    // Used by GET /entities in the distributed case, for pagination
 bool              entityMapsEnabled = false;
 bool              distSubsEnabled   = false;
+OrionldContext*   defaultUserContextP = NULL;
+int               pageSize            = 20;
 
 
 
@@ -125,6 +127,15 @@ char                   postgresServerVersion[128];
 
 
 
+//
+// Variables for notifications from subordinate subscriptions (distributed subscriptions)
+//
+char                   subordinatePath[256];
+int                    subordinatePathLen              = -1;
+OrionLdRestService*    subordinateNotificationServiceP = NULL;
+
+
+
 // -----------------------------------------------------------------------------
 //
 // orionldStateInit - initialize the thread-local variable orionldState
@@ -163,7 +174,7 @@ void orionldStateInit(MHD_Connection* connection)
 
   // Pagination
   orionldState.uriParams.offset        = 0;
-  orionldState.uriParams.limit         = 20;
+  orionldState.uriParams.limit         = pageSize;
 
   // orionldState.delayedKjFreeVecSize    = sizeof(orionldState.delayedKjFreeVec) / sizeof(orionldState.delayedKjFreeVec[0]);
 

src/lib/orionld/common/orionldState.h

@@ -156,6 +156,7 @@ typedef struct OrionldUriParams
   char*     notExists;
   char*     metadata;
   char*     orderBy;
+  bool      reverse;
   bool      collapse;
   bool      reset;
   char*     attributeFormat;
@@ -169,6 +170,7 @@ typedef struct OrionldUriParams
   bool      onlyIds;
   bool      entityMap;
   char*     format;
+  char*     csf;
 
   OrionldContextKind kind;
 
@@ -320,6 +322,7 @@ typedef struct OrionldMongoC
 typedef struct OrionldConnectionState
 {
   OrionldPhase            phase;
+  bool                    orionldErrorDone;          // Don't override error - don't call orionldError()
   bool                    distributed;               // Depends on a URI param, but can be modified (to false) via an HTTP header
   MHD_Connection*         mhdConnection;
   char                    clientIp[64];              // IP address of the requester
@@ -619,6 +622,10 @@ extern EntityMap*        entityMaps;               // Used by GET /entities in t
 extern bool              entityMapsEnabled;        // Enable Entity Maps
 extern bool              distSubsEnabled;          // Enable distributed subscriptions
 extern bool              noArrayReduction;         // Used by arrayReduce in pCheckAttribute.cpp
+extern int               pageSize;                 // Pagination limit
+extern char              defaultUserContextUrl[256];
+extern OrionldContext*   defaultUserContextP;
+
 extern char                localIpAndPort[135];    // Local address for X-Forwarded-For (from orionld.cpp)
 extern unsigned long long  inReqPayloadMaxSize;
 extern unsigned long long  outReqMsgMaxSize;
@@ -663,6 +670,16 @@ extern char                   postgresServerVersion[128];
 
 
 
+// -----------------------------------------------------------------------------
+//
+// Variables for notifications from subordinate subscriptions (distributed subscriptions)
+//
+extern char                subordinateEndpoint[256];
+extern char                subordinatePath[256];
+extern int                 subordinatePathLen;
+extern OrionLdRestService* subordinateNotificationServiceP;
+
+
 // -----------------------------------------------------------------------------
 //
 // orionldStateInit - initialize the thread-local variable orionldState

src/lib/orionld/context/orionldAttributeExpand.cpp

@@ -44,21 +44,23 @@
 char* orionldAttributeExpand
 (
   OrionldContext*       contextP,
-  char*                 shortName,
+  const char*           shortName,
   bool                  useDefaultUrlIfNotFound,
   OrionldContextItem**  contextItemPP
 )
 {
-  if      (strcmp(shortName, "id")               == 0) return shortName;
-  else if (strcmp(shortName, "@id")              == 0) return shortName;
-  else if (strcmp(shortName, "type")             == 0) return shortName;
-  else if (strcmp(shortName, "@type")            == 0) return shortName;
-  else if (strcmp(shortName, "scope")            == 0) return shortName;
-  else if (strcmp(shortName, "location")         == 0) return shortName;
-  else if (strcmp(shortName, "createdAt")        == 0) return shortName;
-  else if (strcmp(shortName, "modifiedAt")       == 0) return shortName;
-  else if (strcmp(shortName, "observationSpace") == 0) return shortName;
-  else if (strcmp(shortName, "operationSpace")   == 0) return shortName;
+  char* sName = (char*) shortName;  // just to avoid a warning for strcmp
+
+  if      (strcmp(sName, "id")               == 0) return sName;
+  else if (strcmp(sName, "@id")              == 0) return sName;
+  else if (strcmp(sName, "type")             == 0) return sName;
+  else if (strcmp(sName, "@type")            == 0) return sName;
+  else if (strcmp(sName, "scope")            == 0) return sName;
+  else if (strcmp(sName, "location")         == 0) return sName;
+  else if (strcmp(sName, "createdAt")        == 0) return sName;
+  else if (strcmp(sName, "modifiedAt")       == 0) return sName;
+  else if (strcmp(sName, "observationSpace") == 0) return sName;
+  else if (strcmp(sName, "operationSpace")   == 0) return sName;
 
 #if 1
   // FIXME: 'observedAt' as an attribute is not a thing - special treatment only if sub-attribute
@@ -70,15 +72,15 @@ char* orionldAttributeExpand
   //
   //  We should probably forbid an attribute to have the name 'observedAt'
   //
-  else if (strcmp(shortName, "observedAt") == 0)
+  else if (strcmp(sName, "observedAt") == 0)
   {
-    orionldContextItemExpand(contextP, shortName, false, contextItemPP);
-    return shortName;
+    orionldContextItemExpand(contextP, sName, false, contextItemPP);
+    return sName;
   }
 #endif
 
-  if (orionldContextItemAlreadyExpanded(shortName) == true)
-    return shortName;
+  if (orionldContextItemAlreadyExpanded(sName) == true)
+    return sName;
 
-  return orionldContextItemExpand(contextP, shortName, useDefaultUrlIfNotFound, contextItemPP);
+  return orionldContextItemExpand(contextP, sName, useDefaultUrlIfNotFound, contextItemPP);
 }

src/lib/orionld/context/orionldAttributeExpand.h

@@ -41,7 +41,7 @@
 extern char* orionldAttributeExpand
 (
   OrionldContext*       contextP,
-  char*                 shortName,
+  const char*           shortName,
   bool                  useDefaultUrlIfNotFound,
   OrionldContextItem**  contextItemPP
 );

src/lib/orionld/contextCache/orionldContextCacheInit.cpp

@@ -185,6 +185,16 @@ void orionldContextCacheInit(void)
     LM_W(("Falling back to Built-in Core Context (hard-coded copy of %s)", builtinCoreContextUrl));
   }
 
+  //
+  // Default User Context
+  //
+  if (defaultUserContextUrl[0] != 0)
+  {
+    defaultUserContextP = orionldContextFromUrl(defaultUserContextUrl, NULL);
+    if (defaultUserContextP == NULL)
+      LM_W(("Unable to download the default user context"));
+  }
+
   if (contextArray == NULL)
     return;
 

src/lib/orionld/dbModel/dbModelToApiRegistration.cpp

@@ -26,6 +26,7 @@
 
 extern "C"
 {
+#include "kalloc/kaStrdup.h"                                     // kaStrdup
 #include "kjson/KjNode.h"                                        // KjNode
 #include "kjson/kjLookup.h"                                      // kjLookup
 #include "kjson/kjBuilder.h"                                     // kjArray, kjString, kjChildAdd, kjChildRemove, ...
@@ -35,6 +36,7 @@ extern "C"
 
 #include "orionld/common/orionldState.h"                         // orionldState
 #include "orionld/common/numberToDate.h"                         // numberToDate
+#include "orionld/common/eqForDot.h"                             // eqForDot
 #include "orionld/context/orionldContextItemAliasLookup.h"       // orionldContextItemAliasLookup
 #include "orionld/dbModel/dbModelToApiRegistration.h"            // Own interface
 
@@ -246,7 +248,12 @@ bool dbModelToApiRegistration(KjNode* dbRegP, bool sysAttrs, bool forCache)
       {
         for (KjNode* propertyP = propertiesP->value.firstChildP; propertyP != NULL; propertyP = propertyP->next)
         {
-          propertyP->name = orionldContextItemAliasLookup(orionldState.contextP, propertyP->name, NULL, NULL);
+          char dotName[256];
+          strncpy(dotName, propertyP->name, sizeof(dotName) - 1);
+          eqForDot(dotName);
+
+          propertyP->name = orionldContextItemAliasLookup(orionldState.contextP, dotName, NULL, NULL);
+          propertyP->name = kaStrdup(&orionldState.kalloc, propertyP->name);
         }
       }
 

src/lib/orionld/distOp/distOpSend.cpp

@@ -498,9 +498,6 @@ bool distOpSend(DistOp* distOpP, const char* dateHeader, const char* xForwardedF
         uriParamAdd(&urlParts, "id", distOpP->entityId, -1);
     }
 
-    if (local == true)
-      uriParamAdd(&urlParts, "local=true", NULL, 10);
-
     if (distOpP->qNode != NULL)
     {
       char buf[256];
@@ -520,6 +517,9 @@ bool distOpSend(DistOp* distOpP, const char* dateHeader, const char* xForwardedF
     }
   }
 
+  if ((local == true) || (distOpP->regP->localOnly == true))
+    uriParamAdd(&urlParts, "local=true", NULL, 10);
+
   if (orionldState.uriParams.lang != NULL)
     uriParamAdd(&urlParts, "lang", orionldState.uriParams.lang, -1);
 

src/lib/orionld/entityMaps/entityMapCreate.cpp

@@ -213,6 +213,7 @@ EntityMap* entityMapCreate(DistOp* distOpList, char* idPattern, QNode* qNode, Or
                                                geoInfoP,
                                                NULL,
                                                geojsonGeometryLongName,
+                                               orionldState.uriParams.orderBy,
                                                true,
                                                false);