From 5cea18d24dc531643be077f28742277d756081c3 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Wed, 21 Feb 2024 20:18:58 +0100 Subject: [PATCH 1/2] Support for URI param 'kind' for GET Contexts --- CHANGES_NEXT_RELEASE | 3 + src/lib/logMsg/traceLevels.h | 1 + src/lib/orionld/common/orionldState.h | 2 + .../orionld/context/orionldContextCreate.cpp | 2 +- .../orionld/context/orionldContextFromUrl.cpp | 5 + .../context/orionldContextItemAliasLookup.cpp | 6 + .../context/orionldContextItemExpand.cpp | 4 + .../context/orionldContextItemLookup.cpp | 15 +- .../contextCache/orionldContextCacheGet.cpp | 39 +- .../contextCache/orionldContextCacheGet.h | 4 +- .../orionldContextCacheLookup.cpp | 7 +- .../dbModel/dbModelToApiSubscription.cpp | 1 - src/lib/orionld/mhd/mhdConnectionInit.cpp | 12 + .../orionld/service/orionldServiceInit.cpp | 1 + .../serviceRoutines/orionldGetContexts.cpp | 2 +- src/lib/orionld/types/OrionLdRestService.h | 1 + src/lib/orionld/types/OrionldContext.h | 4 +- .../ngsild_batch_upsert_and_contexts.test | 24 +- .../0000_ngsild/ngsild_context_creation.test | 26 +- .../0000_ngsild/ngsild_context_deletion.test | 24 +- .../0000_ngsild/ngsild_context_errors.test | 40 +- .../ngsild_context_list_with_kind.test | 467 ++++++++++++++++++ .../0000_ngsild/ngsild_context_location.test | 6 +- ...d_context_server_complex_user_context.test | 14 +- .../cases/0000_ngsild/ngsild_issue_807.test | 8 +- .../0000_ngsild/ngsild_mixed_context.test | 12 +- .../ngsild_new_batch_upsert_and_contexts.test | 24 +- .../ngsild_new_context_http-link-header.test | 8 +- ...line_array_with_mix-string-and-object.test | 8 +- ...h_one_string_that_IS_the_core_context.test | 4 +- ...e_string_that_is_not_the_core_context.test | 8 +- ...ems_of_which_none_is_the_core_context.test | 12 +- ...tems_of_which_one_is_the_core_context.test | 8 +- .../ngsild_new_context_inline_object.test | 4 +- .../ngsild_new_context_inline_string.test | 8 +- .../0000_ngsild/ngsild_new_issue_807.test | 8 +- .../ngsild_subscription_cache_issue-1316.test | 5 +- 37 files changed, 734 insertions(+), 93 deletions(-) create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_context_list_with_kind.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 4fe5efc630..0eb2f3141b 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -10,3 +10,6 @@ * Support for the new URL parameter "format" for output formats (normalized, concise, simplified) * Fixed a possible crash for TRoE and attributes of type Vocab/Json/Language * Modified the @context hosting feature to be according to API spec + * Support for URI param 'kind' for GET Contexts + * Improved counters for GET Context/Contexts + * API spec compliance for GET Context/Contexts diff --git a/src/lib/logMsg/traceLevels.h b/src/lib/logMsg/traceLevels.h index 045c1f85a4..6d6542199c 100644 --- a/src/lib/logMsg/traceLevels.h +++ b/src/lib/logMsg/traceLevels.h @@ -101,6 +101,7 @@ typedef enum TraceLevels LmtContexts = 100, // Contexts LmtContextTree, // Context Tree LmtContextCache, // Context Cache + LmtContextCacheStats, // Context Cache Statistics LmtContextDownload, // Context Download LmtCoreContext, // Core Context diff --git a/src/lib/orionld/common/orionldState.h b/src/lib/orionld/common/orionldState.h index 5128655e75..82717be144 100644 --- a/src/lib/orionld/common/orionldState.h +++ b/src/lib/orionld/common/orionldState.h @@ -170,6 +170,8 @@ typedef struct OrionldUriParams bool entityMap; char* format; + OrionldContextKind kind; + double observedAtAsDouble; uint64_t mask; } OrionldUriParams; diff --git a/src/lib/orionld/context/orionldContextCreate.cpp b/src/lib/orionld/context/orionldContextCreate.cpp index ab7bdb0724..4ea978d21d 100644 --- a/src/lib/orionld/context/orionldContextCreate.cpp +++ b/src/lib/orionld/context/orionldContextCreate.cpp @@ -77,7 +77,7 @@ OrionldContext* orionldContextCreate(const char* url, OrionldContextOrigin origi } contextP->keyValues = keyValues; - contextP->lookups = 0; + contextP->lookups = 1; return contextP; } diff --git a/src/lib/orionld/context/orionldContextFromUrl.cpp b/src/lib/orionld/context/orionldContextFromUrl.cpp index abcf4fda18..0d00aaae70 100644 --- a/src/lib/orionld/context/orionldContextFromUrl.cpp +++ b/src/lib/orionld/context/orionldContextFromUrl.cpp @@ -251,6 +251,11 @@ OrionldContext* orionldContextFromUrl(char* url, char* id) if (contextP != NULL) { + contextP->usedAt = orionldState.requestTime; + + contextP->lookups += 1; + LM_T(LmtContextCacheStats, ("Context '%s': %d lookups", url, contextP->lookups)); + LM_T(LmtContextDownload, ("Found already downloaded URL '%s'", url)); return contextP; } diff --git a/src/lib/orionld/context/orionldContextItemAliasLookup.cpp b/src/lib/orionld/context/orionldContextItemAliasLookup.cpp index 13b8e3b0ff..aba5daa37c 100644 --- a/src/lib/orionld/context/orionldContextItemAliasLookup.cpp +++ b/src/lib/orionld/context/orionldContextItemAliasLookup.cpp @@ -59,10 +59,15 @@ char* orionldContextItemAliasLookup if (contextItemPP != NULL) *contextItemPP = NULL; + if (contextP == NULL) + contextP = orionldCoreContextP; // 1. Is it the default URL? if (strncmp(longName, orionldDefaultUrl, orionldDefaultUrlLen) == 0) + { + orionldCoreContextP->compactions += 1; return (char*) &longName[orionldDefaultUrlLen]; + } // 2. Found in Core Context? contextItemP = orionldContextItemValueLookup(orionldCoreContextP, longName); @@ -91,5 +96,6 @@ char* orionldContextItemAliasLookup *contextItemPP = contextItemP; // Return the short name + contextP->compactions += 1; return contextItemP->name; } diff --git a/src/lib/orionld/context/orionldContextItemExpand.cpp b/src/lib/orionld/context/orionldContextItemExpand.cpp index b353d22477..00c383a71f 100644 --- a/src/lib/orionld/context/orionldContextItemExpand.cpp +++ b/src/lib/orionld/context/orionldContextItemExpand.cpp @@ -96,6 +96,8 @@ char* orionldContextItemExpand if (contextItemPP != NULL) *contextItemPP = NULL; + orionldCoreContextP->expansions += 1; // Really, @vocab expansions of the core context + return longName; } @@ -106,5 +108,7 @@ char* orionldContextItemExpand if (contextItemPP != NULL) *contextItemPP = contextItemP; + contextP->expansions += 1; // Really, @vocab expansions of the core context + return contextItemP->id; } diff --git a/src/lib/orionld/context/orionldContextItemLookup.cpp b/src/lib/orionld/context/orionldContextItemLookup.cpp index 45d15ca868..ada8f9762d 100644 --- a/src/lib/orionld/context/orionldContextItemLookup.cpp +++ b/src/lib/orionld/context/orionldContextItemLookup.cpp @@ -62,12 +62,17 @@ OrionldContextItem* orionldContextItemLookup(OrionldContext* contextP, const cha } } - if (valueMayBeCompactedP != NULL) + if (itemP != NULL) { - if ((itemP->type != NULL) && (strcmp(itemP->type, "@vocab") == 0)) - *valueMayBeCompactedP = true; - else - *valueMayBeCompactedP = false; + contextP->expansions += 1; + + if (valueMayBeCompactedP != NULL) + { + if ((itemP->type != NULL) && (strcmp(itemP->type, "@vocab") == 0)) + *valueMayBeCompactedP = true; + else + *valueMayBeCompactedP = false; + } } return itemP; diff --git a/src/lib/orionld/contextCache/orionldContextCacheGet.cpp b/src/lib/orionld/contextCache/orionldContextCacheGet.cpp index ea9193dd21..f9bad90565 100644 --- a/src/lib/orionld/contextCache/orionldContextCacheGet.cpp +++ b/src/lib/orionld/contextCache/orionldContextCacheGet.cpp @@ -32,7 +32,7 @@ extern "C" #include "logMsg/logMsg.h" // LM_* #include "logMsg/traceLevels.h" // Lmt* -#include "orionld/types/OrionldContext.h" // OrionldContext, orionldOriginToString +#include "orionld/types/OrionldContext.h" // OrionldContext, orionldOriginToString, OrionldContextKind #include "orionld/types/OrionldContextItem.h" // OrionldContextItem #include "orionld/common/orionldState.h" // orionldState #include "orionld/common/numberToDate.h" // numberToDate @@ -67,35 +67,39 @@ KjNode* orionldContextWithDetails(OrionldContext* contextP) KjNode* contextObjP = kjObject(orionldState.kjsonP, NULL); - KjNode* urlStringP = kjString(orionldState.kjsonP, "URL", contextP->url); - KjNode* idStringP = kjString(orionldState.kjsonP, "localId", (contextP->id == NULL)? "None" : contextP->id); - KjNode* kindP = kjString(orionldState.kjsonP, "kind", orionldKindToString(contextP->kind)); - KjNode* createdAtP = kjString(orionldState.kjsonP, "createdAt", createdAtString); - KjNode* extraInfoP = kjObject(orionldState.kjsonP, "extraInfo"); - KjNode* typeStringP = kjString(orionldState.kjsonP, "type", contextP->keyValues? "hash-table" : "array"); - KjNode* originP = kjString(orionldState.kjsonP, "origin", orionldOriginToString(contextP->origin)); + KjNode* urlStringP = kjString(orionldState.kjsonP, "URL", contextP->url); + KjNode* idStringP = kjString(orionldState.kjsonP, "localId", (contextP->id == NULL)? "None" : contextP->id); + KjNode* kindP = kjString(orionldState.kjsonP, "kind", orionldKindToString(contextP->kind)); + KjNode* createdAtP = kjString(orionldState.kjsonP, "createdAt", createdAtString); + KjNode* extraInfoP = kjObject(orionldState.kjsonP, "extraInfo"); + KjNode* typeStringP = kjString(orionldState.kjsonP, "type", (contextP->keyValues == true)? "hash-table" : "array"); + KjNode* originP = kjString(orionldState.kjsonP, "origin", orionldOriginToString(contextP->origin)); + KjNode* compactionsP = kjInteger(orionldState.kjsonP, "compactions", contextP->compactions); + KjNode* expansionsP = kjInteger(orionldState.kjsonP, "expansions", contextP->expansions); kjChildAdd(contextObjP, urlStringP); kjChildAdd(contextObjP, idStringP); kjChildAdd(contextObjP, kindP); kjChildAdd(contextObjP, createdAtP); kjChildAdd(contextObjP, extraInfoP); - kjChildAdd(extraInfoP, typeStringP); - kjChildAdd(extraInfoP, originP); + if (contextP != orionldCoreContextP) { - KjNode* usedAtP = kjString(orionldState.kjsonP, "lastUsage", lastUseString); - KjNode* lookupsP = kjInteger(orionldState.kjsonP, "numberOfHits", contextP->lookups); - + KjNode* usedAtP = kjString(orionldState.kjsonP, "lastUsage", lastUseString); + KjNode* lookupsP = kjInteger(orionldState.kjsonP, "numberOfHits", contextP->lookups); kjChildAdd(contextObjP, usedAtP); kjChildAdd(contextObjP, lookupsP); } + kjChildAdd(extraInfoP, typeStringP); + kjChildAdd(extraInfoP, originP); + kjChildAdd(extraInfoP, compactionsP); + kjChildAdd(extraInfoP, expansionsP); + if (contextP->parent != NULL) { KjNode* parentP = kjString(orionldState.kjsonP, "parent", contextP->parent); - kjChildAdd(extraInfoP, parentP); } @@ -155,15 +159,18 @@ KjNode* orionldContextWithDetails(OrionldContext* contextP) // // orionldContextCacheGet - // -KjNode* orionldContextCacheGet(KjNode* arrayP, bool details) +KjNode* orionldContextCacheGet(KjNode* arrayP, bool details, OrionldContextKind kind) { for (int ix = 0; ix < orionldContextCacheSlotIx; ix++) { - OrionldContext* contextP = orionldContextCacheArray[ix]; + OrionldContext* contextP = orionldContextCacheArray[ix]; if (contextP == NULL) continue; + if ((kind != OrionldContextUnknownKind) && (contextP->kind != kind)) + continue; + if (details == true) { KjNode* contextObjP = orionldContextWithDetails(contextP); diff --git a/src/lib/orionld/contextCache/orionldContextCacheGet.h b/src/lib/orionld/contextCache/orionldContextCacheGet.h index 05ee192240..0cdf55b346 100644 --- a/src/lib/orionld/contextCache/orionldContextCacheGet.h +++ b/src/lib/orionld/contextCache/orionldContextCacheGet.h @@ -30,12 +30,14 @@ extern "C" #include "kjson/KjNode.h" // KjNode } +#include "orionld/types/OrionldContext.h" // OrionldContextKind + // ----------------------------------------------------------------------------- // // orionldContextCacheGet - // -extern KjNode* orionldContextCacheGet(KjNode* arrayP, bool details); +extern KjNode* orionldContextCacheGet(KjNode* arrayP, bool details, OrionldContextKind kind); #endif // SRC_LIB_ORIONLD_CONTEXTCACHE_ORIONLDCONTEXTCACHEGET_H_ diff --git a/src/lib/orionld/contextCache/orionldContextCacheLookup.cpp b/src/lib/orionld/contextCache/orionldContextCacheLookup.cpp index 5afce5dd7f..7edbf5a4b4 100644 --- a/src/lib/orionld/contextCache/orionldContextCacheLookup.cpp +++ b/src/lib/orionld/contextCache/orionldContextCacheLookup.cpp @@ -49,16 +49,11 @@ OrionldContext* orionldContextCacheLookup(const char* url) if (strcmp(url, orionldContextCache[ix]->url) == 0) contextP = orionldContextCache[ix]; - - if ((orionldContextCache[ix]->id != NULL) && (strcmp(url, orionldContextCache[ix]->id) == 0)) + else if ((orionldContextCache[ix]->id != NULL) && (strcmp(url, orionldContextCache[ix]->id) == 0)) contextP = orionldContextCache[ix]; if (contextP != NULL) - { - contextP->usedAt = orionldState.requestTime; - contextP->lookups += 1; return contextP; - } } return NULL; diff --git a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp index dee719efc9..4fa86cf462 100644 --- a/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp +++ b/src/lib/orionld/dbModel/dbModelToApiSubscription.cpp @@ -548,7 +548,6 @@ KjNode* dbModelToApiSubscription } - // // origin // diff --git a/src/lib/orionld/mhd/mhdConnectionInit.cpp b/src/lib/orionld/mhd/mhdConnectionInit.cpp index c8a71badfc..993673b2e0 100644 --- a/src/lib/orionld/mhd/mhdConnectionInit.cpp +++ b/src/lib/orionld/mhd/mhdConnectionInit.cpp @@ -852,6 +852,18 @@ MHD_Result orionldUriArgumentGet(void* cbDataP, MHD_ValueKind kind, const char* orionldState.uriParams.subscriptionId = (char*) value; orionldState.uriParams.mask |= ORIONLD_URIPARAM_SUBSCRIPTION_ID; } + else if (strcmp(key, "kind") == 0) + { + orionldState.uriParams.kind = orionldKindFromString(value); + + if (orionldState.uriParams.kind == OrionldContextUnknownKind) + { + orionldError(OrionldBadRequestData, "Invalid value for uri parameter /kind/", value, 400); + return MHD_YES; + } + + orionldState.uriParams.mask |= ORIONLD_URIPARAM_KIND; + } else if (strcmp(key, "location") == 0) { if (strcmp(value, "true") == 0) diff --git a/src/lib/orionld/service/orionldServiceInit.cpp b/src/lib/orionld/service/orionldServiceInit.cpp index e0484e1a13..a5d05ad803 100644 --- a/src/lib/orionld/service/orionldServiceInit.cpp +++ b/src/lib/orionld/service/orionldServiceInit.cpp @@ -466,6 +466,7 @@ static void restServicePrepare(OrionLdRestService* serviceP, OrionLdRestServiceS serviceP->uriParams |= ORIONLD_URIPARAM_DETAILS; serviceP->uriParams |= ORIONLD_URIPARAM_LOCATION; serviceP->uriParams |= ORIONLD_URIPARAM_URL; + serviceP->uriParams |= ORIONLD_URIPARAM_KIND; serviceP->options |= ORIONLD_SERVICE_OPTION_DONT_ADD_CONTEXT_TO_RESPONSE_PAYLOAD; } diff --git a/src/lib/orionld/serviceRoutines/orionldGetContexts.cpp b/src/lib/orionld/serviceRoutines/orionldGetContexts.cpp index c141dd1d5e..cff5c8df3f 100644 --- a/src/lib/orionld/serviceRoutines/orionldGetContexts.cpp +++ b/src/lib/orionld/serviceRoutines/orionldGetContexts.cpp @@ -83,7 +83,7 @@ bool orionldGetContexts(void) orionldState.noLinkHeader = true; // We don't want the Link header for context requests - orionldState.responseTree = orionldContextCacheGet(contextTree, orionldState.uriParams.details); + orionldState.responseTree = orionldContextCacheGet(contextTree, orionldState.uriParams.details, orionldState.uriParams.kind); return true; } diff --git a/src/lib/orionld/types/OrionLdRestService.h b/src/lib/orionld/types/OrionLdRestService.h index e63887cb27..53ee8f0613 100644 --- a/src/lib/orionld/types/OrionLdRestService.h +++ b/src/lib/orionld/types/OrionLdRestService.h @@ -156,6 +156,7 @@ typedef struct OrionLdRestServiceSimplifiedVector #define ORIONLD_URIPARAM_ENTITYMAP (UINT64_C(1) << 39) #define ORIONLD_URIPARAM_FORMAT (UINT64_C(1) << 40) #define ORIONLD_URIPARAM_EXPAND_VALUES (UINT64_C(1) << 41) +#define ORIONLD_URIPARAM_KIND (UINT64_C(1) << 42) diff --git a/src/lib/orionld/types/OrionldContext.h b/src/lib/orionld/types/OrionldContext.h index b4e3f48115..a98d439ace 100644 --- a/src/lib/orionld/types/OrionldContext.h +++ b/src/lib/orionld/types/OrionldContext.h @@ -146,7 +146,9 @@ typedef struct OrionldContext bool coreContext; double createdAt; double usedAt; - int lookups; + int compactions; // Number of compactions done with this @context + int expansions; // Number of expansions done with this @context + int lookups; // Number of NGSI-LD requests done using this @context bool keyValues; OrionldContextInfo context; OrionldContextOrigin origin; diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert_and_contexts.test b/test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert_and_contexts.test index be77f07fcf..91d7a57bb7 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert_and_contexts.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert_and_contexts.test @@ -726,7 +726,7 @@ Link: Date: Wed, 21 Feb 2024 20:31:29 +0100 Subject: [PATCH 2/2] Fixed a few functests --- .../0000_ngsild/ngsild_batch_upsert_and_contexts.test | 10 +++++----- .../cases/0000_ngsild/ngsild_context_location.test | 2 +- .../ngsild_new_batch_upsert_and_contexts.test | 2 +- .../ngsild_new_context_http-link-header.test | 2 +- .../0000_ngsild/ngsild_new_context_inline_string.test | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert_and_contexts.test b/test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert_and_contexts.test index 91d7a57bb7..5811c69575 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert_and_contexts.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert_and_contexts.test @@ -739,8 +739,8 @@ Date: REGEX(.*) "extraInfo": { "type": "hash-table", "origin": "Downloaded", - "compactions": 0, - "expansions": 281, + "compactions": 5, + "expansions": 286, "hash-table": { "instanceId": "https://uri.etsi.org/ngsi-ld/instanceId", "notifiedAt": "https://uri.etsi.org/ngsi-ld/notifiedAt", @@ -996,7 +996,7 @@ Link: