From 51a71bbd8113d91b0c1858d8f1f1aada294a61be Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Wed, 5 Jul 2023 04:17:34 +0000 Subject: [PATCH] Updated code as per comment --- CHANGES_NEXT_RELEASE | 2 +- doc/manuals/admin/logs.md | 1 + src/app/contextBroker/contextBroker.cpp | 15 +++++++++++++-- src/lib/common/globals.cpp | 1 - src/lib/common/limits.h | 2 -- src/lib/ngsiNotify/QueueNotifier.cpp | 14 ++++++++------ ...ation_queue_overpassing_a_given_threshold.test | 8 ++++---- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0c9dc37103..f1c73a9f48 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: Alarm for notification queue overpassing a given threshold (#4113) +- Fix: alarm for notification queue overpassing a given threshold (#4113) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index 192e835dbe..6553757bdd 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -233,6 +233,7 @@ Alarm conditions: | 5 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm NotificationError ``: ``". | The following WARN text appears in the 'msg' field: "Releasing alarm NotificationError ``", where `` is the same one that triggered the alarm. Orion prints this trace when it successfully sent a notification to that URL. | Notification Failure. The ``text contains the detailed information. | Orion is trying to send the HTTP notification to a given receiver and some problem has occurred. It could be due to a problem with the network connectivity or on the receiver, e.g. the receiver is down. In the second case, the owner of the receiver of the notification should be reported. No specific action has to be performed at Orion Context Broker service. | 6 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm ForwardingError ``": ``". | The following WARN text appears in the 'msg' field: "Releasing alarm ForwardingError ``", where `` is the same one that triggered the alarm. Orion prints this trace when it successfully interact with ContextProvider to that URL.| Forwarding Error. The ``text contains the detailed information. | Orion is trying to interact with ContextProvider and some problem has occurred. It may be due to context provider response for forwarded query or update is empty. No specific action has to be performed at Orion Context Broker service. | 7 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm MqttConnectionError ``": ``". | The following WARN text appears in the 'msg' field: "Releasing alarm MqttConnectionError ``", where `` is the same one that triggered the alarm. Orion prints this trace when it successfully interact with ContextProvider to that URL.| Error connection to MQTT broker. The ``text contains the detailed information. | Orion is trying to connecto to an MQTT broker (associated to a subscription) and some problem has occurred. It may be due to several reasons: MQTT broker in unreachable, user/pass is wrong, etc. No specific action has to be performed at Orion Context Broker service, but maybe in the MQTT broker configuration or in the associated subscription. +| 8 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm NotificaitonQueue ``": ``". | The following WARN text appears in the 'msg' field: "Releasing alarm NotificaitonQueue ``", where `` is the same one that triggered the alarm. Orion prints this trace when notification queue goes back below the threshold. | The notification queue associated to the service (or ``" "default" for default queue) has overpassed the alarm threshold. The ``" text described the particular threshold. | No specific action has to be performed at Orion Context Broker service, but the update flow causing the notification on that service (or default queue) should be lowered in order to reduce pressure on queue. Another possible problem is due to malfunctioning notification receivers, if they are slow processing notifications and responding to Orion. By default, Orion only traces the origin (i.e. raising) and end (i.e. releasing) of an alarm, e.g: diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index 25908db8e3..59ff306550 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -206,6 +206,7 @@ unsigned long fcMaxInterval; int mqttMaxAge; bool logDeprecate; +long int maxthreshold; @@ -256,7 +257,6 @@ bool logDeprecate; #define REQ_POOL_SIZE "size of thread pool for incoming connections" #define IN_REQ_PAYLOAD_MAX_SIZE_DESC "maximum size (in bytes) of the payload of incoming requests" #define OUT_REQ_MSG_MAX_SIZE_DESC "maximum size (in bytes) of outgoing forward and notification request messages" -#define THRESHOLD_MAX_SIZE_DESC "maximum threshold limit for notificationQueue" #define SIMULATED_NOTIF_DESC "simulate notifications instead of actual sending them (only for testing)" #define STAT_COUNTERS "enable request/notification counters statistics" #define STAT_SEM_WAIT "enable semaphore waiting time statistics" @@ -276,8 +276,8 @@ bool logDeprecate; #define INSECURE_NOTIF_DESC "allow HTTPS notifications to peers which certificate cannot be authenticated with known CA certificates" #define NGSIV1_AUTOCAST_DESC "automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations" #define MQTT_MAX_AGE_DESC "max time (in minutes) that an unused MQTT connection is kept, default: 60" -//#define MAX_THRESHOLD_DESC "max time (in minutes) that an unused MQTT connection is kept, default: 60" #define LOG_DEPRECATE_DESC "log deprecation usages as warnings" +#define MAX_THRESHOLD_DESC "maximum threshold for notification queue, default: 80%" @@ -368,6 +368,8 @@ PaArgument paArgs[] = { "-logDeprecate", &logDeprecate, "LOG_DEPRECATE", PaBool, PaOpt, false, false, true, LOG_DEPRECATE_DESC }, + { "-maxthreshold", &maxthreshold, "MAX_THRESHOLD", PaInt, PaOpt, 0, PaNL, PaNL, MAX_THRESHOLD_DESC }, + PA_END_OF_ARGS }; @@ -658,6 +660,15 @@ static void contextBrokerInit(void) QueueNotifier* pQNotifier = new QueueNotifier(notificationQueueSize, notificationThreadNum, serviceV, serviceQueueSizeV, serviceNumThreadV); int rc = pQNotifier->start(); + if (maxthreshold > 0) + { + maxthreshold=notificationQueueSize*maxthreshold/100; + } + else + { + maxthreshold=notificationQueueSize*80; + } + if (rc != 0) { LM_X(1,("Runtime Error starting notification queue workers (%d)", rc)); diff --git a/src/lib/common/globals.cpp b/src/lib/common/globals.cpp index c379efb252..511dbd9d0d 100644 --- a/src/lib/common/globals.cpp +++ b/src/lib/common/globals.cpp @@ -57,7 +57,6 @@ bool notifQueueStatistics = false; bool checkIdv1 = false; unsigned long long inReqPayloadMaxSize = DEFAULT_IN_REQ_PAYLOAD_MAX_SIZE; unsigned long long outReqMsgMaxSize = DEFAULT_OUT_REQ_MSG_MAX_SIZE; -unsigned long long thresholdMaxSize = DEFAULT_THRESHOLD_MAX_SIZE; diff --git a/src/lib/common/limits.h b/src/lib/common/limits.h index e7c0c8e80d..39295ec64d 100644 --- a/src/lib/common/limits.h +++ b/src/lib/common/limits.h @@ -112,8 +112,6 @@ #define DEFAULT_OUT_REQ_MSG_MAX_SIZE (8 * 1024 * 1024) // 8 MB default max size of any outgoing request message (see CLI -outReqMsgMaxSize) -#define DEFAULT_THRESHOLD_MAX_SIZE 3 - /* **************************************************************************** * diff --git a/src/lib/ngsiNotify/QueueNotifier.cpp b/src/lib/ngsiNotify/QueueNotifier.cpp index e5d42a9c25..1d2a88677f 100644 --- a/src/lib/ngsiNotify/QueueNotifier.cpp +++ b/src/lib/ngsiNotify/QueueNotifier.cpp @@ -180,17 +180,19 @@ void QueueNotifier::sendNotifyContextRequest if (enqueued) { - std::string details; - extern int thresholdMaxSize; - details = "(Max threshold limit: 3)"; - if (QueueStatistics::getIn() >= thresholdMaxSize) + extern long int maxthreshold; + extern int notificationQueueSize; + + std::string details = ("notification queue reached maximum threshold"); + + if (maxthreshold >= notificationQueueSize) { - alarmMgr.notificationQueue(service, "notification queue reached the threshold " + details); + alarmMgr.notificationQueue(queueName.c_str(), details); } else { - alarmMgr.notificationQueuesResets(service); + alarmMgr.notificationQueuesResets(queueName.c_str()); } } diff --git a/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test index 572801f225..8f31934fbb 100644 --- a/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test +++ b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test @@ -27,7 +27,7 @@ alarm for notification queue overpassing a given threshold dbInit ${CB_DB_NAME} serv1 dbInit ${CB_DB_NAME} serv2 dbInit ${CB_DB_NAME} serv3 -brokerStart CB 0 IPv4 -multiservice -notificationMode threadpool:3:2 +brokerStart CB 0 IPv4 -multiservice -maxthreshold 100 -notificationMode threadpool:3:2,serv1:3:2,serv2:3:2,serv3:3:2 accumulatorStart --pretty-print localhost $LISTENER_PORT --SHELL-- @@ -80,7 +80,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/notify" + "url": "http://localhost:'$LISTENER_PORT'/waitForever" } } }' @@ -103,7 +103,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/notify" + "url": "http://localhost:'$LISTENER_PORT'/waitForever" } } }' @@ -220,7 +220,7 @@ Content-Length: 0 07. Grep log for notificationQueue alarm ======================================== -Raising alarm NotificationQueue serv2: notification queue reached the threshold (Max threshold limit: 3) +Raising alarm NotificationQueue serv1: notification queue reached maximum threshold --TEARDOWN--