Skip to content

Commit

Permalink
Merge pull request #1357 from FIWARE/issue/1356
Browse files Browse the repository at this point in the history
Fixed issue #1356
  • Loading branch information
kzangeli authored Apr 26, 2023
2 parents 2869591 + 0a2ee23 commit 2d4ab7f
Show file tree
Hide file tree
Showing 10 changed files with 127 additions and 10 deletions.
2 changes: 1 addition & 1 deletion CHANGES_NEXT_RELEASE
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,4 @@
* Issue #1353 properties/relationships in regs is now propertyNames/relationshipNames in all responses for GET Regs
* Issue #280 Implemented trace messages for incoming URL parameters (trace level LmtUriParams (-t 33))
* Issue #280 Implemented trace messages for incoming HTTP headers (trace level LmtHeaders (-t 32))
* Issue #1356 Made Orion-LD accept a configureable max size for incoming/outgoing payload bodies
17 changes: 17 additions & 0 deletions src/app/orionld/orionld.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ bool noCache;
unsigned int connectionMemory;
unsigned int maxConnections;
unsigned int reqPoolSize;

unsigned long long inReqPayloadMaxSize;
unsigned long long outReqMsgMaxSize;

bool simulatedNotification;
bool statCounters;
bool statSemWait;
Expand Down Expand Up @@ -293,6 +297,8 @@ int cSubCounters;
#define CONN_MEMORY_DESC "maximum memory size per connection (in kilobytes)"
#define MAX_CONN_DESC "maximum number of simultaneous connections"
#define REQ_POOL_SIZE "size of thread pool for incoming connections"
#define IN_REQ_PAYLOAD_MAX_SIZE_DESC "maximum size (in bytes) of the payload of incoming requests"
#define OUT_REQ_MSG_MAX_SIZE_DESC "maximum size (in bytes) of outgoing forward and notification request messages"
#define SIMULATED_NOTIF_DESC "simulate notifications instead of actual sending them (only for testing)"
#define STAT_COUNTERS "enable request/notification counters statistics"
#define STAT_SEM_WAIT "enable semaphore waiting time statistics"
Expand Down Expand Up @@ -335,6 +341,14 @@ int cSubCounters;



// -----------------------------------------------------------------------------
//
// MB - megabytes
//
#define MB(mbs) (1024 * 1024 * mbs)



/* ****************************************************************************
*
* paArgs - option vector for the Parse CLI arguments library
Expand Down Expand Up @@ -383,6 +397,9 @@ PaArgument paArgs[] =
{ "-connectionMemory", &connectionMemory, "CONN_MEMORY", PaUInt, PaOpt, 64, 0, 1024, CONN_MEMORY_DESC },
{ "-maxConnections", &maxConnections, "MAX_CONN", PaUInt, PaOpt, 1020, 1, PaNL, MAX_CONN_DESC },
{ "-reqPoolSize", &reqPoolSize, "TRQ_POOL_SIZE", PaUInt, PaOpt, 0, 0, 1024, REQ_POOL_SIZE },

{ "-inReqPayloadMaxSize", &inReqPayloadMaxSize, "IN_REQ_PAYLOAD_MAX_SIZE", PaULong, PaOpt, MB(1), 0, PaNL, IN_REQ_PAYLOAD_MAX_SIZE_DESC },
{ "-outReqMsgMaxSize", &outReqMsgMaxSize, "OUT_REQ_MSG_MAX_SIZE", PaULong, PaOpt, MB(8), 0, PaNL, OUT_REQ_MSG_MAX_SIZE_DESC },
{ "-notificationMode", &notificationMode, "NOTIF_MODE", PaString, PaOpt, _i "transient", PaNL, PaNL, NOTIFICATION_MODE_DESC },
{ "-simulatedNotification", &simulatedNotification, "DROP_NOTIF", PaBool, PaOpt, false, false, true, SIMULATED_NOTIF_DESC },
{ "-statCounters", &statCounters, "STAT_COUNTERS", PaBool, PaOpt, false, false, true, STAT_COUNTERS },
Expand Down
2 changes: 2 additions & 0 deletions src/lib/orionld/common/orionldState.h
Original file line number Diff line number Diff line change
Expand Up @@ -590,6 +590,8 @@ extern bool debugCurl; // From orionld.cpp
extern bool noCache; // From orionld.cpp
extern int cSubCounters; // Number of subscription counter updates before flush from sub-cache to DB
extern char localIpAndPort[135]; // Local address for X-Forwarded-For (from orionld.cpp)
extern unsigned long long inReqPayloadMaxSize;
extern unsigned long long outReqMsgMaxSize;



Expand Down
4 changes: 2 additions & 2 deletions src/lib/orionld/rest/orionldMhdConnectionPayloadRead.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ MHD_Result orionldMhdConnectionPayloadRead
size_t dataLen = *upload_data_size;

//
// If the HTTP header says the request is bigger than our PAYLOAD_MAX_SIZE,
// If the HTTP header says the request is bigger than inReqPayloadMaxSize,
// just silently "eat" the entire message.
//
// The problem occurs when the broker is lied to and there aren't orionldState.in.contentLength
Expand All @@ -60,7 +60,7 @@ MHD_Result orionldMhdConnectionPayloadRead
// See github issue:
// https://github.com/telefonicaid/fiware-orion/issues/2761
//
if (orionldState.in.contentLength > PAYLOAD_MAX_SIZE)
if ((unsigned long long) orionldState.in.contentLength > inReqPayloadMaxSize)
{
//
// Errors can't be returned yet, postpone ...
Expand Down
2 changes: 1 addition & 1 deletion src/lib/rest/httpRequestSend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,7 @@ int httpRequestSendWithCurl


// Check if total outgoing message size is too big
if (outgoingMsgSize > MAX_DYN_MSG_SIZE)
if ((unsigned long long) outgoingMsgSize > outReqMsgMaxSize)
{
if (metricsMgr.isOn())
metricsMgr.add(tenant, servicePath0, METRIC_TRANS_OUT_ERRORS, 1);
Expand Down
12 changes: 6 additions & 6 deletions src/lib/rest/rest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1055,10 +1055,10 @@ ConnectionInfo* connectionTreatInit

orionldHeaderAdd(&orionldState.out.headers, HttpCorrelator, orionldState.correlator, 0);

if ((orionldState.in.contentLength > PAYLOAD_MAX_SIZE) && (orionldState.apiVersion == V2))
if (((unsigned long long) orionldState.in.contentLength > inReqPayloadMaxSize) && (orionldState.apiVersion == V2))
{
char details[256];
snprintf(details, sizeof(details), "payload size: %d, max size supported: %d", orionldState.in.contentLength, PAYLOAD_MAX_SIZE);
snprintf(details, sizeof(details), "payload size: %d, max size supported: %llu", orionldState.in.contentLength, inReqPayloadMaxSize);

alarmMgr.badInput(clientIp, details);
OrionError oe(SccRequestEntityTooLarge, details);
Expand Down Expand Up @@ -1198,7 +1198,7 @@ static MHD_Result connectionTreatDataReceive(ConnectionInfo* ciP, size_t* upload
size_t dataLen = *upload_data_size;

//
// If the HTTP header says the request is bigger than our PAYLOAD_MAX_SIZE,
// If the HTTP header says the request is bigger than inReqPayloadMaxSize,
// just silently "eat" the entire message.
//
// The problem occurs when the broker is lied to and there aren't orionldState.in.contentLength
Expand All @@ -1210,7 +1210,7 @@ static MHD_Result connectionTreatDataReceive(ConnectionInfo* ciP, size_t* upload
// See github issue:
// https://github.com/telefonicaid/fiware-orion/issues/2761
//
if (orionldState.in.contentLength > PAYLOAD_MAX_SIZE)
if ((unsigned long long) orionldState.in.contentLength > inReqPayloadMaxSize)
{
//
// Errors can't be returned yet, postpone ...
Expand Down Expand Up @@ -1431,11 +1431,11 @@ static MHD_Result connectionTreat
//
// If the incoming request was too big, return error about it
//
if (orionldState.in.contentLength > PAYLOAD_MAX_SIZE)
if ((unsigned long long) orionldState.in.contentLength > inReqPayloadMaxSize)
{
char details[256];

snprintf(details, sizeof(details), "payload size: %d, max size supported: %d", orionldState.in.contentLength, PAYLOAD_MAX_SIZE);
snprintf(details, sizeof(details), "payload size: %d, max size supported: %llu", orionldState.in.contentLength, inReqPayloadMaxSize);
alarmMgr.badInput(clientIp, details);
restErrorReplyGet(ciP, SccRequestEntityTooLarge, details, &ciP->answer);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ Usage: orionld [option '-U' (extended usage)]
[option '-connectionMemory' <maximum memory size per connection (in kilobytes)>]
[option '-maxConnections' <maximum number of simultaneous connections>]
[option '-reqPoolSize' <size of thread pool for incoming connections>]
[option '-inReqPayloadMaxSize' <maximum size (in bytes) of the payload of incoming requests>]
[option '-outReqMsgMaxSize' <maximum size (in bytes) of outgoing forward and notification request messages>]
[option '-notificationMode' <notification mode (persistent|transient|threadpool:q:n)>]
[option '-simulatedNotification' (simulate notifications instead of actual sending them (only for testing))]
[option '-statCounters' (enable request/notification counters statistics)]
Expand Down
2 changes: 2 additions & 0 deletions test/functionalTest/cases/0000_cli/command_line_options.test
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ Usage: orionld [option '-U' (extended usage)]
[option '-connectionMemory' <maximum memory size per connection (in kilobytes)>]
[option '-maxConnections' <maximum number of simultaneous connections>]
[option '-reqPoolSize' <size of thread pool for incoming connections>]
[option '-inReqPayloadMaxSize' <maximum size (in bytes) of the payload of incoming requests>]
[option '-outReqMsgMaxSize' <maximum size (in bytes) of outgoing forward and notification request messages>]
[option '-notificationMode' <notification mode (persistent|transient|threadpool:q:n)>]
[option '-simulatedNotification' (simulate notifications instead of actual sending them (only for testing))]
[option '-statCounters' (enable request/notification counters statistics)]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U
#
# This file is part of Orion Context Broker.
#
# Orion Context Broker is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Orion Context Broker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
#
# For those usages not covered by this license please contact with
# iot_support at tid dot es

# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh

--NAME--
Large request - starting the broker to accept very large requests

--SHELL-INIT--
dbInit CB
brokerStart CB 0-255 IPV4 -inReqPayloadMaxSize 50000000

--SHELL--
#
# Creating a file of ~40 MB.
# The default max size of a request is 1 MB but the broker is started with -inReqPayloadMaxSize 50000000 (~50 MB), and a ~40 MB request must be permitted
#
HUGEFILE=$(mktemp)
tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c 40000000 > $HUGEFILE
#
# Note that we are not including an actual JSON payload, but in this test what matters
# is the size, not the format of the data.
#
# As the HUGE payload is accepted, size-wise, it will be parsed and a Parse Error will be the result
#

echo "01. Sending a request of ~40 MB, to a broker that excepts request upto ~50MB - see parse error as a result"
echo "=========================================================================================================="
orionCurl --url "/v2/entities" --payload "$HUGEFILE"
echo
echo


echo "02. Sending a request of ~60 MB, to a broker that excepts request upto ~50MB - see 413 error"
echo "============================================================================================"
tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c 20000000 >> $HUGEFILE
orionCurl --url "/v2/entities" --payload "$HUGEFILE"
rm $HUGEFILE
echo
echo


--REGEXPECT--
01. Sending a request of ~40 MB, to a broker that excepts request upto ~50MB - see parse error as a result
==========================================================================================================
HTTP/1.1 400 Bad Request
Content-Length: 75
Content-Type: application/json
Date: REGEX(.*)
Fiware-Correlator: REGEX([0-9a-f\-]{36})

{
"description": "Errors found in incoming JSON buffer",
"error": "ParseError"
}


02. Sending a request of ~60 MB, to a broker that excepts request upto ~50MB - see 413 error
============================================================================================
HTTP/1.1 413 Content Too Large
Content-Length: 102
Content-Type: application/json
Date: REGEX(.*)
Fiware-Correlator: REGEX([0-9a-f\-]{36})

{
"description": "payload size: 60000000, max size supported: 50000000",
"error": "RequestEntityTooLarge"
}


--TEARDOWN--
brokerStop CB
dbDrop CB
3 changes: 3 additions & 0 deletions test/unittests/main_UnitTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ bool mongocOnly = false;
bool debugCurl = false;
int cSubCounters = 0;
char localIpAndPort[135];
unsigned long long inReqPayloadMaxSize;
unsigned long long outReqMsgMaxSize;



/* ****************************************************************************
Expand Down

0 comments on commit 2d4ab7f

Please sign in to comment.